ml_measurement_error_public/civil_comments/load_perspective_data.R

library(data.table)
library(MASS)

set.seed(1111)

scores <- fread("perspective_scores.csv")
scores <- scores[,id:=as.character(id)]

df <- fread("all_data.csv")

# only use the data that has identity annotations
df <- df[identity_annotator_count > 0]

(df[!(df$id %in% scores$id)])

df <- df[scores,on='id',nomatch=NULL]

df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
          insult_pred = insult_prob >= 0.5,
          profanity_pred = profanity_prob >= 0.5,
          severe_toxicity_pred = severe_toxicity_prob >= 0.5,
          threat_pred = threat_prob >= 0.5,
          toxicity_pred = toxicity_prob >= 0.5,
          identity_attack_coded = identity_attack >= 0.5,
          insult_coded = insult >= 0.5,
          profanity_coded = obscene >= 0.5,
          severe_toxicity_coded = severe_toxicity >= 0.5,
          threat_coded = threat >= 0.5,
          toxicity_coded = toxicity >= 0.5
          )]

gt.0.5 <- function(v) { v >= 0.5 }
dt.apply.any <- function(fun, ...){apply(apply(cbind(...), 2, fun),1,any)}

df <- df[,":="(gender_disclosed = dt.apply.any(gt.0.5, male, female, transgender, other_gender),
               sexuality_disclosed = dt.apply.any(gt.0.5, heterosexual, bisexual, other_sexual_orientation),
               religion_disclosed = dt.apply.any(gt.0.5, christian, jewish, hindu, buddhist, atheist, muslim, other_religion),
               race_disclosed = dt.apply.any(gt.0.5, white, black, asian, latino, other_race_or_ethnicity), 
               disability_disclosed = dt.apply.any(gt.0.5,physical_disability, intellectual_or_learning_disability, psychiatric_or_mental_illness, other_disability))]

df <- df[,white:=gt.0.5(white)]


F1 <- function(y, predictions){
    tp <- sum( (predictions == y) & (predictions==1))
    fn <- sum( (predictions != y) & (predictions!=1))
    fp <- sum( (predictions != y) & (predictions==1))
    precision <- tp / (tp + fp)
    recall <- tp / (tp + fn)
    return (2 * precision * recall ) / (precision + recall)
}


## toxicity is about 93% accurate, with an f1 of 0.8
## identity_attack has high accuracy 97%, but an unfortunant f1 of 0.5.
## threat has high accuracy 99%, but a really bad looking f1 of 0.48.
accuracies <- df[,.(identity_attack_acc = mean(identity_attack_pred == identity_attack_coded),
                    insult_pred_acc = mean(insult_pred == insult_coded),
                    profanity_acc = mean(profanity_pred == profanity_coded),
                    severe_toxicity_acc = mean(severe_toxicity_pred == severe_toxicity_coded),
                    theat_acc = mean(threat_pred == threat_coded),
                    toxicity_acc = mean(toxicity_pred == toxicity_coded))]

f1s <- df[,.(identity_attack_f1 = F1(identity_attack_coded,identity_attack_pred),
                    insult_f1 = F1(insult_coded,insult_pred),
                    profanity_f1 = F1(profanity_coded,profanity_pred),
                    severe_toxicity_f1 = F1(severe_toxicity_coded,severe_toxicity_pred),
                    theat_f1 = F1(threat_coded,threat_pred),
                    toxicity_f1 = F1(toxicity_coded,toxicity_pred))]

positive_cases <- df[,.(identity_attacks = sum(identity_attack_coded),
                        insults = sum(insult_coded),
                        profanities = sum(profanity_coded),
                        severe_toxic_comments = sum(severe_toxicity_coded),
                        threats = sum(threat_coded),
                        toxic_comments = sum(toxicity_coded))]

## there are 50,000 toxic comments, 13000 identity attacks, 30000 insults, 3000 profanities, 8 severe toxic, and 1000 threats.

proportions_cases <- df[,.(prop_identity = mean(identity_attack_coded),
                           prop_insults = mean(insult_coded),
                           prop_profanity = mean(profanity_coded),
                           prop_severe = mean(severe_toxicity_coded),
                           prop_threats = mean(threat_coded),
                           prop_toxic = mean(toxicity_coded))]

## at 11% of comments, "toxicity" seems not so badly skewed. Try toxicity first, and if it doesn't work out try insults.

## now look for an example where differential error affects an identity, or a reaction.
df <- df[,":="(identity_error = identity_attack_coded - identity_attack_pred,
               insult_error = insult_coded - insult_pred,
               profanity_error = profanity_coded - profanity_pred,
               severe_toxic_error = severe_toxicity_coded - severe_toxicity_pred,
               threat_error = threat_coded - threat_pred,
               toxicity_error = toxicity_coded - toxicity_pred)]

## what's correlated with toxicity_error ?
df <- df[,approved := rating == "approved"]
df <- df[,white := white > 0.5]

cortab <- cor(df[,.(toxicity_error,
                    identity_error,
                    toxicity_coded,
                    funny,
                    approved,
                    sad,
                    wow,
                    likes,
                    disagree,
                    male,
                    female,
                    transgender,
                    other_gender,
                    heterosexual,
                    bisexual,
                    other_sexual_orientation,
                    christian,
                    jewish,
                    hindu,
                    buddhist,
                    atheist,
                    other_religion,
                    black,
                    white,
                    asian,
                    latino,
                    other_race_or_ethnicity,
                    physical_disability,
                    intellectual_or_learning_disability,
                    psychiatric_or_mental_illness,
                    other_disability,
                    gender_disclosed,
                    sexuality_disclosed,
                    religion_disclosed,
                    race_disclosed,
                    disability_disclosed)])