groceryheist/ml_measurement_error_public
1
0
Fork 0
Code

Add exploratory data analysis to come up with a real-data example.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2022-11-29 00:29:42 -08:00
parent c42b94110b
commit 3d1964b806
4 changed files with 168 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
civil_comments/Makefile Normal file
View File

@ -0,0 +1,6 @@
srun_1core=srun -A comdata -p compute-bigmem --mem=4G --time=12:00:00 -c 1 --pty /usr/bin/bash -l
perspective_scores.csv: perspective_json_to_csv.sh perspective_results.json
$(srun_1core) ./$^ $@

156
civil_comments/design_example.R Normal file
View File

@ -0,0 +1,156 @@
library(data.table)
library(MASS)
scores <- fread("perspective_scores.csv")
scores <- scores[,id:=as.character(id)]
df <- fread("all_data.csv")
# only use the data that has identity annotations
df <- df[identity_annotator_count > 0]
(df[!(df$id %in% scores$id)])
df <- df[scores,on='id',nomatch=NULL]
## how accurate are the classifiers?
## the API claims that these scores are "probabilities"
## say we care about the model of the classification, not the probability
F1 <- function(y, predictions){
tp <- sum( (predictions == y) & (predictions==1))
fn <- sum( (predictions != y) & (predictions!=1))
fp <- sum( (predictions != y) & (predictions==1))
precision <- tp / (tp + fp)
recall <- tp / (tp + fn)
return (2 * precision * recall ) / (precision + recall)
}
df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
insult_pred = insult_prob >= 0.5,
profanity_pred = profanity_prob >= 0.5,
severe_toxicity_pred = severe_toxicity_prob >= 0.5,
threat_pred = threat_prob >= 0.5,
toxicity_pred = toxicity_prob >= 0.5,
identity_attack_coded = identity_attack >= 0.5,
insult_coded = insult >= 0.5,
profanity_coded = obscene >= 0.5,
severe_toxicity_coded = severe_toxicity >= 0.5,
threat_coded = threat >= 0.5,
toxicity_coded = toxicity >= 0.5
)]
## toxicity is about 93% accurate, with an f1 of 0.8
## identity_attack has high accuracy 97%, but an unfortunant f1 of 0.5.
## threat has high accuracy 99%, but a really bad looking f1 of 0.48.
accuracies <- df[,.(identity_attack_acc = mean(identity_attack_pred == identity_attack_coded),
insult_pred_acc = mean(insult_pred == insult_coded),
profanity_acc = mean(profanity_pred == profanity_coded),
severe_toxicity_acc = mean(severe_toxicity_pred == severe_toxicity_coded),
theat_acc = mean(threat_pred == threat_coded),
toxicity_acc = mean(toxicity_pred == toxicity_coded))]
f1s <- df[,.(identity_attack_f1 = F1(identity_attack_coded,identity_attack_pred),
insult_f1 = F1(insult_coded,insult_pred),
profanity_f1 = F1(profanity_coded,profanity_pred),
severe_toxicity_f1 = F1(severe_toxicity_coded,severe_toxicity_pred),
theat_f1 = F1(threat_coded,threat_pred),
toxicity_f1 = F1(toxicity_coded,toxicity_pred))]
positive_cases <- df[,.(identity_attacks = sum(identity_attack_coded),
insults = sum(insult_coded),
profanities = sum(profanity_coded),
severe_toxic_comments = sum(severe_toxicity_coded),
threats = sum(threat_coded),
toxic_comments = sum(toxicity_coded))]
## there are 50,000 toxic comments, 13000 identity attacks, 30000 insults, 3000 profanities, 8 severe toxic, and 1000 threats.
proportions_cases <- df[,.(prop_identity = mean(identity_attack_coded),
prop_insults = mean(insult_coded),
prop_profanity = mean(profanity_coded),
prop_severe = mean(severe_toxicity_coded),
prop_threats = mean(threat_coded),
prop_toxic = mean(toxicity_coded))]
## at 11% of comments, "toxicity" seems not so badly skewed. Try toxicity first, and if it doesn't work out try insults.
## now look for an example where differential error affects an identity, or a reaction.
df <- df[,":="(identity_error = identity_attack_coded - identity_attack_pred,
insult_error = insult_coded - insult_pred,
profanity_error = profanity_coded - profanity_pred,
severe_toxic_error = severe_toxicity_coded - severe_toxicity_pred,
threat_error = threat_coded - threat_pred,
toxicity_error = toxicity_coded - toxicity_pred)]
## what's correlated with toxicity_error ?
df <- df[,approved := rating == "approved"]
cortab <- cor(df[,.(toxicity_error,
identity_error,
toxicity_coded,
funny,
approved,
sad,
wow,
likes,
disagree,
male,
female,
transgender,
other_gender,
heterosexual,
bisexual,
other_sexual_orientation,
christian,
jewish,
hindu,
buddhist,
atheist,
other_religion,
black,
white,
asian,
latino,
other_race_or_ethnicity,
physical_disability,
intellectual_or_learning_disability,
psychiatric_or_mental_illness,
other_disability)])
## toxicity error is weakly correlated pearson's R = 0.1 with both "white" and "black".
## compare regressions with "white" or "black" as the outcome and "toxicity_coded" or "toxicity_pred" as a predictor.
## here's a great example with presumambly non-differential error: about what identities is toxicity found humorous?
## a bunch of stars reappear when you used the ground-truth data instead of the predictions.
## pro/con of this example: will have to implement family='poisson'.
## shouldn't be that bad though haha.
cortab['toxicity_error',]
cortab['toxicity_error','funny']
cortab['toxicity_coded',]
cortab['identity_error',]
cortab['white',]
glm(white ~ toxicity_coded + psychiatric_or_mental_illness, data = df, family=binomial(link='logit'))
glm(white ~ toxicity_pred + psychiatric_or_mental_illness, data = df, family=binomial(link='logit'))
m1 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity_coded, data = df)
m2 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity_pred, data = df)
m3 <- glm.nb(funny ~ (male + female + transgender + other_gender + heterosexual + bisexual + other_sexual_orientation + christian + jewish + hindu + buddhist + atheist + other_religion + asian + latino + other_race_or_ethnicity + physical_disability + intellectual_or_learning_disability + white + black + psychiatric_or_mental_illness)*toxicity, data = df)
glm(white ~ disagree, data = df, family=binomial(link='logit'))
## example with differential error
glm(white ~ toxicity_coded + toxicity_error, data=df,family=binomial(link='logit'))
glm(toxicity_coded ~ white, data = df, family=binomial(link='logit'))
glm(toxicity_pred ~ white, data = df, family=binomial(link='logit'))

2
civil_comments/perspective_json_to_csv.jq Executable file
View File

@ -0,0 +1,2 @@
#!/usr/bin/bash
cat $1 | jq "[.attributeScores.IDENTITY_ATTACK.summaryScore.value, .attributeScores.INSULT.summaryScore.value, .attributeScores.PROFANITY.summaryScore.value,.attributeScores.SEVERE_TOXICITY.summaryScore.value, .attributeScores.THREAT.summaryScore.value, .attributeScores.TOXICITY.summaryScore.value] | @csv" > $2

4
civil_comments/perspective_json_to_csv.sh Executable file
View File

@ -0,0 +1,4 @@
#!/usr/bin/bash
header=id,identity_attack_prob,insult_prob,profanity_prob,severe_toxicity_prob,threat_prob,toxicity_prob
echo "$header" > $2
cat $1 | jq -r '[.id, .attributeScores.IDENTITY_ATTACK.summaryScore.value, .attributeScores.INSULT.summaryScore.value, .attributeScores.PROFANITY.summaryScore.value,.attributeScores.SEVERE_TOXICITY.summaryScore.value, .attributeScores.THREAT.summaryScore.value, .attributeScores.TOXICITY.summaryScore.value] | @csv' >> $2