42 lines
1.6 KiB
R
42 lines
1.6 KiB
R
library(data.table)
|
|
library(MASS)
|
|
|
|
set.seed(1111)
|
|
|
|
scores <- fread("perspective_scores.csv")
|
|
scores <- scores[,id:=as.character(id)]
|
|
|
|
df <- fread("all_data.csv")
|
|
|
|
# only use the data that has identity annotations
|
|
df <- df[identity_annotator_count > 0]
|
|
|
|
(df[!(df$id %in% scores$id)])
|
|
|
|
df <- df[scores,on='id',nomatch=NULL]
|
|
|
|
df[, ":="(identity_attack_pred = identity_attack_prob >=0.5,
|
|
insult_pred = insult_prob >= 0.5,
|
|
profanity_pred = profanity_prob >= 0.5,
|
|
severe_toxicity_pred = severe_toxicity_prob >= 0.5,
|
|
threat_pred = threat_prob >= 0.5,
|
|
toxicity_pred = toxicity_prob >= 0.5,
|
|
identity_attack_coded = identity_attack >= 0.5,
|
|
insult_coded = insult >= 0.5,
|
|
profanity_coded = obscene >= 0.5,
|
|
severe_toxicity_coded = severe_toxicity >= 0.5,
|
|
threat_coded = threat >= 0.5,
|
|
toxicity_coded = toxicity >= 0.5
|
|
)]
|
|
|
|
gt.0.5 <- function(v) { v >= 0.5 }
|
|
dt.apply.any <- function(fun, ...){apply(apply(cbind(...), 2, fun),1,any)}
|
|
|
|
df <- df[,":="(gender_disclosed = dt.apply.any(gt.0.5, male, female, transgender, other_gender),
|
|
sexuality_disclosed = dt.apply.any(gt.0.5, heterosexual, bisexual, other_sexual_orientation),
|
|
religion_disclosed = dt.apply.any(gt.0.5, christian, jewish, hindu, buddhist, atheist, muslim, other_religion),
|
|
race_disclosed = dt.apply.any(gt.0.5, white, black, asian, latino, other_race_or_ethnicity),
|
|
disability_disclosed = dt.apply.any(gt.0.5,physical_disability, intellectual_or_learning_disability, psychiatric_or_mental_illness, other_disability))]
|
|
|
|
df <- df[,white:=gt.0.5(white)]
|