groceryheist/ml_measurement_error_public
1
0
Fork 0
Code

git-annex in ntq8312@kibo:/data/ntq8312/ml_measurement_error_public

Browse Source
This commit is contained in:
ntq8312 2022-11-09 16:18:48 -06:00
parent 5c931a7198
commit c42b94110b
2 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
civil_comments/get_perspective_scores.py
View File

@ -11,16 +11,17 @@ API_KEY = open('perspective_api_key').read()
client = discovery.build("commentanalyzer","v1alpha",developerKey=API_KEY,discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",static_discovery=False,) client = discovery.build("commentanalyzer","v1alpha",developerKey=API_KEY,discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",static_discovery=False,)
csvreader = csv.DictReader(open("all_data.csv")) csvreader = csv.DictReader(open("all_data.csv"), dialect='unix')
outfile = Path("perspective_results.json") outfile = Path("perspective_results.json")
already_scored = set() already_scored = set()
if outfile.exists(): if outfile.exists():
already_scored = set([json.loads(l)['id'] for l in open(str(outfile),'r')]) already_scored = set([json.loads(l)['id'] for l in open(str(outfile),'r')])
with open("perspective_results.json",'w') as of: print(f"loaded {len(already_scored)} scored comments")
with open("perspective_results.json",'a') as of:
for line in csvreader: for line in csvreader:
if line['id'] not in already_scored: if line['id'] not in already_scored and len(line.get('comment_text','')) > 0:
analyze_request = {'comment':{'text':line['comment_text']}, analyze_request = {'comment':{'text':line['comment_text']},
'languages':['en'], 'languages':['en'],
'requestedAttributes':{'TOXICITY':{}, 'requestedAttributes':{'TOXICITY':{},

1
civil_comments/perspective_results.json Symbolic link
View File

@ -0,0 +1 @@
../.git/annex/objects/ZV/z8/SHA256E-s2293825121--6cdc8f8fb64fad2e51027e2564928e8938bf5fc6ca0cd6c31cb2e67aafe0a203.json/SHA256E-s2293825121--6cdc8f8fb64fad2e51027e2564928e8938bf5fc6ca0cd6c31cb2e67aafe0a203.json