groceryheist/cdsc_reddit
1
0
Fork 0
Code Activity

git-annex in

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2022-04-06 11:11:11 -07:00
parent 98c1317af5
commit 197518a222
19 changed files with 260 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
dumps/check_comments_shas.py
View File

@@ -8,7 +8,7 @@ import hashlib
shasums1 = requests.get("https://files.pushshift.io/reddit/comments/sha256sum.txt").text
#shasums2 = requests.get("https://files.pushshift.io/reddit/comments/daily/sha256sum.txt").text
shasums = shasums1 + shasums2
shasums = shasums1
dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments"
for l in shasums.strip().split('\n'):