2020-07-07 06:31:52 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
# run from a build_machine
|
|
|
|
|
|
|
|
|
|
import requests
|
|
|
|
|
from os import path
|
|
|
|
|
import hashlib
|
|
|
|
|
|
2020-10-03 23:42:22 +00:00
|
|
|
shasums1 = requests.get("https://files.pushshift.io/reddit/comments/sha256sum.txt").text
|
2021-12-11 05:23:32 +00:00
|
|
|
#shasums2 = requests.get("https://files.pushshift.io/reddit/comments/daily/sha256sum.txt").text
|
2020-07-07 06:31:52 +00:00
|
|
|
|
2022-04-06 18:11:11 +00:00
|
|
|
shasums = shasums1
|
2020-07-07 06:31:52 +00:00
|
|
|
dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments"
|
|
|
|
|
|
|
|
|
|
for l in shasums.strip().split('\n'):
|
|
|
|
|
sha256_hash = hashlib.sha256()
|
|
|
|
|
parts = l.split(' ')
|
|
|
|
|
|
|
|
|
|
correct_sha256 = parts[0]
|
|
|
|
|
filename = parts[-1]
|
|
|
|
|
print(f"checking {filename}")
|
|
|
|
|
fpath = path.join(dumpdir,filename)
|
|
|
|
|
if path.isfile(fpath):
|
|
|
|
|
with open(fpath,'rb') as f:
|
|
|
|
|
for byte_block in iter(lambda: f.read(4096),b""):
|
|
|
|
|
sha256_hash.update(byte_block)
|
|
|
|
|
|
|
|
|
|
if sha256_hash.hexdigest() == correct_sha256:
|
|
|
|
|
print(f"{filename} checks out")
|
|
|
|
|
else:
|
|
|
|
|
print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!")
|
|
|
|
|
else:
|
|
|
|
|
print(f"Skipping {filename} as it doesn't exist")
|
|
|
|
|
|
