Check the shas when we download dumps
This commit is contained in:
parent
33e088492c
commit
9cd0954288
31
check_comments_shas.py
Normal file
31
check_comments_shas.py
Normal file
@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
# run from a build_machine
|
||||
|
||||
import requests
|
||||
from os import path
|
||||
import hashlib
|
||||
|
||||
shasums = requests.get("https://files.pushshift.io/reddit/comments/sha256sums.txt").text
|
||||
|
||||
dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments"
|
||||
|
||||
for l in shasums.strip().split('\n'):
|
||||
sha256_hash = hashlib.sha256()
|
||||
parts = l.split(' ')
|
||||
|
||||
correct_sha256 = parts[0]
|
||||
filename = parts[-1]
|
||||
print(f"checking {filename}")
|
||||
fpath = path.join(dumpdir,filename)
|
||||
if path.isfile(fpath):
|
||||
with open(fpath,'rb') as f:
|
||||
for byte_block in iter(lambda: f.read(4096),b""):
|
||||
sha256_hash.update(byte_block)
|
||||
|
||||
if sha256_hash.hexdigest() == correct_sha256:
|
||||
print(f"{filename} checks out")
|
||||
else:
|
||||
print(f"ERROR! {filename} has the wrong hash. Redownload and recheck!")
|
||||
else:
|
||||
print(f"Skipping {filename} as it doesn't exist")
|
||||
|
@ -8,5 +8,4 @@ wget -r --no-parent -A 'RC_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_
|
||||
wget -r --no-parent -A 'RC_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url
|
||||
wget -r --no-parent -A 'RC_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url
|
||||
|
||||
|
||||
|
||||
./check_comment_shas.py
|
||||
|
@ -11,4 +11,4 @@ wget -r --no-parent -A 'RS_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_
|
||||
wget -r --no-parent -A 'RS_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/
|
||||
wget -r --no-parent -A 'RS_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/
|
||||
|
||||
|
||||
./check_submissions_shas.py
|
||||
|
Loading…
Reference in New Issue
Block a user