groceryheist/cdsc_reddit
1
0
Fork 0
Code

tweak parallelism in hopes for speed.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2025-01-11 20:22:18 -08:00
parent 4168d0d4cf
commit b2f1c1342f
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
similarities/weekly_cosine_similarities.py
View File

@ -110,8 +110,8 @@ def cosine_similarities_weekly(tfidf_path, outfile, term_colname, included_subre
# for week in weeks: # for week in weeks:
# week_similarities_helper(week) # week_similarities_helper(week)
with Pool(cpu_count()) as pool: # maybe it can be done with 128 cores on the huge machine? with Pool(128) as pool: # maybe it can be done with 128 cores on the huge machine?
list(pool.imap(week_similarities_helper, weeks)) list(pool.imap_unordered(week_similarities_helper, weeks, 5))
pool.close() pool.close()