groceryheist/cdsc_reddit
1
0
Fork 0
Code

more print debugging

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-28 19:27:42 -08:00
parent c317ef6475
commit e2e7d7dbb1
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
similarities/weekly_cosine_similarities.py
View File

@ -84,13 +84,14 @@ def cosine_similarities_weekly(tfidf_path, outfile, term_colname, included_subre
nterms = conn.execute(f"SELECT MAX({term_colname + '_id'}) as nterms FROM read_parquet('{tfidf_path}/*/*.parquet')").df() nterms = conn.execute(f"SELECT MAX({term_colname + '_id'}) as nterms FROM read_parquet('{tfidf_path}/*/*.parquet')").df()
nterms = nterms.nterms.values nterms = nterms.nterms.values
print(nterms)
print(int(nterms[0]))
weeks = conn.execute(f"SELECT DISTINCT week FROM read_parquet('{tfidf_path}/*/*.parquet')").df() weeks = conn.execute(f"SELECT DISTINCT week FROM read_parquet('{tfidf_path}/*/*.parquet')").df()
weeks = weeks.week.values weeks = weeks.week.values
conn.close() conn.close()
print(f"computing weekly similarities") print(f"computing weekly similarities")
week_similarities_helper = partial(_week_similarities,simfunc=simfunc, tfidf_path=tfidf_path, term_colname=term_colname, outdir=outfile, min_df=min_df, max_df=max_df, included_subreddits=included_subreddits, topN=None, subreddit_names=subreddit_names,nterms=int(nterms[0])) week_similarities_helper = partial(_week_similarities,simfunc=simfunc, tfidf_path=tfidf_path, term_colname=term_colname, outdir=outfile, min_df=min_df, max_df=max_df, included_subreddits=included_subreddits, topN=None, subreddit_names=subreddit_names,nterms=nterms)
for week in weeks: for week in weeks:
week_similarities_helper(week) week_similarities_helper(week)