From 35555428620807a22344364f69fda6992472a9db Mon Sep 17 00:00:00 2001 From: Nathan TeBlunthuis Date: Mon, 30 Dec 2024 16:10:50 -0800 Subject: [PATCH] use min/max df constraints in counting nterms. --- similarities/weekly_cosine_similarities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/similarities/weekly_cosine_similarities.py b/similarities/weekly_cosine_similarities.py index 2a735c7..47134c8 100755 --- a/similarities/weekly_cosine_similarities.py +++ b/similarities/weekly_cosine_similarities.py @@ -83,7 +83,7 @@ def cosine_similarities_weekly(tfidf_path, outfile, term_colname, included_subre subreddit_names = conn.execute(f"SELECT DISTINCT subreddit, subreddit_id from read_parquet('{tfidf_path}/*/*.parquet') ORDER BY subreddit_id;").df() if static_tfidf_path is not None: - nterms = conn.execute(f"SELECT COUNT(DISTINCT({term_colname + '_id'})) as nterms FROM read_parquet('{static_tfidf_path}/*.parquet')").df() + nterms = conn.execute(f"SELECT COUNT(DISTINCT({term_colname + '_id'})) as nterms FROM read_parquet('{static_tfidf_path}/*.parquet') WHERE count >= {min_df} AND count <={max_df}").df() else: nterms = conn.execute(f"SELECT MAX({term_colname + '_id'}) as nterms FROM read_parquet('{tfidf_path}/*/*.parquet')").df() nterms = nterms.nterms.values