groceryheist/cdsc_reddit
1
0
Fork 0
Code

enforce min_df constraint in counting lsi features.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-30 16:17:31 -08:00
parent 3555542862
commit 79d1826ba4
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
similarities/weekly_cosine_similarities.py
View File

@ -75,7 +75,7 @@ def cosine_similarities_weekly_lsi(*args, n_components=100, lsi_model=None, **kw
return cosine_similarities_weekly(*args, simfunc=simfunc, **kwargs)
#tfidf = spark.read.parquet('/gscratch/comdata/users/nathante/competitive_exclusion_reddit/data/tfidf_weekly/comment_submission_terms_tfidf.parquet')
def cosine_similarities_weekly(tfidf_path, outfile, term_colname, included_subreddits = None, topN = None, simfunc=column_similarities, min_df=None, max_df=None, static_tfidf_path=None):
def cosine_similarities_weekly(tfidf_path, outfile, term_colname, included_subreddits = None, topN = None, simfunc=column_similarities, min_df=0, max_df=None, static_tfidf_path=None):
print(outfile)
# do this step in parallel if we have the memory for it.
# should be doable with pool.map
@ -83,7 +83,11 @@ def cosine_similarities_weekly(tfidf_path, outfile, term_colname, included_subre
subreddit_names = conn.execute(f"SELECT DISTINCT subreddit, subreddit_id from read_parquet('{tfidf_path}/*/*.parquet') ORDER BY subreddit_id;").df()
if static_tfidf_path is not None:
nterms = conn.execute(f"SELECT COUNT(DISTINCT({term_colname + '_id'})) as nterms FROM read_parquet('{static_tfidf_path}/*.parquet') WHERE count >= {min_df} AND count <={max_df}").df()
q = f"SELECT COUNT(DISTINCT({term_colname + '_id'})) as nterms FROM read_parquet('{static_tfidf_path}/*.parquet') WHERE count >= {min_df}"
if max_df is not None:
q = q + f"AND count <= {max_df}"
nterms = conn.execute(q).df()
else:
nterms = conn.execute(f"SELECT MAX({term_colname + '_id'}) as nterms FROM read_parquet('{tfidf_path}/*/*.parquet')").df()
nterms = nterms.nterms.values
@ -111,7 +115,8 @@ def author_cosine_similarities_weekly(outfile, infile='/gscratch/comdata/output/
max_df,
included_subreddits,
topN,
min_df=2,
min_df=min_df,
max_df=max_df,
static_tfidf_path=static_tfidf_path
)
@ -126,14 +131,15 @@ def term_cosine_similarities_weekly(outfile, infile='/gscratch/comdata/output/re
static_tfidf_path=static_tfidf_path)
def author_cosine_similarities_weekly_lsi(outfile, infile = '/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors_test.parquet', included_subreddits=None, n_components=100,lsi_model=None,static_tfidf_path=None):
def author_cosine_similarities_weekly_lsi(outfile, infile = '/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors_test.parquet', included_subreddits=None, n_components=100,lsi_model=None,static_tfidf_path=None, min_df=2):
return cosine_similarities_weekly_lsi(infile,
outfile,
'author',
included_subreddits=included_subreddits,
n_components=n_components,
lsi_model=lsi_model,
static_tfidf_path=static_tfidf_path
static_tfidf_path=static_tfidf_path,
min_df=min_df
)