collective/cdsc_reddit
18
0
Fork 0
Code Activity

support passing in list of tfidf vectors.

Browse Source 
Also lowercases included subreddits.
This commit is contained in:
Nate E TeBlunthuis
2021-04-26 11:16:28 -07:00
parent 003a48aea5
commit 0fe120e4ab
3 changed files with 21 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
similarities/tfidf.py
View File

@@ -11,13 +11,13 @@ def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_
df = df.filter(~ f.col(term_colname).isin(exclude))
if included_subreddits is not None:
include_subs = list(open(included_subreddits))
include_subs = set(map(str.strip,map(str.lower, open(included_subreddits))))
else:
include_subs = select_topN_subreddits(topN)
df = func(df, include_subs, term_colname)
df.write.parquet(outpath,mode='overwrite',compression='snappy')
df.write.parquet(outpath,mode='overwrite',copmression='snappy')
spark.stop()
@@ -28,40 +28,44 @@ def tfidf_weekly(inpath, outpath, topN, term_colname, exclude, included_subreddi
return _tfidf_wrapper(build_weekly_tfidf_dataset, inpath, outpath, topN, term_colname, exclude, included_subreddits)
def tfidf_authors(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet',
topN=25000):
topN=25000,
included_subreddits=None):
return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
outpath,
topN,
'author',
['[deleted]','AutoModerator'],
included_subreddits=None
included_subreddits=included_subreddits
)
def tfidf_terms(outpath='/gscratch/comdata/output/reddit_similarity/tfidf/comment_terms.parquet',
topN=25000):
topN=25000,
included_subreddits=None):
return tfidf("/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
outpath,
topN,
'term',
[],
included_subreddits=None
included_subreddits=included_subreddits
)
def tfidf_authors_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_authors.parquet',
topN=25000):
topN=25000,
included_subreddits=None):
return tfidf_weekly("/gscratch/comdata/output/reddit_ngrams/comment_authors.parquet",
outpath,
topN,
'author',
['[deleted]','AutoModerator'],
included_subreddits=None
included_subreddits=included_subreddits
)
def tfidf_terms_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf_weekly/comment_terms.parquet',
topN=25000):
topN=25000,
included_subreddits=None):
return tfidf_weekly("/gscratch/comdata/output/reddit_ngrams/comment_terms.parquet",
@@ -69,7 +73,7 @@ def tfidf_terms_weekly(outpath='/gscratch/comdata/output/reddit_similarity/tfidf
topN,
'term',
[],
included_subreddits=None
included_subreddits=included_subreddits
)