From 0bbdc6bd5e468d3897ba5252564befd8422b9181 Mon Sep 17 00:00:00 2001 From: Nathan TeBlunthuis Date: Thu, 19 Dec 2024 20:09:00 -0800 Subject: [PATCH] typo fix. --- similarities/tfidf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/similarities/tfidf.py b/similarities/tfidf.py index 9617ddd..fd255f9 100755 --- a/similarities/tfidf.py +++ b/similarities/tfidf.py @@ -8,7 +8,7 @@ from functools import partial def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_subreddits, included_terms=None, min_df=None, max_df=None): spark = SparkSession.builder.config(map={'spark.executor.memory':'900g','spark.executor.cores':128}).getOrCreate() df = spark.read.parquet(inpath) - df = df.repartition(128*15, cols=['subreddit',term_colname]) + df = df.repartition(128*15, col=['subreddit',term_colname]) df = df.filter(~ f.col(term_colname).isin(exclude)) if included_subreddits is not None: