groceryheist/cdsc_reddit
1
0
Fork 0
Code

bugfix.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-03 19:18:38 -08:00
parent 0436450ea8
commit 39c0fa7a29
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ngrams/term_frequencies.py
View File

@ -62,7 +62,7 @@ def my_tokenizer(text, mwe_pass, mwe_tokenize, stopWords, ngram_output):
for sentence in sentences: for sentence in sentences:
if random() <= 0.1: if random() <= 0.1:
grams = list(chain(*map(lambda i : ngrams(sentence,i),range(4)))) grams = list(chain(*map(lambda i : ngrams(sentence,i),range(4))))
Path(ngram_output).mkdir(parents=True, exist_ok=True) Path(ngram_output).parent.mkdir(parents=True, exist_ok=True)
with open(ngram_output,'a') as gram_file: with open(ngram_output,'a') as gram_file:
for ng in grams: for ng in grams:
gram_file.write(' '.join(ng) + '\n') gram_file.write(' '.join(ng) + '\n')