From ed54ab2decb462d8f38e4d266e944329335f9ac6 Mon Sep 17 00:00:00 2001 From: Matthew Gaughan Date: Fri, 17 May 2024 09:41:40 -0500 Subject: [PATCH] assembling data for qual interp --- text_analysis/contribModel.py | 21 +++++++++++++++++---- text_analysis/topicModel.py | 21 +++++++++++++++++---- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/text_analysis/contribModel.py b/text_analysis/contribModel.py index 012b3f3..7bef892 100644 --- a/text_analysis/contribModel.py +++ b/text_analysis/contribModel.py @@ -139,11 +139,24 @@ def prevalent_topics(vect_documents, file_list): else: count_of_multiple += 1 topic_arrays.append(topic_distribution) - most_frequent(top_topic) + #most_frequent(top_topic) print(count_of_multiple) df = pd.DataFrame(topic_arrays) - averages = df.mean() - print(averages) + #print(df.sort_values(by=['0']).head(5)) + for i in range(4): + print("-----------------------Topic " + str(i) + " --------------------------------") + top5 = df.nlargest(10, i) + top_indices = top5.index.to_list() + print(top5) + for index in top_indices: + print(file_list[index]) + bottom5 = df.nsmallest(10, i) + bottom_indices = bottom5.index.to_list() + print(bottom5) + for index in bottom_indices: + print(file_list[index]) + #averages = df.mean() + #print(averages) def most_frequent(topic_prevalence): most_frequent_array = [] @@ -157,7 +170,7 @@ def most_frequent(topic_prevalence): if __name__ == "__main__": #eadme_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_readme/p1" - contributing_directory = "/data/users/mgaughan/kkex//time_specific_files/partitioned_contributing/p2" + contributing_directory = "/data/users/mgaughan/kkex//time_specific_files/contributing3" listed_corpus, wordcounts, wordlengths, file_list = get_data_from_dir(contributing_directory) print("Mean wordcount: ", mean(wordcounts)) print("Median wordcount: ", median(wordcounts)) diff --git a/text_analysis/topicModel.py b/text_analysis/topicModel.py index f893c45..4a1a589 100644 --- a/text_analysis/topicModel.py +++ b/text_analysis/topicModel.py @@ -123,7 +123,7 @@ def get_most_prevalent(distributions, documents): return most_prevalent def prevalent_topics(vect_documents, file_list): - lda = joblib.load('0509_lda.jl') + lda = joblib.load('0509_readme_lda.jl') distributions = lda.transform(vect_documents) #figuring out what the max distribution is and then figuring out the mode top_topic = [] @@ -140,8 +140,21 @@ def prevalent_topics(vect_documents, file_list): #most_frequent(top_topic) print(count_of_multiple) df = pd.DataFrame(topic_arrays) - averages = df.mean() - print(averages) + #print(df.sort_values(by=['0']).head(5)) + for i in range(8): + print("-----------------------Topic " + str(i) + " --------------------------------") + top5 = df.nlargest(10, i) + top_indices = top5.index.to_list() + print(top5) + for index in top_indices: + print(file_list[index]) + bottom5 = df.nsmallest(10, i) + bottom_indices = bottom5.index.to_list() + print(bottom5) + for index in bottom_indices: + print(file_list[index]) + #averages = df.mean() + #print(averages) def most_frequent(topic_prevalence): most_frequent_array = [] @@ -154,7 +167,7 @@ def most_frequent(topic_prevalence): if __name__ == "__main__": - readme_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_readme/p1" + readme_directory = "/data/users/mgaughan/kkex/time_specific_files/readme3" contributing_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_contributing/p2" listed_corpus, wordcounts, wordlengths, file_list = get_data_from_dir(readme_directory) print("Mean wordcount: ", mean(wordcounts))