mgaughan/24_deb_pkg_gov
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

assembling data for qual interp

Browse Source
This commit is contained in:
Matthew Gaughan 2024-05-17 09:41:40 -05:00
parent 9ba2f30f33
commit ed54ab2dec
2 changed files with 34 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
text_analysis/contribModel.py
View File

@ -139,11 +139,24 @@ def prevalent_topics(vect_documents, file_list):
else: else:
count_of_multiple += 1 count_of_multiple += 1
topic_arrays.append(topic_distribution) topic_arrays.append(topic_distribution)
most_frequent(top_topic) #most_frequent(top_topic)
print(count_of_multiple) print(count_of_multiple)
df = pd.DataFrame(topic_arrays) df = pd.DataFrame(topic_arrays)
averages = df.mean() #print(df.sort_values(by=['0']).head(5))
print(averages) for i in range(4):
print("-----------------------Topic " + str(i) + " --------------------------------")
top5 = df.nlargest(10, i)
top_indices = top5.index.to_list()
print(top5)
for index in top_indices:
print(file_list[index])
bottom5 = df.nsmallest(10, i)
bottom_indices = bottom5.index.to_list()
print(bottom5)
for index in bottom_indices:
print(file_list[index])
#averages = df.mean()
#print(averages)
def most_frequent(topic_prevalence): def most_frequent(topic_prevalence):
most_frequent_array = [] most_frequent_array = []
@ -157,7 +170,7 @@ def most_frequent(topic_prevalence):
if __name__ == "__main__": if __name__ == "__main__":
#eadme_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_readme/p1" #eadme_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_readme/p1"
contributing_directory = "/data/users/mgaughan/kkex//time_specific_files/partitioned_contributing/p2" contributing_directory = "/data/users/mgaughan/kkex//time_specific_files/contributing3"
listed_corpus, wordcounts, wordlengths, file_list = get_data_from_dir(contributing_directory) listed_corpus, wordcounts, wordlengths, file_list = get_data_from_dir(contributing_directory)
print("Mean wordcount: ", mean(wordcounts)) print("Mean wordcount: ", mean(wordcounts))
print("Median wordcount: ", median(wordcounts)) print("Median wordcount: ", median(wordcounts))

21
text_analysis/topicModel.py
View File

@ -123,7 +123,7 @@ def get_most_prevalent(distributions, documents):
return most_prevalent return most_prevalent
def prevalent_topics(vect_documents, file_list): def prevalent_topics(vect_documents, file_list):
lda = joblib.load('0509_lda.jl') lda = joblib.load('0509_readme_lda.jl')
distributions = lda.transform(vect_documents) distributions = lda.transform(vect_documents)
#figuring out what the max distribution is and then figuring out the mode #figuring out what the max distribution is and then figuring out the mode
top_topic = [] top_topic = []
@ -140,8 +140,21 @@ def prevalent_topics(vect_documents, file_list):
#most_frequent(top_topic) #most_frequent(top_topic)
print(count_of_multiple) print(count_of_multiple)
df = pd.DataFrame(topic_arrays) df = pd.DataFrame(topic_arrays)
averages = df.mean() #print(df.sort_values(by=['0']).head(5))
print(averages) for i in range(8):
print("-----------------------Topic " + str(i) + " --------------------------------")
top5 = df.nlargest(10, i)
top_indices = top5.index.to_list()
print(top5)
for index in top_indices:
print(file_list[index])
bottom5 = df.nsmallest(10, i)
bottom_indices = bottom5.index.to_list()
print(bottom5)
for index in bottom_indices:
print(file_list[index])
#averages = df.mean()
#print(averages)
def most_frequent(topic_prevalence): def most_frequent(topic_prevalence):
most_frequent_array = [] most_frequent_array = []
@ -154,7 +167,7 @@ def most_frequent(topic_prevalence):
if __name__ == "__main__": if __name__ == "__main__":
readme_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_readme/p1" readme_directory = "/data/users/mgaughan/kkex/time_specific_files/readme3"
contributing_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_contributing/p2" contributing_directory = "/data/users/mgaughan/kkex/time_specific_files/partitioned_contributing/p2"
listed_corpus, wordcounts, wordlengths, file_list = get_data_from_dir(readme_directory) listed_corpus, wordcounts, wordlengths, file_list = get_data_from_dir(readme_directory)
print("Mean wordcount: ", mean(wordcounts)) print("Mean wordcount: ", mean(wordcounts))