mgaughan/24_deb_pkg_gov
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backing up updates to octo

Browse Source
This commit is contained in:
Matthew Gaughan 2024-01-30 17:56:47 -06:00
parent ad719470e6
commit b1fc221075
1 changed files with 60 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
non_coding_measures_scrape.py
View File

@ -5,51 +5,76 @@ import sys
import subprocess import subprocess
import csv import csv
import ast import ast
import time
#sys.path.append('../octohatrack') #sys.path.append('../octohatrack')
#import octohatrack.__main__ as oh #import octohatrack.__main__ as oh
csv_path = "011824_uni_contrib.csv" csv_path = "013024_octo_data.csv"
csv_013024_path = "013024_te_rows.csv"
def main(): def main():
wd = os.getcwd() wd = os.getcwd()
with open(csv_path, 'w', newline='') as output_file: with open(csv_path, 'w', newline='') as output_file:
keys = ["project_name", "project_owner", "api_contrib_count", "issue_contrib_count", "file_contrib_count", "wiki_contrib_count"] keys = ["project_name","underproduction_mean","underproduction_low","underproduction_high","debian_vcs_link","upstream_vcs_link","age_of_project","contributors","collaborators","milestone_count", "api_contrib_count", "issue_contrib_count", "file_contrib_count", "wiki_contrib_count", ]
dict_writer = csv.DictWriter(output_file, keys) dict_writer = csv.DictWriter(output_file, keys)
dict_writer.writeheader() dict_writer.writeheader()
with open("expanded_data_final.csv", "r") as file: with open(csv_013024_path, 'w', newline='') as output_file:
reader = csv.reader(file) keys = ["project_name","underproduction_mean","underproduction_low","underproduction_high","debian_vcs_link","upstream_vcs_link","age_of_project","contributors","collaborators","milestone_count"]
for i, line in enumerate(reader): dict_writer2 = csv.DictWriter(output_file, keys)
project_dict ={} dict_writer2.writeheader()
if "github" not in line[5]: with open("kk_final_expanded_data_final.csv", "r") as file:
continue #with open("013024_te_rows.csv", "r") as file:
repo_name = line[5].strip()[19:] reader = csv.reader(file)
print(repo_name) for i, line in enumerate(reader):
project_dict["project_name"] = repo_name.split("/")[1] time.sleep(5)
project_dict["project_owner"]= repo_name.split("/")[0] project_dict ={}
os.chdir("../octohatrack") if "github" not in line[5]:
try: continue
octohatrack_results = subprocess.run(['python3', '-m', 'octohatrack', repo_name, '--wait-for-reset'], capture_output = True, text=True, timeout=60).stdout repo_name = line[5].strip()[19:]
except: if repo_name[-4:] == ".git":
print("issue with the repository, string indices must be integers") repo_name = repo_name[:-4]
continue print(repo_name)
os.chdir(wd) project_dict["project_name"] = repo_name.split("/")[1]
split_results = octohatrack_results.split("\n") project_dict['underproduction_mean'] = line[1]
for entry in split_results: project_dict['underproduction_low'] = line[2]
if "{'api_contributors'" in entry: project_dict['underproduction_high'] = line[3]
formatted = ast.literal_eval(entry) project_dict['debian_vcs_link'] = line[4]
project_dict["api_contrib_count"] = len(formatted['api_contributors']) project_dict['upstream_vcs_link'] = line[5]
#project_dict["api_contrib_list"] = formatted['api_contributors'] project_dict['age_of_project'] = line[6]
project_dict["issue_contrib_count"] = len(formatted['issue_pr_contributors']) project_dict['contributors'] = line[7]
#project_dict["issue_contrib_list"] = formatted['issue_pr_contributors'] project_dict['collaborators'] = line[8]
project_dict["file_contrib_count"] = len(formatted['file_contributors']) project_dict['milestone_count'] = line[9]
#project_dict["file_contrib_list"] = formatted['file_contributors'] #print(project_dict)
project_dict["wiki_contrib_count"] = len(formatted['file_contributors']) os.chdir(wd)
#project_dict["wiki_contrib_list"] = formatted['file_contributors'] os.chdir("../octohatrack")
print(project_dict) ##--- FAILS FOR:
dict_writer.writerow(project_dict) ## - archived GitHub things
with open('/data/users/mgaughan/b_kkex_contrib_uni_011824/' + 'contrib_roster_' + project_dict["project_name"] + '.json', 'w') as data_path: ## - not github hosted
json.dump(formatted, data_path) try:
octohatrack_results = subprocess.run(['python3', '-m', 'octohatrack', repo_name], capture_output = True, text=True, timeout=60).stdout
except (subprocess.TimeoutExpired, TypeError) as e:
dict_writer2.writerow(project_dict)
print(e)
continue
os.chdir(wd)
split_results = octohatrack_results.split("\n")
for entry in split_results:
if "{'api_contributors'" in entry:
formatted = ast.literal_eval(entry)
#print(formatted)
project_dict["api_contrib_count"] = len(formatted['api_contributors'])
#project_dict["api_contrib_list"] = formatted['api_contributors']
project_dict["issue_contrib_count"] = len(formatted['issue_pr_contributors'])
#project_dict["issue_contrib_list"] = formatted['issue_pr_contributors']
project_dict["file_contrib_count"] = len(formatted['file_contributors'])
#project_dict["file_contrib_list"] = formatted['file_contributors']
project_dict["wiki_contrib_count"] = len(formatted['wiki_contributors'])
#project_dict["wiki_contrib_list"] = formatted['file_contributors']
print(project_dict)
dict_writer.writerow(project_dict)
with open('/data/users/mgaughan/b_kkex_contrib_uni_013024/' + 'contrib_roster_' + project_dict["project_name"] + '.json', 'w') as data_path:
json.dump(formatted, data_path)
if __name__ == "__main__": if __name__ == "__main__":
main() main()