From eaa84d33e0cc6aba1b1d1a5935ef151bac114900 Mon Sep 17 00:00:00 2001 From: Matthew Gaughan Date: Sat, 16 Mar 2024 06:14:03 -0500 Subject: [PATCH] new file to grab time-specific gov docs --- get_spec_file.py | 66 ++++++++++++++++++++++++++++++++++++++++++++++++ pr_data_get.py | 19 +++++++------- 2 files changed, 76 insertions(+), 9 deletions(-) create mode 100644 get_spec_file.py diff --git a/get_spec_file.py b/get_spec_file.py new file mode 100644 index 0000000..1882953 --- /dev/null +++ b/get_spec_file.py @@ -0,0 +1,66 @@ +import csv +from git import Repo +import os +import datetime as dt +import time +import shutil +import pandas as pd +import dateutil +from tqdm import tqdm +import math +import io + +working_dir = "/data/users/mgaughan/kkex/time_specific_files_readme" +temp_dir = "/data/users/mgaughan/tmp3/" + +# getting the specific readme or contributing file from a given commit +# inputs: upstream vcs link, commit hash, yes/no is it a readme +def get_file(vcs_link, commit_hash, is_readme): + if "github" in vcs_link or "gitlab" in vcs_link: + #making an evaluation that sub branches aren't being used and that people would fork if needed + #this only looks at main + vcs_link = "/".join(vcs_link.split("/")[0:5]) + full_temp_path = temp_dir + vcs_link.split('/')[4] + ".git" + else: + full_temp_path = temp_dir + vcs_link.split('/')[- 1] + ".git" + repo = Repo.clone_from(vcs_link, full_temp_path) + commit = repo.commit(commit_hash) + #getting the name of the file from the root directory + for filename in os.listdir(full_temp_path): + if is_readme: + target_filename = "README.md" + if "README" in filename: + target_filename = filename + else: + target_filename = "CONTRIBUTING.md" + if "CONTRIBUTING" in filename: + target_filename = filename + targetfile = commit.tree / target_filename + if is_readme: + last_path = "readme" + else: + last_path = "contributing" + with open("/data/users/mgaughan/kkex/time_specific_files/" + last_path + "/" + full_temp_path[len(temp_dir):-4] + "_" + last_path + ".md", "w") as file: + with io.BytesIO(targetfile.data_stream.read()) as f: + file.write(f.read().decode('utf-8')) + file.close() + shutil.rmtree(full_temp_path, ignore_errors=True) + +def for_all_files(): + #toggle this based on readme or contributing files + readme_is = True + csv_path = "kk_test_031324_pr_did.csv" + with open(csv_path, 'r') as file: + #csv_reader = csv.DictReader(file) + lines = [line for line in file] + for row in tqdm(csv.reader(lines), total=len(lines)): + if row[0] == "": + continue + #print(row[0]) + #get_file(row[0], row[?], readme_is) + get_file('https://github.com/tqdm/tqdm', 'fbe7952cce11e8073378b063bdae7ab277a96eb8', True) + +if __name__ == "__main__": + for_all_files() + #get_file('https://github.com/tqdm/tqdm', 'fbe7952cce11e8073378b063bdae7ab277a96eb8', True) + #get_file('https://github.com/krahets/hello-algo/tree/dev1', 'f615ad42ef3c58cfc6f080b8fb0cd0eb741706a9', True ) \ No newline at end of file diff --git a/pr_data_get.py b/pr_data_get.py index 077d5af..65da5f1 100644 --- a/pr_data_get.py +++ b/pr_data_get.py @@ -12,7 +12,7 @@ import math key = os.environ.get('KKEXKEY') early_cutoff = dt.datetime(2008,2, 8) -temp_dir = "/data/users/mgaughan/tmp" +temp_dir = "/data/users/mgaughan/tmp/" ''' - rate of change, rate of all/day @@ -24,8 +24,6 @@ def file_get_pr(upstream_vcs_link, me_read): #print(upstream_vcs_link.split('/')[4]) project_dict = {} project_dict['upstream_vcs_link'] = upstream_vcs_link - if upstream_vcs_link == " https://gitlab.com/ubports/core/cmake-extras": - return {} upstream_vcs_link = upstream_vcs_link.strip() if "github" in upstream_vcs_link or "gitlab" in upstream_vcs_link: #making an evaluation that sub branches aren't being used and that people would fork if needed @@ -36,6 +34,9 @@ def file_get_pr(upstream_vcs_link, me_read): else: full_temp_path = temp_dir + upstream_vcs_link.split('/')[- 1] + ".git" print(upstream_vcs_link) + if upstream_vcs_link == "https://gitlab.com/ubports/core": + shutil.rmtree(full_temp_path, ignore_errors=True) + return {} repo = Git(uri=upstream_vcs_link, gitpath=full_temp_path) try: commits = repo.fetch() @@ -144,13 +145,13 @@ def pr_count(start, end, commits, author_roster, commit_roster): return [by_week, by_week_merge, new_authors, new_committers, author_roster, commit_roster] def for_files(): - csv_path = "final_data/kk_final_readme_roster.csv" + csv_path = "final_data/deb_readme_roster.csv" count = 0 with open(csv_path, 'r') as file: csv_reader = csv.DictReader(file) - with open('kk_test_031424_pr_did.csv', "w") as writing_file: + with open('kk_test_031624_pr_did.csv', "w") as writing_file: # this would also have to get switched fro the cont dataset - keys = ['upstream_vcs_link', "first_readme", "before_allcom_read", "before_mrg_read", "after_allcom_read", "after_mrg_read", 'before_auth_new', 'after_commit_new', 'after_auth_new', 'before_commit_new'] + keys = ['upstream_vcs_link', "first_readme", "readme_commit_hash", "before_allcom_read", "before_mrg_read", "after_allcom_read", "after_mrg_read", 'before_auth_new', 'after_commit_new', 'after_auth_new', 'before_commit_new'] dict_writer = csv.DictWriter(writing_file, keys) dict_writer.writeheader() for row in csv_reader: @@ -162,9 +163,9 @@ def for_files(): if __name__ == "__main__": - #for_files() - file_get_pr("https://github.com/tqdm/tqdm", True) + for_files() + #file_get_pr("https://github.com/tqdm/tqdm", True) #file_get_pr("https://github.com/GameServerManagers/LinuxGSM", True) #file_get_pr("https://github.com/walling/unorm/issues/new/", True) - file_get_pr("https://github.com/krahets/hello-algo/tree/dev1", True) + #file_get_pr("https://github.com/krahets/hello-algo/tree/dev1", True)