From 1d678bb420ee12adb86d1f077715700793e67eb2 Mon Sep 17 00:00:00 2001 From: Matthew Gaughan Date: Sun, 10 Mar 2024 16:05:46 -0500 Subject: [PATCH] progress on pr_did --- pr_data_get.py | 81 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/pr_data_get.py b/pr_data_get.py index 3ec9c35..1abc27a 100644 --- a/pr_data_get.py +++ b/pr_data_get.py @@ -1,3 +1,4 @@ +import csv from perceval.backends.core.git import Git import os import datetime as dt @@ -6,6 +7,7 @@ import shutil import pandas as pd import dateutil from tqdm import tqdm +import math key = os.environ.get('KKEXKEY') @@ -13,13 +15,12 @@ early_cutoff = dt.datetime(2008,2, 8) temp_dir = "/data/users/mgaughan/tmp" ''' -- what we really want is the 6 weeks before and after these events -- lets start by getting 6 weeks (42 days) before and after the first appearance of a given document - rate of change, rate of PRs/day ''' -def file_get_pr(upstream_vcs_link): +def file_get_pr(upstream_vcs_link, me_read): + # if we're looking at readmes me_read is true and if not, if we're looking at contributing files, it's false #this is the window of days on either side of the event that we're looking at - window = 42 + window = 182 #print(upstream_vcs_link.split('/')[4]) project_dict = {} project_dict['upstream_vcs_link'] = upstream_vcs_link @@ -48,44 +49,70 @@ def file_get_pr(upstream_vcs_link): #print(commit['data']['CommitDate']) #print(type(dateutil.parser.parse(commit['data']['CommitDate']))) for file in files: - if "CONTRIBUTING.md" == file['file'] and has_contributing == False: + if "CONTRIBUTING" in file['file'] and has_contributing == False: has_contributing = True first_date_contributing = dateutil.parser.parse(commit['data']['CommitDate']) - if "README.md" == file['file'] and has_readme == False: + if "README" in file['file'] and has_readme == False: has_readme = True first_date_readme = dateutil.parser.parse(commit['data']['CommitDate']) shutil.rmtree(full_temp_path, ignore_errors=True) - project_dict['first_contributing'] = first_date_contributing - project_dict['first_readme'] = first_date_readme - before_cont = pr_count(first_date_contributing + dt.timedelta(days=-window, hours=0), first_date_contributing, commit_list) - project_dict['b6w_prs_cont'] = before_cont[0] - project_dict['b6w_mrg_cont'] = before_cont[1] - after_cont = pr_count(first_date_contributing, first_date_contributing + dt.timedelta(days=window, hours=0), commit_list) - project_dict['a6w_prs_cont'] = after_cont[0] - project_dict['a6w_mrg_cont'] = after_cont[1] - before_read = pr_count(first_date_readme+ dt.timedelta(days=-window, hours=0), first_date_readme, commit_list) - project_dict['b6w_prs_read'] = before_read[0] - project_dict['b6w_mrg_read'] = before_read[1] - after_read = pr_count(first_date_readme, first_date_readme + dt.timedelta(days=window, hours=0), commit_list) - project_dict['a6w_prs_read'] = after_read[0] - project_dict['a6w_mrg_read'] = after_read[1] + if me_read: + project_dict['first_readme'] = first_date_readme + before_read = pr_count(first_date_readme+ dt.timedelta(days=-window, hours=0), first_date_readme, commit_list) + project_dict['b6w_prs_read'] = before_read[0] + project_dict['b6w_mrg_read'] = before_read[1] + after_read = pr_count(first_date_readme, first_date_readme + dt.timedelta(days=window, hours=0), commit_list) + project_dict['a6w_prs_read'] = after_read[0] + project_dict['a6w_mrg_read'] = after_read[1] + else: + project_dict['first_contributing'] = first_date_contributing + before_cont = pr_count(first_date_contributing + dt.timedelta(days=-window, hours=0), first_date_contributing, commit_list) + project_dict['b6w_prs_cont'] = before_cont[0] + project_dict['b6w_mrg_cont'] = before_cont[1] + after_cont = pr_count(first_date_contributing, first_date_contributing + dt.timedelta(days=window, hours=0), commit_list) + project_dict['a6w_prs_cont'] = after_cont[0] + project_dict['a6w_mrg_cont'] = after_cont[1] print(project_dict) return project_dict + +#TODO: pr_count should return an array of values for weekly/6mo def pr_count(start, end, commits): count = 0 merge_count = 0 + by_week = [0] * 27 + by_week_merge =[0] * 27 + current_week = 0 for commit in tqdm(commits): if dateutil.parser.parse(commit[0]) > start: - count += 1 - if commit[1]: - merge_count += 1 + if math.floor((dateutil.parser.parse(commit[0]) - start).days / 7) <= 26: + by_week[math.floor((dateutil.parser.parse(commit[0]) - start).days / 7)] += 1 + if commit[1]: + by_week_merge[math.floor((dateutil.parser.parse(commit[0]) - start).days / 7)] += 1 if dateutil.parser.parse(commit[0]) > end: - return [count, merge_count] - + print(len(by_week)) + return [by_week, by_week_merge] +#TODO: need to do this for all files in the dataset of readme or contributing +def for_files(): + csv_path = "final_data/kk_final_readme_roster.csv" + count = 0 + with open(csv_path, 'r') as file: + csv_reader = csv.DictReader(file) + with open('kk_test_031024_pr_did.csv', "w") as writing_file: + keys = ['upstream_vcs_link', "first_readme", "b6w_prs_read", "b6w_mrg_read", "a6w_prs_read", "a6w_mrg_read"] + dict_writer = csv.DictWriter(writing_file, keys) + dict_writer.writeheader() + #training wheels on right now + for row in csv_reader: + count += 1 + print(row['upstream_vcs_link']) + dict_row = file_get_pr(row['upstream_vcs_link'].strip(), True) + dict_writer.writerow(dict_row) + if __name__ == "__main__": - file_get_pr("https://github.com/tqdm/tqdm") - file_get_pr("https://github.com/GameServerManagers/LinuxGSM") + for_files() + #file_get_pr("https://github.com/tqdm/tqdm", False) + #file_get_pr("https://github.com/GameServerManagers/LinuxGSM")