progress on pr_did

pr_data_get.py

@@ -1,3 +1,4 @@
+import csv
 from perceval.backends.core.git import Git
 import os
 import datetime as dt
@@ -6,6 +7,7 @@ import shutil
 import pandas as pd
 import dateutil
 from tqdm import tqdm
+import math
 
 key = os.environ.get('KKEXKEY')
 
@@ -13,13 +15,12 @@ early_cutoff = dt.datetime(2008,2, 8)
 temp_dir = "/data/users/mgaughan/tmp"
 
 '''
-- what we really want is the 6 weeks before and after these events 
-- lets start by getting 6 weeks (42 days) before and after the first appearance of a given document
 - rate of change, rate of PRs/day
 '''
-def file_get_pr(upstream_vcs_link):
+def file_get_pr(upstream_vcs_link, me_read):
+    # if we're looking at readmes me_read is true and if not, if we're looking at contributing files, it's false
     #this is the window of days on either side of the event that we're looking at
-    window = 42
+    window = 182
     #print(upstream_vcs_link.split('/')[4])
     project_dict = {}
     project_dict['upstream_vcs_link'] = upstream_vcs_link
@@ -48,44 +49,70 @@ def file_get_pr(upstream_vcs_link):
         #print(commit['data']['CommitDate'])
         #print(type(dateutil.parser.parse(commit['data']['CommitDate'])))
         for file in files:
-            if "CONTRIBUTING.md" == file['file'] and has_contributing == False:
+            if "CONTRIBUTING" in file['file'] and has_contributing == False:
                 has_contributing = True
                 first_date_contributing = dateutil.parser.parse(commit['data']['CommitDate'])
-            if "README.md" == file['file'] and has_readme == False:
+            if "README" in file['file'] and has_readme == False:
                 has_readme = True
                 first_date_readme = dateutil.parser.parse(commit['data']['CommitDate'])
     shutil.rmtree(full_temp_path, ignore_errors=True)
-    project_dict['first_contributing'] = first_date_contributing
+    if me_read:
-    project_dict['first_readme'] = first_date_readme
+        project_dict['first_readme'] = first_date_readme
-    before_cont = pr_count(first_date_contributing + dt.timedelta(days=-window, hours=0), first_date_contributing, commit_list)
+        before_read = pr_count(first_date_readme+ dt.timedelta(days=-window, hours=0), first_date_readme, commit_list)
-    project_dict['b6w_prs_cont'] = before_cont[0]
+        project_dict['b6w_prs_read'] = before_read[0]
-    project_dict['b6w_mrg_cont'] = before_cont[1]
+        project_dict['b6w_mrg_read'] = before_read[1]
-    after_cont = pr_count(first_date_contributing, first_date_contributing + dt.timedelta(days=window, hours=0), commit_list)
+        after_read = pr_count(first_date_readme, first_date_readme + dt.timedelta(days=window, hours=0), commit_list)
-    project_dict['a6w_prs_cont'] = after_cont[0]
+        project_dict['a6w_prs_read'] = after_read[0]
-    project_dict['a6w_mrg_cont'] = after_cont[1]
+        project_dict['a6w_mrg_read'] = after_read[1]
-    before_read = pr_count(first_date_readme+ dt.timedelta(days=-window, hours=0), first_date_readme, commit_list)
+    else:
-    project_dict['b6w_prs_read'] = before_read[0]
+        project_dict['first_contributing'] = first_date_contributing
-    project_dict['b6w_mrg_read'] = before_read[1]
+        before_cont = pr_count(first_date_contributing + dt.timedelta(days=-window, hours=0), first_date_contributing, commit_list)
-    after_read = pr_count(first_date_readme, first_date_readme + dt.timedelta(days=window, hours=0), commit_list)
+        project_dict['b6w_prs_cont'] = before_cont[0]
-    project_dict['a6w_prs_read'] = after_read[0]
+        project_dict['b6w_mrg_cont'] = before_cont[1]
-    project_dict['a6w_mrg_read'] = after_read[1]
+        after_cont = pr_count(first_date_contributing, first_date_contributing + dt.timedelta(days=window, hours=0), commit_list)
+        project_dict['a6w_prs_cont'] = after_cont[0]
+        project_dict['a6w_mrg_cont'] = after_cont[1]
     print(project_dict)
     return project_dict
 
+
+#TODO: pr_count should return an array of values for weekly/6mo
 def pr_count(start, end, commits):
     count = 0
     merge_count = 0
+    by_week = [0] * 27
+    by_week_merge =[0] * 27
+    current_week = 0
     for commit in tqdm(commits):
         if dateutil.parser.parse(commit[0]) > start:
-            count += 1
+            if math.floor((dateutil.parser.parse(commit[0]) - start).days / 7) <= 26:
-            if commit[1]:
+                by_week[math.floor((dateutil.parser.parse(commit[0]) - start).days / 7)] += 1
-                merge_count += 1
+                if commit[1]:
+                    by_week_merge[math.floor((dateutil.parser.parse(commit[0]) - start).days / 7)] += 1
         if dateutil.parser.parse(commit[0]) > end:
-            return [count, merge_count]
+            print(len(by_week))
-
+            return [by_week, by_week_merge]
 
+#TODO: need to do this for all files in the dataset of readme or contributing
+def for_files():
+    csv_path = "final_data/kk_final_readme_roster.csv"
+    count = 0 
+    with open(csv_path, 'r') as file:
+        csv_reader = csv.DictReader(file)
+        with open('kk_test_031024_pr_did.csv', "w") as writing_file:
+            keys = ['upstream_vcs_link', "first_readme", "b6w_prs_read", "b6w_mrg_read", "a6w_prs_read", "a6w_mrg_read"]
+            dict_writer = csv.DictWriter(writing_file, keys)
+            dict_writer.writeheader()
+            #training wheels on right now
+            for row in csv_reader:
+                count += 1
+                print(row['upstream_vcs_link'])
+                dict_row = file_get_pr(row['upstream_vcs_link'].strip(), True)
+                dict_writer.writerow(dict_row)
+    
 
 if __name__ == "__main__":
-    file_get_pr("https://github.com/tqdm/tqdm")
+    for_files()
-    file_get_pr("https://github.com/GameServerManagers/LinuxGSM")
+    #file_get_pr("https://github.com/tqdm/tqdm", False)
+    #file_get_pr("https://github.com/GameServerManagers/LinuxGSM")