updates + backing up readme data

contrib_data_get.py

@@ -0,0 +1,177 @@
+import csv
+from perceval.backends.core.git import Git
+import os
+import datetime as dt
+import time
+import shutil 
+import pandas as pd
+import dateutil
+from tqdm import tqdm
+import math
+
+key = os.environ.get('KKEXKEY')
+
+early_cutoff = dt.datetime(2008,2, 8)
+temp_dir = "/data/users/mgaughan/tmp/"
+
+'''
+- rate of change, rate of all/day
+'''
+
+def file_get_pr(upstream_vcs_link, me_read):
+    # if we're looking at readmes me_read is true and if not, if we're looking at contributing files, it's false
+    #this is the window of days on either side of the event that we're looking at
+    window = 182
+    #print(upstream_vcs_link.split('/')[4])
+    project_dict = {}
+    project_dict['upstream_vcs_link'] = upstream_vcs_link
+    upstream_vcs_link = upstream_vcs_link.strip()
+    if "github" in upstream_vcs_link or "gitlab" in upstream_vcs_link:
+        #making an evaluation that sub branches aren't being used and that people would fork if needed
+        #this only looks at main
+        upstream_vcs_link = "/".join(upstream_vcs_link.split("/")[0:5])
+        print(upstream_vcs_link)
+        full_temp_path = temp_dir + upstream_vcs_link.split('/')[4] + ".git"
+    else:
+        full_temp_path = temp_dir + upstream_vcs_link.split('/')[- 1] + ".git"
+    print(upstream_vcs_link)
+    if upstream_vcs_link == "https://gitlab.com/ubports/core" or upstream_vcs_link == "https://gitlab.freedesktop.org/xorg/lib":
+        shutil.rmtree(full_temp_path, ignore_errors=True)
+        return {}
+    repo = Git(uri=upstream_vcs_link, gitpath=full_temp_path)
+    try:
+        commits = repo.fetch()
+    except:
+        print("perceval issue")
+        return
+    has_readme = False
+    has_contributing = False
+    merge_pre_rm, merge_post_rm, merge_pre_cont, merge_post_cont = 0, 0, 0, 0
+    #list of tuples which has date and whether it was a merge
+    commit_list = []
+    first_date_readme = ""
+    for commit in commits:
+        #print(commit['data'])
+        if "Merge" in commit['data'].keys():
+            commit_list.append([commit['data']['CommitDate'], True, commit['data']['Author'], commit['data']['Commit']])
+            if has_contributing:
+                merge_post_cont += 1
+            else:
+                merge_pre_cont += 1
+        else: 
+            commit_list.append([commit['data']['CommitDate'], False, commit['data']['Author'], commit['data']['Commit']])
+        files = commit['data']['files']
+        #print(commit['data']['CommitDate'])
+        #print(type(dateutil.parser.parse(commit['data']['CommitDate'])))
+        for file in files:
+            if "CONTRIBUTING" in file['file'] and has_contributing == False:
+                has_contributing = True
+                first_date_contributing = dateutil.parser.parse(commit['data']['CommitDate'])
+                project_dict['contributing_commit_hash'] = commit['data']['commit']
+            if "README" in file['file'] and has_readme == False:
+                has_readme = True
+                first_date_readme = dateutil.parser.parse(commit['data']['CommitDate'])
+                #project_dict['readme_commit_hash'] = commit['data']['commit']
+    shutil.rmtree(full_temp_path, ignore_errors=True)
+    if first_date_readme == "":
+        return {}
+    if me_read:
+        project_dict['first_readme'] = first_date_readme
+        before_read = pr_count(first_date_readme+ dt.timedelta(days=-window, hours=0), first_date_readme, commit_list, [], [])
+        if before_read != None:
+            project_dict['before_allcom_read'] = before_read[0]
+            project_dict['before_mrg_read'] = before_read[1]
+            project_dict['before_auth_new'] = before_read[2]
+            project_dict['before_commit_new'] = before_read[3]
+        else:
+            return {}
+        after_read = pr_count(first_date_readme, first_date_readme + dt.timedelta(days=window, hours=0), commit_list, before_read[4], before_read[5])
+        if after_read != None:
+            project_dict['after_allcom_read'] = after_read[0]
+            project_dict['after_mrg_read'] = after_read[1]
+            project_dict['after_auth_new'] = after_read[2]
+            project_dict['after_commit_new'] = after_read[3]
+        else: 
+            return {}
+    else:
+        project_dict['first_contributing'] = first_date_contributing
+        before_cont = pr_count(first_date_contributing + dt.timedelta(days=-window, hours=0), first_date_contributing, commit_list, [], [])
+        if before_cont != None:
+            project_dict['before_allcom_cont'] = before_cont[0]
+            project_dict['before_mrg_cont'] = before_cont[1]
+            project_dict['before_auth_new'] = before_cont[2]
+            project_dict['before_commit_new'] = before_cont[3]
+        else:
+            return {}
+        after_cont = pr_count(first_date_contributing, first_date_contributing + dt.timedelta(days=window, hours=0), commit_list, before_cont[4], before_cont[5])
+        if after_cont != None:
+            project_dict['after_allcom_cont'] = after_cont[0]
+            project_dict['after_mrg_cont'] = after_cont[1]
+            project_dict['after_auth_new'] = after_cont[2]
+            project_dict['after_commit_new'] = after_cont[3]
+        else: 
+            return {}
+    print(project_dict)
+    return project_dict
+
+
+#TODO: pr_count should return an array of values for weekly/6mo
+def pr_count(start, end, commits, author_roster, commit_roster):
+    count = 0
+    merge_count = 0
+    by_week = [0] * 27
+    by_week_merge =[0] * 27
+    current_week = 0
+    new_authors = 0
+    new_committers = 0
+    for commit in tqdm(commits):
+        if dateutil.parser.parse(commit[0]) <= start:
+            if commit[2] not in author_roster:
+                author_roster.append(commit[2])
+            if commit[1] and commit[3] not in commit_roster:
+                commit_roster.append(commit[3])
+        if dateutil.parser.parse(commit[0]) > start:
+            if math.floor((dateutil.parser.parse(commit[0]) - start).days / 7) <= 26:
+                by_week[math.floor((dateutil.parser.parse(commit[0]) - start).days / 7)] += 1
+                if commit[1]:
+                    by_week_merge[math.floor((dateutil.parser.parse(commit[0]) - start).days / 7)] += 1
+                    if commit[3] not in commit_roster:
+                        new_committers += 1
+                        #remaining question of whether to make this the author of the merge commit[2] or the committer of the merge commit[3]
+                        commit_roster.append(commit[3])
+                if commit[2] not in author_roster:
+                    new_authors += 1
+                    author_roster.append(commit[2])
+        if dateutil.parser.parse(commit[0]) > end:
+            print(len(by_week))
+            return [by_week, by_week_merge, new_authors, new_committers, author_roster, commit_roster]
+
+def for_files():
+    csv_path = "final_data/deb_contribfile_roster.csv"
+    #csv_path = "final_data/deb_readme_roster.csv"
+    count = 0 
+    with open(csv_path, 'r') as file:
+        csv_reader = csv.DictReader(file)
+        with open('kk_031624_contrib_did.csv', "w") as writing_file:
+            # this would also have to get switched fro the cont dataset
+            keys = ['upstream_vcs_link', "first_contributing", "contributing_commit_hash", "before_allcom_cont", "before_mrg_cont", "after_allcom_cont", "after_mrg_cont", 'before_auth_new', 'after_commit_new', 'after_auth_new', 'before_commit_new']
+            dict_writer = csv.DictWriter(writing_file, keys)
+            dict_writer.writeheader()
+            for row in csv_reader:
+                count += 1
+                print(row['upstream_vcs_link'])
+                # this would have to get switched to false for the cont dataset
+                try:
+                    dict_row = file_get_pr(row['upstream_vcs_link'].strip(), False)
+                except:
+                    dict_row = {}
+                dict_writer.writerow(dict_row)
+    
+
+if __name__ == "__main__":
+    for_files()
+    #file_get_pr("https://github.com/tqdm/tqdm", True)
+    #file_get_pr("https://github.com/GameServerManagers/LinuxGSM", True)
+    #file_get_pr("https://github.com/walling/unorm/issues/new/", True)
+    #file_get_pr("https://github.com/krahets/hello-algo/tree/dev1", True)
+    

get_spec_file.py

@@ -49,15 +49,19 @@ def get_file(vcs_link, commit_hash, is_readme):
 def for_all_files():
     #toggle this based on readme or contributing files
     readme_is = True
-    csv_path = "kk_test_031324_pr_did.csv"
+    csv_path = "kk_031624_pr_did.csv"
+    index = -1
     with open(csv_path, 'r') as file:
         #csv_reader = csv.DictReader(file)
         lines = [line for line in file]
         for row in tqdm(csv.reader(lines), total=len(lines)):
+            index += 1
+            if index == 0:
+                continue
             if row[0] == "":
                 continue
             #print(row[0])
-            #get_file(row[0], row[?], readme_is)
+            get_file(row[0], row[2], readme_is)
             #get_file('https://github.com/tqdm/tqdm', 'fbe7952cce11e8073378b063bdae7ab277a96eb8', True)
 
 if __name__ == "__main__":