24_deb_pkg_gov/non_coding_measures_scrape.py

import pexpect 
import os 
import json 
import sys
import subprocess
import csv 
import ast


#sys.path.append('../octohatrack')
#import octohatrack.__main__ as oh
csv_path = "011824_uni_contrib.csv"

def main():
    wd = os.getcwd()
    with open(csv_path, 'w', newline='') as output_file:
        keys = ["project_name", "project_owner", "api_contrib_count", "issue_contrib_count", "file_contrib_count", "wiki_contrib_count"]
        dict_writer = csv.DictWriter(output_file, keys)
        dict_writer.writeheader()
        with open("expanded_data_final.csv", "r") as file:
            reader = csv.reader(file)
            for i, line in enumerate(reader):
                project_dict ={}
                if "github" not in line[5]:
                    continue
                repo_name = line[5].strip()[19:]
                print(repo_name)
                project_dict["project_name"] = repo_name.split("/")[1]
                project_dict["project_owner"]= repo_name.split("/")[0]
                os.chdir("../octohatrack")
                try:
                    octohatrack_results = subprocess.run(['python3', '-m', 'octohatrack', repo_name, '--wait-for-reset'], capture_output = True, text=True, timeout=60).stdout
                except:
                    print("issue with the repository, string indices must be integers")
                    continue
                os.chdir(wd)
                split_results = octohatrack_results.split("\n")
                for entry in split_results:
                    if "{'api_contributors'" in entry:
                        formatted = ast.literal_eval(entry)
                        project_dict["api_contrib_count"] = len(formatted['api_contributors'])
                        #project_dict["api_contrib_list"] = formatted['api_contributors']
                        project_dict["issue_contrib_count"] = len(formatted['issue_pr_contributors'])
                        #project_dict["issue_contrib_list"] = formatted['issue_pr_contributors']
                        project_dict["file_contrib_count"] = len(formatted['file_contributors'])
                        #project_dict["file_contrib_list"] = formatted['file_contributors']
                        project_dict["wiki_contrib_count"] = len(formatted['file_contributors'])
                        #project_dict["wiki_contrib_list"] = formatted['file_contributors']
                print(project_dict)
                dict_writer.writerow(project_dict)
                with open('/data/users/mgaughan/b_kkex_contrib_uni_011824/' + 'contrib_roster_' + project_dict["project_name"] + '.json', 'w') as data_path:
                    json.dump(formatted, data_path)

if __name__ == "__main__":
    main()