first commit dates for readme projects

062424_did_first_commit_readme_2.csv

@@ -0,0 +1,173 @@
+upstream_vcs_link,first_commit
+https://github.com/mupen64plus/mupen64plus-core.git,Fri Oct 2 00:10:43 2009 -0400
+https://github.com/wedesoft/aiscm,Fri Dec 27 13:00:00 2013 +0000
+https://github.com/EttusResearch/uhd,Thu Jan 7 16:46:00 2010 -0800
+https://gitlab.com/ddcci-driver-linux/ddcci-driver-linux.git,Wed Jul 8 00:07:11 2015 +0200
+https://invent.kde.org/plasma/sddm-kcm.git,Sun Feb 3 22:18:19 2013 +0900
+https://github.com/ardentryst/ardentryst,Wed Dec 31 02:57:54 2014 +0100
+https://github.com/derf/feh.git,Fri Feb 5 17:52:34 2010 +0100
+https://gitlab.gnome.org/GNOME/tracker-miners,Wed Dec 14 11:11:14 2005 +0000
+https://github.com/timothycrosley/isort,Mon Sep 2 15:22:53 2013 -0700
+https://github.com/eclipse-ee4j/eclipselink.git,Tue Aug 14 16:42:15 2007 +0000
+https://github.com/npm/abbrev-js,Tue Mar 9 13:15:25 2010 -0800
+https://github.com/maxbube/mydumper.git,Sun Feb 1 02:13:20 2009 +0200
+https://github.com/rstudio/httpuv.git,Mon Feb 4 14:19:37 2013 -0800
+https://github.com/ledgersmb/PGObject-Type-DateTime.git,Wed Nov 13 00:43:16 2013 -0800
+https://github.com/knxd/knxd,Tue Nov 25 18:06:28 2014 +0100
+https://github.com/pypa/pipenv.git,Thu Jan 19 19:44:22 2017 -0500
+https://github.com/libsndfile/libsamplerate,Thu Jan 22 11:16:39 2004 +0000
+https://github.com/sebastianbergmann/php-timer,Sat May 8 16:42:36 2010 +0200
+https://github.com/rocky/Perl-Device-Cdio.git,Wed Feb 8 05:06:05 2006 +0000
+https://github.com/phpDocumentor/TypeResolver,Wed Jun 10 21:06:46 2015 +0200
+https://github.com/ionos-cloud/netconsole,Tue Oct 30 17:26:51 2018 +0100
+https://github.com/resque/redis-namespace,Tue Oct 13 14:48:51 2009 -0700
+https://github.com/thoughtbot/shoulda-context,Wed Mar 14 18:11:47 2007 +0000
+https://github.com/ukui/ukui-settings-daemon,Tue Dec 12 10:52:40 2017 +0800
+https://github.com/sddm/sddm.git,Fri Jan 18 00:46:14 2013 +0000
+https://github.com/altdesktop/playerctl,Tue Apr 15 04:31:16 2014 -0400
+https://github.com/mypaint/libmypaint,Sun Nov 23 13:09:05 2008 +0000
+https://github.com/psi-im/qconf.git,Fri May 30 06:17:35 2008 +0000
+https://github.com/rspatial/raster.git,Mon Jul 30 19:08:04 2018 -0700
+https://github.com/evandelisle/showq.git,Sun Apr 10 20:13:56 2011 +0200
+https://anongit.kde.org/kfourinline.git,Sun Jun 18 14:00:37 2000 +0000
+https://github.com/jmoiron/humanize,Fri Oct 7 19:49:53 2011 -0400
+https://github.com/mruby-debian/mruby,Thu Apr 19 02:11:15 2012 -0700
+https://github.com/mika/jenkins-debian-glue,Fri Aug 5 01:02:54 2011 +0200
+https://github.com/babelouest/glewlwyd,Tue Nov 22 08:54:34 2016 -0500
+https://github.com/elastic/elasticsearch-ruby,Tue Apr 30 14:16:06 2013 +0200
+https://github.com/lxqt/lxqt-about.git,Tue Jun 26 10:46:21 2012 +0200
+https://github.com/hughsie/libxmlb.git,Tue Sep 18 19:13:06 2018 +0100
+https://github.com/jwbargsten/perl-proc-processtable,Fri Oct 19 05:38:23 2012 -0700
+https://github.com/jhawthorn/fzy,Fri Jul 11 21:25:33 2014 -0700
+https://github.com/greenbone/gvm-libs,Sun Nov 6 20:06:15 2005 +0000
+https://github.com/frogatto/frogatto.git,Thu Jun 26 06:06:05 2008 +0000
+https://invent.kde.org/network/smb4k.git,Sat Oct 31 07:43:37 2009 +0000
+https://github.com/uclouvain/openjpeg.git,Thu Nov 27 10:07:24 2003 +0000
+https://github.com/ofalk/libdnet,Thu Oct 11 04:14:42 2001 +0000
+https://github.com/python-cmd2/cmd2.git,Fri May 16 10:40:15 2008 -0400
+https://github.com/nichtich/Catmandu-Wikidata.git,Wed Dec 4 11:09:46 2013 +0100
+https://github.com/visionmedia/node-fresh.git,Sun Jun 10 12:09:05 2012 -0700
+https://github.com/zmartzone/mod_auth_openidc.git,Thu Mar 27 20:18:52 2014 +0100
+https://github.com/pyFFTW/pyFFTW,Tue Jan 31 13:21:08 2012 +0000
+https://github.com/ZerBea/hcxdumptool,Sun Feb 25 09:18:41 2018 +0100
+https://github.com/ShawnDEvans/smbmap.git,Wed Apr 22 17:10:48 2015 -0400
+https://github.com/fln/gnarwl,Tue Sep 18 00:42:26 2012 -0700
+https://github.com/jschaf/pylint-flask,Sat Jun 27 16:41:46 2015 -0400
+https://github.com/dds-bridge/dds.git,Tue Nov 25 07:16:33 2014 -0500
+https://github.com/oetiker/rrdtool-1.x.git,Sun Feb 25 22:25:02 2001 +0000
+https://github.com/axel-download-accelerator/axel.git,Fri Jan 11 11:57:58 2008 +0000
+https://github.com/DanielAdolfsson/ndppd,Tue Sep 13 21:26:12 2011 +0200
+https://github.com/dhewm/dhewm3.git,Tue Nov 22 15:28:15 2011 -0600
+https://gitlab.gnome.org/GNOME/hitori.git,Mon Mar 2 00:03:36 2009 +0000
+https://github.com/dbr/tvdb_api.git,Mon May 5 22:32:27 2008 +0930
+https://github.com/tokuhirom/Furl.git,Fri Oct 15 17:55:24 2010 +0900
+https://github.com/ytdl-org/youtube-dl.git,Mon Jul 21 23:12:31 2008 +0200
+https://github.com/mvo5/synaptic,Sun Feb 2 14:10:39 2003 -0300
+https://github.com/marcelm/alignlib.git,Tue Nov 27 13:31:50 2018 +0100
+https://github.com/sebastinas/python-libdiscid.git,Wed Apr 10 18:08:31 2013 +0200
+https://gitlab.com/DavidGriffith/frotz,Wed Dec 7 15:46:49 2011 +0000
+https://github.com/ge-ne/bibtool.git,Wed Feb 7 21:26:17 2007 +0000
+https://github.com/mogaal/sendemail,Sun Sep 12 12:09:43 2010 -0430
+https://github.com/drowe67/freedv-gui,Thu Aug 13 13:07:43 2015 +0000
+https://github.com/gnustep/apps-gorm,Wed Dec 8 15:04:57 1999 +0000
+https://github.com/prometheus/node_exporter,Thu Apr 18 07:44:52 2013 -0700
+https://github.com/tinfoil/devise-two-factor.git,Mon May 12 11:46:54 2014 -0700
+https://github.com/pear/Crypt_GPG.git,Thu Jan 3 21:06:18 2008 +0000
+https://github.com/csmith-project/csmith.git,Tue May 3 18:13:31 2011 -0600
+https://github.com/blitzpp/blitz,Mon Jan 18 22:31:10 2016 -0500
+https://github.com/porridge/bambam,Sun Dec 30 09:45:54 2007 +0000
+https://github.com/connorimes/powercap,Wed Jun 22 10:30:48 2016 -0500
+https://github.com/gildor478/ocaml-fileutils.git,Sat Sep 14 01:25:34 2019 +0200
+https://github.com/bbatsov/powerpack.git,Fri Jul 12 15:44:22 2013 +0300
+https://gitlab.com/libpipeline/libpipeline.git,Sun Aug 3 10:16:19 2003 +0000
+https://gitlab.freedesktop.org/mobile-broadband/ModemManager.git,Thu Jul 31 09:43:00 2008 +0300
+https://github.com/pypa/pipx.git,Sun Oct 7 22:35:38 2018 -0700
+https://github.com/Blosc/c-blosc,Mon May 18 17:07:45 2009 +0000
+https://github.com/invesalius/invesalius3.git,Wed Jun 10 17:30:02 2009 +0000
+https://gitlab.gnome.org/GNOME/atkmm.git,Wed Jun 23 01:55:10 2010 +0200
+https://github.com/christophgysin/pasystray.git,Fri Oct 7 23:54:18 2011 +0300
+https://github.com/CESNET/libyang,Mon May 25 15:15:01 2020 +0200
+https://github.com/leamas/ddupdate,Mon Dec 25 17:50:56 2017 +0100
+https://github.com/sebastianbergmann/php-text-template,Thu Nov 19 12:48:18 2009 +0100
+https://gitlab.com/drobilla/ganv.git,Tue Dec 6 21:01:38 2011 +0000
+https://github.com/jupyter/jupyter-sphinx-theme,Wed Dec 7 09:54:07 2011 -0500
+https://github.com/ocrmypdf/OCRmyPDF.git,Tue Apr 9 09:44:46 2013 -0700
+https://github.com/andrew-bibb/cmst.git,Sat Mar 15 09:47:58 2014 -0400
+https://github.com/Seldaek/jsonlint,Tue Dec 13 15:44:46 2011 +0100
+https://github.com/geopython/stetl.git,Tue Aug 28 11:29:43 2012 +0200
+https://github.com/gkz/type-check.git,Sun Sep 29 11:32:18 2013 -0700
+https://github.com/ossimlabs/ossim.git,Wed Sep 2 16:20:52 2015 -0400
+https://gitlab.gnome.org/GNOME/libmediaart.git,Tue Aug 13 07:28:43 2013 -0700
+https://github.com/mikeboers/PyMemoize,Thu Sep 16 20:32:05 2010 -0400
+https://github.com/canonical/lightdm.git,Thu Apr 29 21:08:26 2010 +1000
+https://github.com/jekyll/jekyll-watch,Thu Jun 26 18:45:47 2014 -0400
+https://github.com/Kitware/KWStyle,Wed Jan 22 10:28:45 2014 -0500
+https://github.com/residuum/PuRestJson.git,Sat Aug 20 18:32:30 2011 +0200
+https://github.com/hynek/argon2-cffi,Mon Dec 7 17:14:57 2015 +0100
+https://github.com/ros/urdfdom_headers,Fri Jun 22 14:35:26 2012 -0700
+https://github.com/zmartzone/cjose.git,Mon Mar 14 15:38:19 2016 -0600
+https://github.com/slgobinath/SafeEyes.git,Fri Jan 8 12:45:22 2016 +0530
+https://github.com/chrys87/simple-orca-plugin-system.git,Tue Jan 5 20:16:08 2016 +0100
+https://github.com/ros/robot_state_publisher,Thu Aug 2 10:59:41 2012 -0700
+https://github.com/akopytov/sysbench.git,Fri Jan 20 16:13:23 2006 +0000
+https://github.com/savonet/ocaml-ffmpeg.git,Fri Oct 12 02:10:24 2012 -0700
+https://github.com/jlmelville/uwot.git,Tue Jun 12 07:34:53 2018 -0700
+https://github.com/jhnc/findimagedupes.git,Mon Jun 19 14:06:12 2006 +0100
+https://github.com/myriadrf/gr-limesdr.git,Tue Apr 24 12:37:35 2018 +0300
+https://github.com/dvehrs/podget.git,Mon Nov 9 15:58:35 2015 -0700
+https://github.com/deiv/driftnet,Thu Jun 14 20:35:06 2012 +0200
+https://github.com/tduehr/omniauth-cas3,Wed Dec 14 16:14:25 2011 -0500
+https://github.com/phar-io/manifest.git,Thu Nov 24 17:35:20 2016 +0100
+https://github.com/maxmind/libmaxminddb,Mon Dec 31 15:21:11 2012 +0100
+https://github.com/cespare/reflex.git,Mon May 20 19:09:33 2013 -0700
+https://github.com/flareteam/flare-engine,Fri Jun 3 22:02:11 2011 -0500
+https://github.com/savonet/ocaml-alsa.git,Tue Nov 23 20:27:04 2021 +0000
+https://github.com/jonschlinkert/global-modules,Fri Mar 6 08:15:23 2015 -0500
+https://github.com/danfruehauf/NetworkManager-ssh,Mon Feb 4 12:18:42 2013 +1100
+https://github.com/sstrigler/JSJaC.git,Fri Oct 8 16:45:37 2004 +0000
+https://github.com/EasyScreenCast/EasyScreenCast.git,Sat Jun 29 12:59:07 2013 +0200
+https://github.com/edicl/hunchentoot,Mon May 26 14:22:29 2008 +0000
+https://github.com/rails/sass-rails,Tue May 24 14:46:54 2011 -0700
+https://github.com/chartjs/Chart.js.git,Sun Mar 17 23:57:50 2013 +0000
+https://github.com/WorkflowConversion/CTDConverter,Tue Feb 18 02:42:10 2014 -0800
+https://github.com/einaros/tinycolor.git,Sat Dec 10 12:51:50 2011 +0100
+https://github.com/viraptor/arpy.git,Thu Aug 5 03:02:58 2010 +0100
+https://framagit.org/simgrid/simgrid.git,Tue Jan 27 19:53:06 2004 +0000
+https://github.com/xaicron/p5-JSON-WebToken.git,Tue Jun 26 22:28:10 2012 +0900
+https://github.com/jonathanhogg/usepackage.git,Wed Jul 19 16:20:03 1995 +0000
+https://github.com/jnunemaker/flipper,Sun Jul 22 12:39:38 2012 -0700
+https://github.com/Storyyeller/enjarify,Thu Jun 4 12:13:55 2015 -0700
+https://github.com/pgbackrest/pgbackrest,Mon Nov 11 13:08:39 2013 -0800
+https://anongit.kde.org/knavalbattle.git,Sun Nov 5 00:12:44 2000 +0000
+https://github.com/horde/mnemo.git,Wed Apr 7 09:51:45 2010 +0200
+https://github.com/Electrostatics/apbs,Thu Jan 2 08:58:28 2014 -0800
+https://github.com/Yelp/py_zipkin,Thu Aug 4 10:58:27 2016 -0700
+https://github.com/mapbox/mapnik-vector-tile.git,Fri Apr 12 20:19:13 2013 -0700
+https://github.com/lxqt/compton-conf.git,Fri Nov 8 17:43:56 2013 +0800
+https://github.com/TooTallNate/node-bindings,Sat Jan 28 12:24:24 2012 -0800
+https://github.com/chef/ffi-yajl.git,Wed Oct 23 21:37:05 2013 -0700
+https://github.com/01org/isa-l.git,Wed Feb 24 14:54:34 2016 -0700
+https://github.com/stompgem/stomp.git,Fri Aug 26 13:36:04 2005 +0000
+https://github.com/ebiggers/libdeflate.git,Sat Dec 27 23:11:49 2014 -0600
+https://github.com/diyan/pywinrm.git,Fri Jul 6 08:25:55 2012 -0700
+https://github.com/borntyping/python-colorlog.git,Wed Sep 5 18:57:08 2012 +0100
+https://github.com/libpd/abl_link.git,Thu Nov 3 15:39:50 2016 +0100
+https://github.com/digitalbazaar/rdf-canonize.git,Mon Feb 15 15:04:58 2016 -0500
+https://github.com/jaspernbrouwer/powerline-gitstatus.git,Thu Jan 29 21:13:57 2015 +0100
+https://github.com/mnooner256/pyqrcode.git,Fri Jun 7 08:40:07 2013 -0500
+https://github.com/aquynh/capstone,Tue Nov 26 18:32:11 2013 -0800
+https://github.com/un33k/python-slugify,Sun Oct 14 18:44:50 2012 -0700
+https://github.com/swaywm/wlroots,Tue Apr 25 11:32:52 2017 -0400
+https://github.com/Talinx/jp2a,Tue Jun 20 18:46:47 2006 +0000
+https://github.com/davesteele/aioprocessing,Mon Jul 28 19:49:22 2014 -0400
+https://gitlab.gnome.org/GNOME/gnome-chess.git,Sat Oct 28 16:37:30 2006 +0000
+https://github.com/cloudpipe/cloudpickle,Mon Apr 13 12:33:00 2015 -0400
+https://github.com/puppetlabs/puppetlabs-mysql,Tue May 24 23:22:43 2011 -0700
+https://github.com/MatMoul/g810-led.git,Sun Jun 12 22:16:07 2016 +0200
+https://github.com/UbuntuBudgie/budgie-indicator-applet,Sun Nov 6 08:20:47 2016 +0000
+https://github.com/mhekkel/libzeep.git,Mon Nov 10 13:56:55 2008 +0000
+https://github.com/rvagg/node-errno,Mon Apr 30 14:10:12 2012 +1000
+https://github.com/fossfreedom/alternative-toolbar,Mon Dec 22 23:09:43 2014 +0000
+https://github.com/awesomeWM/awesome.git,Wed Sep 5 20:15:00 2007 +0200
+https://github.com/Debian/perl-cross-debian, Sun Nov 25 05:36:03 2012 -0800
+https://github.com/codehelp/perl-cross-debian, Sun Nov 25 05:36:03 2012 -0800

get_first_commit.py

@@ -0,0 +1,85 @@
+import csv
+from perceval.backends.core.git import Git
+import os
+import datetime as dt
+from datetime import timezone
+import time
+import shutil 
+import dateutil
+from tqdm import tqdm
+import math
+
+key = os.environ.get('KKEXKEY')
+temp_dir = "/data/users/mgaughan/tmp/"
+temp_dir1 = "tmp/"
+
+'''
+- rate of change, rate of all/day
+'''
+def file_get_contribs(upstream_vcs_link, event_date):
+    # if we're looking at readmes me_read is true and if not, if we're looking at contributing files, it's false
+    #this is the window of days on either side of the event that we're looking at
+    window = 56
+    #print(upstream_vcs_link.split('/')[4])
+    project_dict = {}
+    project_dict['upstream_vcs_link'] = upstream_vcs_link
+    #event_date = dateutil.parser.parse(event_date).astimezone(timezone.utc)
+    upstream_vcs_link = upstream_vcs_link.strip()
+    if "github" in upstream_vcs_link or "gitlab" in upstream_vcs_link:
+        #making an evaluation that sub branches aren't being used and that people would fork if needed
+        #this only looks at main
+        upstream_vcs_link = "/".join(upstream_vcs_link.split("/")[0:5])
+        print(upstream_vcs_link)
+        full_temp_path = temp_dir + upstream_vcs_link.split('/')[4] + ".git"
+    else:
+        full_temp_path = temp_dir + upstream_vcs_link.split('/')[- 1] + ".git"
+    print(upstream_vcs_link)
+    if upstream_vcs_link == "https://gitlab.com/ubports/core" or upstream_vcs_link == "https://gitlab.freedesktop.org/xorg/lib":
+        shutil.rmtree(full_temp_path, ignore_errors=True)
+        return {}
+    repo = Git(uri=upstream_vcs_link, gitpath=full_temp_path)
+    commits = repo.fetch()
+    merge_pre, merge_post = 0, 0
+    #list of tuples which has date and whether it was a merge
+    commit_list = []
+    first_date_readme = ""
+    index = 0 
+    for commit in commits:
+        if index == 0:
+            project_dict['first_commit'] = commit['data']['CommitDate']
+            break
+        #print(commit['data'])
+    shutil.rmtree(full_temp_path, ignore_errors=True)
+    print(project_dict)
+    return project_dict
+
+
+
+def for_files():
+    csv_path = "final_data/deb_readme_did.csv"
+    count = 0 
+    with open(csv_path, 'r') as file:
+        csv_reader = csv.DictReader(file)
+        with open('062424_did_first_commit_readme_2.csv', "w") as writing_file:
+            # this would also have to get switched fro the cont dataset
+            keys = ['upstream_vcs_link', "first_commit"]
+            dict_writer = csv.DictWriter(writing_file, keys)
+            dict_writer.writeheader()
+            for row in csv_reader:
+                count += 1
+                if count < 2110:
+                    continue
+                print(row['upstream_vcs_link'])
+                # this would have to get switched to false for the cont dataset
+                dict_row = file_get_contribs(row['upstream_vcs_link'].strip(), row['event_date'])
+                dict_writer.writerow(dict_row)
+    
+
+if __name__ == "__main__":
+    #for_files()
+    file_get_contribs("https://github.com/Debian/perl-cross-debian", "00-00-00")
+    #file_get_pr("https://github.com/tqdm/tqdm", True)
+    #file_get_pr("https://github.com/GameServerManagers/LinuxGSM", True)
+    #file_get_pr("https://github.com/walling/unorm/issues/new/", True)
+    #file_get_pr("https://github.com/krahets/hello-algo/tree/dev1", True)
+    

new_contrib_window.py

@@ -10,8 +10,6 @@ from tqdm import tqdm
 import math
 
 key = os.environ.get('KKEXKEY')
-
-early_cutoff = dt.datetime(2008,2, 8)
 temp_dir = "/data/users/mgaughan/tmp/"
 temp_dir1 = "tmp/"
 
@@ -95,7 +93,7 @@ def for_files():
     count = 0 
     with open(csv_path, 'r') as file:
         csv_reader = csv.DictReader(file)
-        with open('42524_new_readme.csv', "w") as writing_file:
+        with open('062424_did_first_commit.csv', "w") as writing_file:
             # this would also have to get switched fro the cont dataset
             keys = ['upstream_vcs_link', "event_date", 'before_contrib_new', 'after_contrib_new', 'after_collab_new', 'before_collab_new', 'merge_pre', 'merge_post']
             dict_writer = csv.DictWriter(writing_file, keys)