new analysis of commit activity

commit_analysis/commit_count_collation.R

@@ -4,7 +4,7 @@ library(lubridate)
 library(tidyr)
 library(purrr)
 
-ve_commit_fp <- "/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/extensions_visualeditor_commits.csv"
+ve_commit_fp <- "/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/mediawiki_core_commits.csv"
 
 transform_commit_data <- function(filepath){
   #basic, loading in the file 
@@ -15,9 +15,9 @@ transform_commit_data <- function(filepath){
 
   
   # TODO: this is project/event specific
-  #event_date <- as.Date("2013-07-01")
+  event_date <- as.Date("2013-07-01")
   #event_date <- as.Date("2013-04-25")
-  event_date <- as.Date("2012-12-11")
+  #event_date <- as.Date("2012-12-11")
   
   # isolate project id
   project_id <- sub("_.*$", "", file_name)
@@ -48,13 +48,34 @@ transform_commit_data <- function(filepath){
   }
   
   df <- df |>
-    mutate(relative_week = relative_week(commit_date, event_date)) |>
-    arrange(relative_week) |>
+    mutate(relative_week = relative_week(commit_date, event_date))
+  #  arrange(relative_week) |>
+  #  group_by(author_email) |>
+  #  mutate(new_author = ifelse(row_number() <= 5, 1, 0),
+  #         new_author_wmf = if_else(grepl("@wikimedia", author_email), new_author, 0),
+  #         new_author_unaff = if_else(!grepl("@wikimedia", author_email), new_author, 0)) |>
+  #  ungroup()
+  
+  # cut the df to all before 06-01-2013
+  # group by author_email 
+  # list all author_emails with >5 commits 
+  # for big df: if author not in the list, 'new' author
+  old_author_list <- df |>
+    filter(commit_date < as.Date("2013-06-01"))|>
     group_by(author_email) |>
-    mutate(new_author = ifelse(row_number() <= 5, 1, 0),
-           new_author_wmf = if_else(grepl("@wikimedia", author_email), new_author, 0),
-           new_author_unaff = if_else(!grepl("@wikimedia", author_email), new_author, 0)) |>
-    ungroup()
+    summarise(commit_count = n()) |>
+    filter(commit_count > 5) |>
+    pull(author_email)
+  
+  # Label authors as 'new' if they are not in the old_author_list
+  df <- df |>
+    mutate(new_author = ifelse(author_email %in% old_author_list, 0, 1), 
+            new_author_wmf = if_else(grepl("@wikimedia", author_email), 
+                                     new_author, 0),
+            new_author_unaff = if_else(!grepl("@wikimedia", author_email) &
+                                         !grepl("l10n-bot@translatewiki.net|tools.libraryupgrader@tools.wmflabs.org", author_email),
+                                       new_author, 0))
+
   
   #filler for when there are weeks without commits
   all_weeks <- seq(relative_week(start_date, event_date), relative_week(end_date, event_date))
@@ -93,8 +114,8 @@ transform_commit_data <- function(filepath){
               wikimedia_commit_count = sum(grepl("@wikimedia", author_email)),
               wikia_commit_count = sum(grepl("@wikia-inc.com", author_email)),
               bot_commit_count = sum(grepl("l10n-bot@translatewiki.net|tools.libraryupgrader@tools.wmflabs.org", author_email)),
-              wmf_ft_commit_count = sum(new_author_wmf),
-              unaff_ft_commit_count = sum(new_author_unaff),
+              wmf_new_commit_count = sum(new_author_wmf),
+              unaff_new_commit_count = sum(new_author_unaff),
               .groups = 'drop') |>
     right_join(complete_weeks_df, by=c("relative_week", "project_id", "age")) |>
     replace_na(list(commit_count = 0)) |>
@@ -103,8 +124,8 @@ transform_commit_data <- function(filepath){
     replace_na(list(jenkins_commit_count = 0)) |>
     replace_na(list(mediawiki_dev_commit_count = 0)) |>
     replace_na(list(wikia_commit_count = 0)) |>
-    replace_na(list(wmf_ft_commit_count = 0)) |>
-    replace_na(list(unaff_ft_commit_count = 0)) |>
+    replace_na(list(wmf_new_commit_count = 0)) |>
+    replace_na(list(unaff_new_commit_count = 0)) |>
     mutate(before_after = if_else(relative_week < 0, 0, 1)) |>
     select(-author_emails, -committer_emails)
   # then, to get the authorship details in
@@ -143,7 +164,7 @@ transform_commit_data <- function(filepath){
 }
 
 transformed <- transform_commit_data(ve_commit_fp)
-output_filepath <-"/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/en-testing_0316_extensions_ve_weekly_commit_count_data.csv"
+output_filepath <-"/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/event_0401_mediawiki_core_weekly_commit_count_data.csv"
 
 write.csv(transformed, output_filepath, row.names = FALSE)
 

commit_analysis/commit_plotting.R

@@ -1,9 +1,41 @@
 library(tidyverse)
-count_data_fp <-"/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/en-testing_0217_extensions_ve_weekly_commit_count_data.csv"
+count_data_fp <-"/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/event_0401_mediawiki_core_weekly_commit_count_data.csv"
 input_df <- read.csv(count_data_fp, header = TRUE) 
 
 input_df$nonbot_commit_count <- input_df$commit_count - input_df$bot_commit_count
 
+library(scales)
+library(ggplot2)
+
+long_df <- input_df |>
+  tidyr::pivot_longer(cols = c(nonbot_commit_count, unaff_new_commit_count, wmf_new_commit_count),
+                      names_to = "commit_type",
+                      values_to = "lengthened_commit_count")
+
+affiliationColors <-
+  setNames( c('black','#5da2d8', '#c7756a')
+            ,c("nonbot_commit_count","unaff_new_commit_count", "wmf_new_commit_count"))
+
+new_authors <- long_df |>
+  ggplot(aes(x=relative_week, 
+             y=lengthened_commit_count, 
+             color=factor(commit_type))) +
+  geom_line() +
+  geom_point() + 
+  labs(x = "Relative Week", y = "Commits", color="Commit Type") +
+  scale_color_manual(values = affiliationColors,
+                     labels = c("nonbot_commit_count" = "Total Nonbot Commits",
+                                "unaff_new_commit_count" = "New Unaffiliated Commits",
+                                "wmf_new_commit_count" = "New WMF Commits")) +
+  ggtitle("MW-core Commits Around Wide Release ('New' contributors <= 5 commits before 06-01-2013)") +
+  theme_bw() + 
+  theme(legend.position = "top")
+new_authors
+
+ggsave(filename = "0401-core-event-new-commits.png", plot = new_authors, width = 12, height = 9, dpi = 800)
+
+
+
 window_num <- 12
 intermediate_df <- input_df |>
   mutate(nonbot_commit_count = commit_count - bot_commit_count)|>
@@ -13,9 +45,6 @@ intermediate_df <- input_df |>
   dplyr::select(-wikia_commit_count) |>
   filter(relative_week >= (- window_num) & relative_week <= (window_num)) 
 
-library(scales)
-library(ggplot2)
-
 time_plot <- intermediate_df |>
   ggplot(aes(x=relative_week, y=nonbot_commit_count)) +
   labs(x="Weekly Offset", y="Nonbot Commit Count") +

mgaughan-rstudio-server_25101978.out

@@ -0,0 +1,17 @@
+1. SSH tunnel from your workstation using the following command:
+
+   ssh -N -L 8787:n3439:60937 mjilg@klone.hyak.uw.edu
+
+   and point your web browser to http://localhost:8787
+
+2. log in to RStudio Server using the following credentials:
+
+   user: mjilg
+   password: Uoe+nDeFvcaRiOows2lT
+
+When done using RStudio Server, terminate the job by:
+
+1. Exit the RStudio Session ("power" button in the top right corner of the RStudio window)
+2. Issue the following command on the login node:
+
+      scancel -f 25101978