#library(tidyverse) library(dplyr) library(lubridate) library(tidyr) library(purrr) library(stringr) https_commit_fp <- "/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/extensions_visualeditor_commits.csv" contains_http_but_not_url <- function(text) { if (is.na(text)) { return(FALSE) } # Split text by whitespace and check each word words <- str_split(text, "\\s+")[[1]] for (word in words) { if (!str_detect(word,"://")){ #http if (str_detect(word, "http")){ return(TRUE) } if (str_detect(word, "login")){ return(TRUE) } if (str_detect(word, "ssl")){ return(TRUE) } if (str_detect(word, "tls")){ return(TRUE) } if (startsWith(word, "cert")){ return(TRUE) } } } return(FALSE) } transform_commit_data <- function(filepath){ #basic, loading in the file df = read.csv(filepath, header = TRUE) temp_df <- df dir_path = dirname(filepath) file_name = basename(filepath) # TODO: this is project/event specific #event_date <- as.Date("2015-07-02") event_date <- as.Date("2013-07-01") #event_date <- as.Date("2013-04-25") #event_date <- as.Date("2012-12-11") # isolate project id project_id <- sub("_.*$", "", file_name) #make sure the dates are formatted correctly and state the project_id df <- df |> mutate(commit_date = ymd_hms(commit_date)) |> mutate(project_id = project_id) #get information about project age either in the "present" #or at the time of first commit oldest_commit_date <- min(as.Date(df$commit_date)) project_age <- as.numeric(as.Date("2025-02-10") - oldest_commit_date) #add that to the data df <- df |> mutate(age = project_age) #drop out data from ''before'' the release process df <- df |> filter(commit_date >= as.Date("2012-11-11")) #we are looking at weekly data, 6m before and 6m after #start_date <- event_date %m-% months(6) calculated_start_date <- event_date %m-% months(12) start_date <- max(calculated_start_date, oldest_commit_date) end_date <- event_date %m+% months(12) #getting the relative weeks to the publication date relative_week <- function(date, ref_date) { as.integer(as.numeric(difftime(date, ref_date, units = "days")) %/% 7) } df <- df |> mutate(relative_week = relative_week(commit_date, event_date)) # arrange(relative_week) |> # group_by(author_email) |> # mutate(new_author = ifelse(row_number() <= 5, 1, 0), # new_author_wmf = if_else(grepl("@wikimedia", author_email), new_author, 0), # new_author_unaff = if_else(!grepl("@wikimedia", author_email), new_author, 0)) |> # ungroup() # cut the df to all before 06-01-2013 # group by author_email # list all author_emails with >5 commits # for big df: if author not in the list, 'new' author old_author_list <- df |> filter(commit_date < as.Date("2013-06-06"))|> group_by(author_email) |> summarise(commit_count = n()) |> filter(commit_count > 5) |> pull(author_email) # Label authors as 'new' if they are not in the old_author_list df <- df |> mutate(new_author = ifelse(author_email %in% old_author_list, 0, 1), new_author_wmf = if_else(grepl("@wikimedia", author_email), new_author, 0), new_author_unaff = if_else(!grepl("@wikimedia", author_email) & !grepl("l10n-bot@translatewiki.net|tools.libraryupgrader@tools.wmflabs.org", author_email), new_author, 0)) #filler for when there are weeks without commits all_weeks <- seq(relative_week(start_date, event_date), relative_week(end_date, event_date)) complete_weeks_df <- expand.grid(relative_week = all_weeks, project_id = project_id, age = project_age) #for each week, get the list of unique authors that committed #cumulative_authors <- df %>% # arrange(relative_week) %>% # group_by(relative_week) %>% # summarize(cumulative_author_emails = list(unique(author_email)), .groups = 'drop') #same for each committer #cumulative_committers <- df %>% # arrange(relative_week) %>% # group_by(relative_week) %>% # summarize(cumulative_committer_emails = list(unique(committer_email)), .groups = 'drop') #now cut out the commit data that we don't care about df <- df |> filter(author_email != "jenkins-bot@gerrit.wikimedia.org") #in order: # - we group by project, week, ages # - and we summarize commit and authorship details # - we then fill in information for missingness # - and add in vars for before/after # - and weekly index weekly_commits <- df |> group_by(project_id, relative_week, age) |> summarise(commit_count = n(), author_emails = list(unique(author_email)), committer_emails = list(unique(committer_email)), mediawiki_dev_commit_count = sum(grepl("@users.mediawiki.org", author_email)), wikimedia_commit_count = sum(grepl("@wikimedia", author_email)), wikia_commit_count = sum(grepl("@wikia-inc.com", author_email)), bot_commit_count = sum(grepl("l10n-bot@translatewiki.net|tools.libraryupgrader@tools.wmflabs.org", author_email)), wmf_new_commit_count = sum(new_author_wmf), unaff_new_commit_count = sum(new_author_unaff), .groups = 'drop') |> right_join(complete_weeks_df, by=c("relative_week", "project_id", "age")) |> replace_na(list(commit_count = 0)) |> replace_na(list(wikimedia_commit_count = 0)) |> replace_na(list(l10n_commit_count = 0)) |> replace_na(list(jenkins_commit_count = 0)) |> replace_na(list(mediawiki_dev_commit_count = 0)) |> replace_na(list(wikia_commit_count = 0)) |> replace_na(list(wmf_new_commit_count = 0)) |> replace_na(list(unaff_new_commit_count = 0)) |> mutate(before_after = if_else(relative_week < 0, 0, 1)) |> select(-author_emails, -committer_emails) # then, to get the authorship details in # we check if the email data is present, if not we fill in blank # we bring in the information about authorship lists that we already had # then comparing the current week's author list with the previous week's cumulative list, or empty # ---- the length of that difference is the 'new' value # then we delete out the author list information #weekly_with_authorship <- weekly_commits |> # mutate( # author_emails = ifelse(is.na(author_emails), list(character()), author_emails), # committer_emails = ifelse(is.na(committer_emails), list(character()), committer_emails) # ) |> # left_join(cumulative_authors, by = "relative_week") |> # left_join(cumulative_committers, by = "relative_week") |> # mutate(new_author_emails = mapply(function(x, y) length(setdiff(x, y)), author_emails, lag(cumulative_author_emails, default = list(character(1)))), # new_committer_emails = mapply(function(x, y) length(setdiff(x, y)), committer_emails, lag(cumulative_committer_emails, default = list(character(1))))) #weekly_with_authorship <- weekly_with_authorship |> # mutate( # wikimedia_author_emails = mapply(function(x) length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), author_emails), # non_wikimedia_author_emails = mapply(function(x) length(x) - length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), author_emails), # wikimedia_committer_emails = mapply(function(x) length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), committer_emails), # non_wikimedia_committer_emails = mapply(function(x) length(x) - length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), committer_emails), # new_wikimedia_authors = mapply(function(x, y) length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), author_emails, lag(cumulative_author_emails, default = list(character(1)))), # new_non_wikimedia_authors = mapply(function(x, y) length(setdiff(x, y)) - length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), author_emails, lag(cumulative_author_emails, default = list(character(1)))), # new_wikimedia_committers = mapply(function(x, y) length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), committer_emails, lag(cumulative_committer_emails, default = list(character(1)))), # new_non_wikimedia_committers = mapply(function(x, y) length(setdiff(x, y)) - length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), committer_emails, lag(cumulative_committer_emails, default = list(character(1)))) # ) |> weekly_commits <- weekly_commits |> filter(relative_week >= (-34) & relative_week <= 13 ) #gracefully exit return(weekly_commits) } transform_relevant_commit_data <- function(filepath){ #basic, loading in the file df = read.csv(filepath, header = TRUE) temp_df <- df dir_path = dirname(filepath) file_name = basename(filepath) # TODO: this is project/event specific event_date <- as.Date("2015-07-02") #event_date <- as.Date("2013-07-01") #event_date <- as.Date("2013-04-25") #event_date <- as.Date("2012-12-11") # isolate project id project_id <- sub("_.*$", "", file_name) #make sure the dates are formatted correctly and state the project_id df <- df |> mutate(commit_date = ymd_hms(commit_date)) |> mutate(project_id = project_id) #get information about project age either in the "present" #or at the time of first commit oldest_commit_date <- min(as.Date(df$commit_date)) project_age <- as.numeric(as.Date("2025-02-10") - oldest_commit_date) #add that to the data df <- df |> mutate(age = project_age) #drop out data from ''before'' the release process df <- df |> filter(commit_date >= as.Date("2015-04-01")) #we are looking at weekly data, 6m before and 6m after #start_date <- event_date %m-% months(6) calculated_start_date <- event_date %m-% months(12) start_date <- max(calculated_start_date, oldest_commit_date) end_date <- event_date %m+% months(12) #getting the relative weeks to the publication date relative_week <- function(date, ref_date) { as.integer(as.numeric(difftime(date, ref_date, units = "days")) %/% 7) } df <- df |> mutate(relative_week = relative_week(commit_date, event_date))|> mutate(mentions_http = if_else(sapply(message, contains_http_but_not_url), TRUE, FALSE)) |> filter(mentions_http == TRUE) # arrange(relative_week) |> # group_by(author_email) |> # mutate(new_author = ifelse(row_number() <= 5, 1, 0), # new_author_wmf = if_else(grepl("@wikimedia", author_email), new_author, 0), # new_author_unaff = if_else(!grepl("@wikimedia", author_email), new_author, 0)) |> # ungroup() # cut the df to all before 06-01-2015 # group by author_email # list all author_emails with >5 commits # for big df: if author not in the list, 'new' author old_author_list <- df |> filter(commit_date < as.Date("2015-06-01"))|> group_by(author_email) |> summarise(commit_count = n()) |> filter(commit_count > 5) |> pull(author_email) # Label authors as 'new' if they are not in the old_author_list df <- df |> mutate(new_author = ifelse(author_email %in% old_author_list, 0, 1), new_author_wmf = if_else(grepl("@wikimedia", author_email), new_author, 0), new_author_unaff = if_else(!grepl("@wikimedia", author_email) & !grepl("l10n-bot@translatewiki.net|tools.libraryupgrader@tools.wmflabs.org", author_email), new_author, 0)) #filler for when there are weeks without commits all_weeks <- seq(relative_week(start_date, event_date), relative_week(end_date, event_date)) complete_weeks_df <- expand.grid(relative_week = all_weeks, project_id = project_id, age = project_age) #for each week, get the list of unique authors that committed #cumulative_authors <- df %>% # arrange(relative_week) %>% # group_by(relative_week) %>% # summarize(cumulative_author_emails = list(unique(author_email)), .groups = 'drop') #same for each committer #cumulative_committers <- df %>% # arrange(relative_week) %>% # group_by(relative_week) %>% # summarize(cumulative_committer_emails = list(unique(committer_email)), .groups = 'drop') #now cut out the commit data that we don't care about df <- df |> filter(author_email != "jenkins-bot@gerrit.wikimedia.org") #in order: # - we group by project, week, ages # - and we summarize commit and authorship details # - we then fill in information for missingness # - and add in vars for before/after # - and weekly index weekly_commits <- df |> group_by(project_id, relative_week, age) |> summarise(commit_count = n(), author_emails = list(unique(author_email)), committer_emails = list(unique(committer_email)), mediawiki_dev_commit_count = sum(grepl("@users.mediawiki.org", author_email)), wikimedia_commit_count = sum(grepl("@wikimedia", author_email)), wikia_commit_count = sum(grepl("@wikia-inc.com", author_email)), bot_commit_count = sum(grepl("l10n-bot@translatewiki.net|tools.libraryupgrader@tools.wmflabs.org", author_email)), wmf_new_commit_count = sum(new_author_wmf), unaff_new_commit_count = sum(new_author_unaff), .groups = 'drop') |> right_join(complete_weeks_df, by=c("relative_week", "project_id", "age")) |> replace_na(list(commit_count = 0)) |> replace_na(list(wikimedia_commit_count = 0)) |> replace_na(list(l10n_commit_count = 0)) |> replace_na(list(jenkins_commit_count = 0)) |> replace_na(list(mediawiki_dev_commit_count = 0)) |> replace_na(list(wikia_commit_count = 0)) |> replace_na(list(bot_commit_count = 0)) |> replace_na(list(wmf_new_commit_count = 0)) |> replace_na(list(unaff_new_commit_count = 0)) |> mutate(before_after = if_else(relative_week < 0, 0, 1)) |> select(-author_emails, -committer_emails) # then, to get the authorship details in # we check if the email data is present, if not we fill in blank # we bring in the information about authorship lists that we already had # then comparing the current week's author list with the previous week's cumulative list, or empty # ---- the length of that difference is the 'new' value # then we delete out the author list information #weekly_with_authorship <- weekly_commits |> # mutate( # author_emails = ifelse(is.na(author_emails), list(character()), author_emails), # committer_emails = ifelse(is.na(committer_emails), list(character()), committer_emails) # ) |> # left_join(cumulative_authors, by = "relative_week") |> # left_join(cumulative_committers, by = "relative_week") |> # mutate(new_author_emails = mapply(function(x, y) length(setdiff(x, y)), author_emails, lag(cumulative_author_emails, default = list(character(1)))), # new_committer_emails = mapply(function(x, y) length(setdiff(x, y)), committer_emails, lag(cumulative_committer_emails, default = list(character(1))))) #weekly_with_authorship <- weekly_with_authorship |> # mutate( # wikimedia_author_emails = mapply(function(x) length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), author_emails), # non_wikimedia_author_emails = mapply(function(x) length(x) - length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), author_emails), # wikimedia_committer_emails = mapply(function(x) length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), committer_emails), # non_wikimedia_committer_emails = mapply(function(x) length(x) - length(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x)), committer_emails), # new_wikimedia_authors = mapply(function(x, y) length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), author_emails, lag(cumulative_author_emails, default = list(character(1)))), # new_non_wikimedia_authors = mapply(function(x, y) length(setdiff(x, y)) - length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), author_emails, lag(cumulative_author_emails, default = list(character(1)))), # new_wikimedia_committers = mapply(function(x, y) length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), committer_emails, lag(cumulative_committer_emails, default = list(character(1)))), # new_non_wikimedia_committers = mapply(function(x, y) length(setdiff(x, y)) - length(setdiff(grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", x, value = TRUE), grep("@wikimedia|@users.mediawiki.org|@wikia-inc.com", y, value = TRUE))), committer_emails, lag(cumulative_committer_emails, default = list(character(1)))) # ) |> weekly_commits <- weekly_commits |> filter(relative_week >= (-14) & relative_week <= 52 ) #gracefully exit return(weekly_commits) } transformed <- transform_commit_data(https_commit_fp) output_filepath <-"/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/event_0421_extensions_ve_weekly_count.csv" write.csv(transformed, output_filepath, row.names = FALSE)