govdoc-cr-analysis/cleaning_scripts/get_weekly_commit_counts.R

library(tidyverse)
# test data directory: /gscratch/comdata/users/mjilg/program_testing/
# load in the paritioned directories
library(dplyr)
library(lubridate)

#for a given file we want to get the count data and produce a csv
readme_pub_info <- "/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/final_data/metadata/0205_README_publication_commits.csv"
contributing_pub_info <- "/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/final_data/metadata/0205_CONTRIBUTING_publication_commits.csv"
readme_dir <- "/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/final_data/main_commit_data/readme/"
contributing_dir <- "/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/final_data/main_commit_data/contributing/"

test_file <- "/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/13125_hyak_test/main_commit_data/contributing/_linuxmint_cjs.git_commits.csv"

transform_commit_data <- function(filepath, ref_df){
  #basic, loading in the file 
  df = read.csv(filepath, header = TRUE) 
  temp_df <- df
  dir_path = dirname(filepath)
  file_name = basename(filepath)
  
  # isolate project id
  project_id <- sub("_commits\\.csv$", "", file_name)
  project_id <- sub("^_", "", project_id)
  
  #make sure the dates are formatted correctly and state the project_id
  df <- df |>
    mutate(commit_date = ymd_hms(commit_date)) |>
    mutate(project_id = project_id)
  
  #find the publication entry, in the specified df
  matched_entry <- ref_df |>
    filter(repo_id == project_id)
  pub_commit_date <- min(as.Date(matched_entry$commit_date))
  
  #get information about project age either in the "present" 
  #or at the time of first commit
  oldest_commit_date <- min(as.Date(df$commit_date))
  project_age <- as.numeric(as.Date("2024-06-24") - oldest_commit_date)
  age_at_commit <- as.numeric(pub_commit_date - oldest_commit_date)
  
  #add that to the data 
  df <- df |> 
    mutate(age = project_age,
           age_at_commit = age_at_commit)
  
  #we are looking at weekly data, 6m before and 6m after
  start_date <- pub_commit_date %m-% months(6)
  #calculated_start_date <- pub_commit_date %m-% months(6)
  #start_date <- max(calculated_start_date, oldest_commit_date)
  end_date <- pub_commit_date %m+% months(6)
  
  #getting the relative weeks to the publication date
  relative_week <- function(date, ref_date) {
    as.integer(as.numeric(difftime(date, ref_date, units = "days")) %/% 7)
  }
  
  df <- df |>
    mutate(relative_week = relative_week(commit_date, pub_commit_date))
  
  #filler for when there are weeks without commits
  all_weeks <- seq(relative_week(start_date, pub_commit_date), relative_week(end_date, pub_commit_date))
  complete_weeks_df <- expand.grid(relative_week = all_weeks, 
                                   project_id = project_id,
                                   age = project_age,
                                   age_at_commit = age_at_commit)
  
  #for each week, get the list of unique authors that committed
  cumulative_authors <- df %>%
    arrange(relative_week) %>%
    group_by(relative_week) %>%
    summarize(cumulative_author_emails = list(unique(author_email)), .groups = 'drop')
  #same for each committer
  cumulative_committers <- df %>%
    arrange(relative_week) %>%
    group_by(relative_week) %>%
    summarize(cumulative_committer_emails = list(unique(committer_email)), .groups = 'drop')
  
  #now cut out the commit data that we don't care about 
  df <- df |>
    filter(as.Date(pub_commit_date) >= start_date & as.Date(pub_commit_date) <= end_date)
  
  #in order:
  # - we group by project, week, ages
  # - and we summarize commit and authorship details
  # - we then fill in information for missingness
  # - and add in vars for before/after
  # - and weekly index
  weekly_commits <- df |>
    group_by(project_id, relative_week, age, age_at_commit) |>
    summarise(commit_count = n(),
              author_emails = list(unique(author_email)),
              committer_emails = list(unique(committer_email)),
              .groups = 'drop') |>
    right_join(complete_weeks_df, by=c("relative_week", "project_id", "age", "age_at_commit")) |>
    replace_na(list(commit_count = 0)) |>
    mutate(before_after = if_else(relative_week < 0, 0, 1)) 
  # then, to get the authorship details in
  # we check if the email data is present, if not we fill in blank 
  # we bring in the information about authorship lists that we already had 
  # then comparing the current week's author list with the previous week's cumulative list, or empty
  #  ---- the length of that difference is the 'new' value
  # then we delete out the author list information
  weekly_with_authorship <- weekly_commits |>
    mutate(
      author_emails = ifelse(is.na(author_emails), list(character()), author_emails),
      committer_emails = ifelse(is.na(committer_emails), list(character()), committer_emails)
    ) |>
    left_join(cumulative_authors, by = "relative_week") |>
    left_join(cumulative_committers, by = "relative_week") |>
    mutate(new_author_emails = mapply(function(x, y) length(setdiff(x, y)), author_emails, lag(cumulative_author_emails, default = list(character(1)))),
           new_committer_emails = mapply(function(x, y) length(setdiff(x, y)), committer_emails, lag(cumulative_committer_emails, default = list(character(1))))) |>
    select(-author_emails, -committer_emails, -cumulative_author_emails, -cumulative_committer_emails)
  
  #gracefully exit
  return(weekly_with_authorship)
}

#then for all files in a directory
transform_directory_of_commit_data <- function(is_readme) {
  ref_df <- read.csv(contributing_pub_info)
  dir_path <- contributing_dir
  if (is_readme){
    ref_df <- read.csv(readme_pub_info)
    dir_path <- readme_dir
  }
  counted_list <- list()
  file_list <- list.files(path = dir_path, pattern = "*.csv", full.names = TRUE)
  for (filepath in file_list) {
    transformed_data <- transform_commit_data(filepath, ref_df)
    counted_list <- append(counted_list, list(transformed_data))
  }
  counted_df <- bind_rows(counted_list)
  
  return(counted_df)
}

#test_ref_df <- read.csv(contributing_pub_info)
#test_count_df <- transform_commit_data(test_file, test_ref_df)

#below is for contributing file 
#big_df <- transform_directory_of_commit_data(is_readme=FALSE)
#output_filepath <-"/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/final_data/metadata/co_0205_CONTRIBUTING_weekly_count_data.csv"

#below is for readme
big_df <- transform_directory_of_commit_data(is_readme=TRUE)
output_filepath <-"/mmfs1/gscratch/comdata/users/mjilg/govdoc-cr-data/final_data/metadata/0205_README_weekly_count_data.csv"
#validation testing 
#length(unique(big_df$project_id))
#filtered_df <- test_big_df %>%
#  filter(commit_count != 0, new_author_emails == 0, new_committer_emails == 0)
test_df <- big_df |>
  filter(age_at_commit >= 0)



#another graceful exit
write.csv(test_df, output_filepath, row.names = FALSE)