import pandas as pd
import os
import datetime as dt
import json
import shutil
import re

from datetime import datetime, timedelta


def repo_id_from_upstream(vcs_link):
    if 'github' in vcs_link:
        repo_id = vcs_link[len('https://github.com/'):]
    else:
        parts = vcs_link.split('/')
        repo_id = '/'.join(parts[3:])
    und_repo_id = '_'.join(repo_id.split("/"))
    return und_repo_id

def get_readme_list(csv_path):
    #main_df = pd.read_csv('final_data/deb_full_data.csv')
    existing_list = pd.read_csv(csv_path)
    existing_list['repo_id'] = existing_list['upstream_vcs_link'].apply(repo_id_from_upstream)
    present_data = existing_list[['upstream_vcs_link', 'repo_id']]
    present_data.to_csv('012925_readme_manifest.csv', index=False)


def get_contributing_list(csv_path):
    existing_list = pd.read_csv(csv_path)
    existing_list['repo_id'] = existing_list['upstream_vcs_link'].apply(repo_id_from_upstream)
    present_data = existing_list[['upstream_vcs_link', 'repo_id']]
    present_data.to_csv('012925_contributing_manifest.csv', index=False)


def check_collected_files(dirpath):
    if "contributing" in dirpath:
        manifest = pd.read_csv('012925_contributing_manifest.csv')
    else:
        manifest = pd.read_csv('012925_readme_manifest.csv')

    csv_files = [f for f in os.listdir(dirpath) if f.endswith('_commits.csv')]
    file_ids = {f[1:-12] if f.startswith('_') else f[:-12] for f in csv_files}

    manifest_ids = set(manifest["repo_id"])

    matched_repo_ids = file_ids & manifest_ids
    unmatched_repo_ids = file_ids - matched_repo_ids
    unmatched_files = [f for f in csv_files if (f[1:-12] if f.startswith('_') else f[:-12]) in unmatched_repo_ids]

    return unmatched_files, matched_repo_ids

def hard_codes(vcs_link, filename):
    if vcs_link == "https://github.com/df7cb/pg_filedump.git" and filename == "pg_filedump.git_README.pg_filedump":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/sleuthkit/sleuthkit" and filename == "sleuthkit_README_win32.txt":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/metlov/cycle.git" and filename == "cycle.git_README_ru.html":
        rel_filename = filename
        return True, rel_filename
    if vcs_link ==  "https://github.com/winchen/engauge_debian" and filename == "engauge_debian_README_for_osx":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/babelouest/yder" and filename == "yder_README_8md.html":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/SebWouters/CheMPS2" and filename == "CheMPS2_README_8md_source.html":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/TACC/Lmod" and filename == "Lmod_README_lua_modulefiles.txt":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/hunspell/hyphen.git" and filename == "hyphen.git_README_hyph_en_US.txt":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/greenbone/openvas" and filename == "openvas_UPGRADE_README":
        rel_filename = filename
        return True, rel_filename
    if vcs_link == "https://github.com/MightyCreak/diffuse.git" and filename == "diffuse.git_README_ru":
        rel_filename = filename
        return True, rel_filename
    return False, ""

def cp_existing_docs(matched_repo_ids, is_readme):
    if not is_readme:
        existing_dir = "/data/users/mgaughan/kkex/time_specific_files/contributing4/"
        dest_dir = "/data/users/mgaughan/kkex/012825_cam_revision_main/second_batch/first_version_documents/existing_contributing/"
        manifest = pd.read_csv('012925_contributing_manifest.csv')
    else:
        existing_dir = "/data/users/mgaughan/kkex/time_specific_files/readme4/"
        dest_dir = "/data/users/mgaughan/kkex/012825_cam_revision_main/second_batch/first_version_documents/existing_readme/"
        manifest = pd.read_csv('012925_readme_manifest.csv')
    filtered_df = manifest[manifest['repo_id'].isin(matched_repo_ids)]
    print(len(filtered_df))
    successes = 0
    failures = []
    for index,row in filtered_df.iterrows():
        vcs_link = row['upstream_vcs_link']
        project = row['upstream_vcs_link'].split("/")[-1]
        rel_filename = ""
        #print(project)
        is_success = False
        for filename in os.listdir(existing_dir):
            if filename.startswith(project):
                rel_filename = filename
                #print(rel_filename)
                target_path = existing_dir + rel_filename
                dest_path = os.path.join(dest_dir, f"{row['repo_id']}_hullabaloo_{rel_filename.split('_')[-1]}")
                #print(dest_path)
                successes+= 1
                is_success = True
                shutil.copy(target_path, dest_path)
                break
            '''
            if rel_filename != "":
                target_path = existing_dir + rel_filename
                #print(target_path)
                dest_path = os.path.join(dest_dir, f"{row['repo_id']}_hullabaloo_{rel_filename}")
                #print(target_path, full_temp_path[:-4], filename, destination_path)
                shutil.copy(target_path, dest_path)
                return filename
            else:
                print("error!")
                print(project)
            '''
        if not is_success:
            failures.append(project)
    print(successes)
    print(failures)
    # match those to the files in "/data/users/mgaughan/time_specific_files/{doctype}3"
    # copy matched files to "/data/users/mgaughan/kkex/012825_cam_revision_main/first_version_documents/{doctype}/"

def document_in_diffs(diffs, doc_name):
        #return any(d['parent_filepath'] == doc_name or d['child_filepath'] == doc_name for d in diffs)
        if doc_name == "README":
            pattern = ": 'README"
        else:
            pattern = ": 'CONTRIBUTING"
        #if re.search(pattern, diffs):
        if pattern in diffs:
            return True
        return False

def get_doc_publication(dirpath, file, doc_name):
    unmatched_df = pd.read_csv(dirpath + file)
    # because we only care about the date, not going to un-flatten the timezones
    unmatched_df['commit_date'] = pd.to_datetime(unmatched_df['commit_date'], utc=True)
    unmatched_df = unmatched_df.sort_values(by='commit_date', ascending=True)
    #print(unmatched_df.head())
    unmatched_df['diff_contains_document'] = unmatched_df['diff_info'].apply(lambda diffs: document_in_diffs(diffs, doc_name))
    relevant_commits = unmatched_df[unmatched_df['diff_contains_document'] == True]
    first_instance = relevant_commits.head(1)
    #lagging 
    commit_dt = first_instance['commit_date'].iloc[0]
    lagged_date = commit_dt + pd.Timedelta(days=7)
    filtered_df = relevant_commits[relevant_commits['commit_date'] < lagged_date].sort_values(by='commit_date', ascending=False)
    lagged_hash = filtered_df['commit_hash'].iloc[0]
    #first_instance = unmatched_df.loc["'parent_filepath': 'README'" in unmatched_df['diff_info']].head(1)
    return first_instance, lagged_hash

def process_vcs_links(link):
    if "github" in link:
        repo_id = link[len('https://github.com/'):]
    else:
        parts = link.split('/')
        domain = parts[2]
        repo_id = '/'.join(parts[3:])
    und_repo_id = '_'.join(repo_id.split("/"))
    return und_repo_id

def get_doc_pub_for_all(is_readme):
    full_data = pd.read_csv('for_batching/deb_full_data.csv')
    full_data['upstream_vcs_link'] = full_data['upstream_vcs_link'].str.strip()
    full_data['repo_id'] = full_data['upstream_vcs_link'].apply(lambda link: pd.Series(process_vcs_links(link)))
    if is_readme:
        dirpath = "/data/users/mgaughan/kkex/012825_cam_revision_main/final_data/main_commit_data/readme/"
        doc_name = "README"
    else: 
        dirpath = "/data/users/mgaughan/kkex/012825_cam_revision_main/final_data/main_commit_data/contributing/"
        doc_name = "CONTRIBUTING"
    csv_files = [f for f in os.listdir(dirpath) if f.endswith('_commits.csv')]
    first_instances = []
    index = 0
    for file in csv_files:
        index += 1
        first_instance, lagged_hash = get_doc_publication(dirpath, file, doc_name)
        first_instance['lagged_hash'] = lagged_hash
        if file.startswith('_'):
            repo_id = file[1:-12]
        else:
            repo_id = file[:-12]
        first_instance['repo_id'] = repo_id
        #   find the upstream vcs from the full_data list 
        id_array = repo_id.split("_")
        project_handle = ("/").join(id_array)
        first_instance['project_handle'] = project_handle

        #matching with vcs 
        result = full_data[full_data['repo_id'] == repo_id]
        first_match = result['upstream_vcs_link'].values[0]
        first_instance['upstream_vcs_link'] = first_match
        #print(first_instance['upstream_vcs_link'])
        first_instance.drop(['message', 'diff_info'], axis=1, inplace=True)
        # https://invent.kde.org/plasma/powerdevil.git
        if (not first_instance.empty and first_instance['upstream_vcs_link'] is not None):
            first_instances.append(first_instance)
        else:
            print(f"at index: {index}")
            print(f"no first commit found for {repo_id}") 
    #save the 
    first_instances_df = pd.concat(first_instances, ignore_index=True)
    first_instances_df.to_csv(f'0205_{doc_name}_publication_commits.csv', index = False)

def list_for_doc_collection(unmatched_files, is_readme):
    if is_readme:
        dirpath = "/data/users/mgaughan/kkex/012825_cam_revision_main/second_batch/readme_commits/"
        doc_name = "README"
    else: 
        dirpath = "/data/users/mgaughan/kkex/012825_cam_revision_main/second_batch/contributing_commits/"
        doc_name = "CONTRIBUTING"
    full_data = pd.read_csv('for_batching/deb_full_data.csv')
    first_instances = []
    #print(doc_name)
    index = 0 
    for file in unmatched_files:
        index += 1
        #print(dirpath)
        first_instance = get_doc_publication(dirpath, file, doc_name)
        if file.startswith('_'):
            repo_id = file[1:-12]
        else:
            repo_id = file[:-12]
        first_instance['repo_id'] = repo_id
        id_array = repo_id.split("_")
        project_handle = ("/").join(id_array)
        first_instance['project_handle'] = project_handle
        full_data['upstream_vcs_link'] = full_data['upstream_vcs_link'].str.strip()

        #matching with vcs 
        mask = pd.Series([True] * len(full_data))
        for id_part in id_array:
            mask &= full_data['upstream_vcs_link'].str.contains(id_part, case=False, na=False)
        result = full_data[mask]

        #print(repo_id)
        first_match = result['upstream_vcs_link'].values[0]
        #print(first_match)
        first_instance['upstream_vcs_link'] = first_match
        #print(first_instance['upstream_vcs_link'])
        first_instance.drop(['message', 'diff_info'], axis=1, inplace=True)
        # https://invent.kde.org/plasma/powerdevil.git
        if (not first_instance.empty and first_instance['upstream_vcs_link'] is not None):
            first_instances.append(first_instance)
        else:
            print(f"at index: {index}")
            print(f"no first commit found for {repo_id}") 
        if index > 15:
            break
    first_instances_df = pd.concat(first_instances, ignore_index=True)
    first_instances_df.to_csv(f'020125_{doc_name}_for_download.csv', index = False)

if __name__ == "__main__":
    #get_contributing_licsv_files = [f for f in os.listdir(dirpath) if f.endswith('_commits.csv')]
    is_readme = True
    get_doc_pub_for_all(is_readme)
    #unmatched_files, matched_repo_ids = check_collected_files("/data/users/mgaughan/kkex/012825_cam_revision_main/second_batch/readme_commits")
    #print(len(matched_repo_ids))
    #cp_existing_docs(matched_repo_ids, is_readme)
    #list_for_doc_collection(unmatched_files, is_readme)
    #print(len(matched_repo_ids))
    #print(len(unmatched_files))