import re
import numpy as np
import pandas as pd
import glob
import copy

# Gensim
import gensim
import gensim.corpora as corpora, Dictionary
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModelfrom, Phrases

# spacy and nltk for lemmatization
import spacy
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer

stopwords = stopwords.words('english')

#loading data in, getting misc descriptors
def get_data_from_dir(directory):
    files = glob.glob(f"{directory}/*")
    data_list = []
    for file in files:
        text = open(file, encoding='utf-8').read()
        #TODO: here is where we can get data about word length and document length
        data_list.append(text)
    return data_list

#preprocessing text data
def preprocess(corpus_list):
    D = copy.copy(corpus_list)
    #mvp right now, can certainly be expanded as iterations of text analysis are done
    D = [[token for token in simple_preprocess(doc) if token not in stopwords]for doc in D]
    lemmatizer = WordNetLemmatizer()
    D_lemma = [[lemmatizer.lemmatize(token) for token in doc] for doc in D]
    return D_lemma

#preparing processed data for model usage
def text_preparation(lemmatized_text):
    #bigrams
    D_bigrams = copy.copy(lemmatized_text)
    bigram = Phrases(D_bigrams, min_count=2)
    for i in range(len(lemmatized_text)):
        for token in bigram[D_bigrams[i]]:
            if '_' in token:
                D_bigrams[i].append(token)
    #id2word
    id2word = Dictionary(D_bigrams)
    id2word.filter_extremes(no_below=2, no_above=0.5)
    #bow representation 
    bag_of_words = [id2word.doc2bow(doc) for doc in D_bigrams]
    return bag_of_words, id2word

#TODO: identify best LDA model here
def lda_model_identification(bow, id2word, bigrams ):
    coherence = []
    perplexity = []
    # For between 1 and 6 topics
    for k in [5, 10, 15, 20]: 
        print('num of topics // k =: '+ str(k))
        Lda = gensim.models.ldamodel.LdaModel
        # Train a model for that number of topics
        ldamodel = Lda(bow, num_topics=k, id2word = id2word, passes=40,\
                    iterations=200, chunksize = 1000, eval_every = None)
        # Get the coherence value for the trained model
        cm = gensim.models.coherencemodel.CoherenceModel(model=ldamodel, texts=bigrams,\
                                                        dictionary=id2word, coherence='c_v')
        coherence.append((k,cm.get_coherence()))
        # Append the perplexity for the trained model
        perplexity.append((k,ldamodel.log_perplexity(bow)))
    print(coherence)
    print(perplexity)   

#TODO: implement best LDA model here 

#TODO: evaluate model and identified topics 


if __name__ == "__main__":
    document_directory = "TKTKTKKT"
    listed_corpus = get_data_from_dir(document_directory)
    lemmatized_corpus = preprocess(listed_corpus)
    prepped_corpus, id2word = text_preparation(lemmatized_corpus)