From 5a40465a629a1d7d95dbec9730d3950842bcb4f5 Mon Sep 17 00:00:00 2001 From: Nathan TeBlunthuis Date: Wed, 8 Jun 2022 17:01:27 -0700 Subject: [PATCH] add support for umap->hdbscan clustering method --- clustering/Makefile | 21 +- clustering/clustering_base.py | 41 ++++ clustering/grid_sweep.py | 16 ++ clustering/lsi_base.py | 17 +- clustering/umap_hdbscan_clustering.py | 221 ++++++++++++++++++++++ clustering/umap_hdbscan_clustering_lsi.py | 114 +++++++++++ 6 files changed, 428 insertions(+), 2 deletions(-) create mode 100644 clustering/umap_hdbscan_clustering.py create mode 100644 clustering/umap_hdbscan_clustering_lsi.py diff --git a/clustering/Makefile b/clustering/Makefile index 9643f52..2ba9c0c 100644 --- a/clustering/Makefile +++ b/clustering/Makefile @@ -3,6 +3,9 @@ srun_singularity=source /gscratch/comdata/users/nathante/cdsc_reddit/bin/activat similarity_data=/gscratch/comdata/output/reddit_similarity clustering_data=/gscratch/comdata/output/reddit_clustering kmeans_selection_grid=--max_iters=[3000] --n_inits=[10] --n_clusters=[100,500,1000,1250,1500,1750,2000] + +umap_hdbscan_selection_grid=--min_cluster_sizes=[2] --min_samples=[2,3,4,5] --cluster_selection_epsilons=[0,0.01,0.05,0.1,0.15,0.2] --cluster_selection_methods=[eom,leaf] --n_neighbors=[5,15,25,50,75,100] --learning_rate=[1] --min_dist=[0,0.1,0.25,0.5,0.75,0.9,0.99] --local_connectivity=[1] + hdbscan_selection_grid=--min_cluster_sizes=[2,3,4,5] --min_samples=[2,3,4,5] --cluster_selection_epsilons=[0,0.01,0.05,0.1,0.15,0.2] --cluster_selection_methods=[eom,leaf] affinity_selection_grid=--dampings=[0.5,0.6,0.7,0.8,0.95,0.97,0.99] --preference_quantiles=[0.1,0.3,0.5,0.7,0.9] --convergence_iters=[15] @@ -91,12 +94,28 @@ ${terms_10k_output_lsi}/hdbscan/selection_data.csv:selection.py ${terms_10k_inpu ${authors_tf_10k_output_lsi}/hdbscan/selection_data.csv:clustering.py ${authors_tf_10k_input_lsi} clustering_base.py hdbscan_clustering.py $(srun_singularity) python3 hdbscan_clustering_lsi.py --inpath=${authors_tf_10k_input_lsi} --outpath=${authors_tf_10k_output_lsi}/hdbscan --savefile=${authors_tf_10k_output_lsi}/hdbscan/selection_data.csv $(hdbscan_selection_grid) +${authors_tf_10k_output_lsi}/umap_hdbscan/selection_data.csv:umap_hdbscan_clustering_lsi.py + $(srun_singularity) python3 umap_hdbscan_clustering_lsi.py --inpath=${authors_tf_10k_input_lsi} --outpath=${authors_tf_10k_output_lsi}/umap_hdbscan --savefile=${authors_tf_10k_output_lsi}/umap_hdbscan/selection_data.csv $(umap_hdbscan_selection_grid) + + ${terms_10k_output_lsi}/best_hdbscan.feather:${terms_10k_output_lsi}/hdbscan/selection_data.csv pick_best_clustering.py $(srun_singularity) python3 pick_best_clustering.py $< $@ --min_clusters=50 --max_isolates=5000 --min_cluster_size=2 ${authors_tf_10k_output_lsi}/best_hdbscan.feather:${authors_tf_10k_output_lsi}/hdbscan/selection_data.csv pick_best_clustering.py $(srun_singularity) python3 pick_best_clustering.py $< $@ --min_clusters=50 --max_isolates=5000 --min_cluster_size=2 +${authors_tf_10k_output_lsi}/best_umap_hdbscan_2.feather:${authors_tf_10k_output_lsi}/umap_hdbscan/selection_data.csv pick_best_clustering.py + $(srun_singularity) python3 pick_best_clustering.py $< $@ --min_clusters=50 --max_isolates=5000 --min_cluster_size=2 + +best_umap_hdbscan.feather:${authors_tf_10k_output_lsi}/best_umap_hdbscan_2.feather + +# {'lsi_dimensions': 700, 'outpath': '/gscratch/comdata/output/reddit_clustering/subreddit_comment_authors-tf_10k_LSI/umap_hdbscan', 'silhouette_score': 0.27616957, 'name': 'mcs-2_ms-5_cse-0.05_csm-leaf_nn-15_lr-1.0_md-0.1_lc-1_lsi-700', 'n_clusters': 547, 'n_isolates': 2093, 'silhouette_samples': '/gscratch/comdata/output/reddit_clustering/subreddit_comment_authors-tf_10k_LSI/umap_hdbscan/silhouette_samples-mcs-2_ms-5_cse-0.05_csm-leaf_nn-15_lr-1.0_md-0.1_lc-1_lsi-700.feather', 'min_cluster_size': 2, 'min_samples': 5, 'cluster_selection_epsilon': 0.05, 'cluster_selection_method': 'leaf', 'n_neighbors': 15, 'learning_rate': 1.0, 'min_dist': 0.1, 'local_connectivity': 1, 'n_isolates_str': '2093', 'n_isolates_0': False} + +best_umap_grid=--min_cluster_sizes=[2] --min_samples=[5] --cluster_selection_epsilons=[0.05] --cluster_selection_methods=[leaf] --n_neighbors=[15] --learning_rate=[1] --min_dist=[0.1] --local_connectivity=[1] --save_step1=True + +umap_hdbscan_coords: + python3 umap_hdbscan_clustering_lsi.py --inpath=${authors_tf_10k_input_lsi} --outpath=${authors_tf_10k_output_lsi}/umap_hdbscan --savefile=/dev/null ${best_umap_grid} + clean_affinity: rm -f ${authors_10k_output}/affinity/selection_data.csv rm -f ${authors_tf_10k_output}/affinity/selection_data.csv @@ -159,7 +178,7 @@ clean_lsi_terms: clean: clean_affinity clean_kmeans clean_hdbscan -PHONY: clean clean_affinity clean_kmeans clean_hdbscan clean_authors clean_authors_tf clean_terms terms_10k authors_10k authors_tf_10k +PHONY: clean clean_affinity clean_kmeans clean_hdbscan clean_authors clean_authors_tf clean_terms terms_10k authors_10k authors_tf_10k best_umap_hdbscan.feather umap_hdbscan_coords # $(clustering_data)/subreddit_comment_authors_30k.feather/SUCCESS:selection.py $(similarity_data)/subreddit_comment_authors_30k.feather clustering.py # $(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors_30k.feather $(clustering_data)/subreddit_comment_authors_30k $(selection_grid) -J 10 && touch $(clustering_data)/subreddit_comment_authors_30k.feather/SUCCESS diff --git a/clustering/clustering_base.py b/clustering/clustering_base.py index 3778fc3..ced627d 100644 --- a/clustering/clustering_base.py +++ b/clustering/clustering_base.py @@ -1,3 +1,4 @@ +import pickle from pathlib import Path import numpy as np import pandas as pd @@ -24,6 +25,13 @@ class clustering_job: self.outpath.mkdir(parents=True, exist_ok=True) self.cluster_data.to_feather(self.outpath/(self.name + ".feather")) self.hasrun = True + self.cleanup() + + def cleanup(self): + self.cluster_data = None + self.mat = None + self.clustering=None + self.subreddits=None def get_info(self): if not self.hasrun: @@ -57,6 +65,7 @@ class clustering_job: return score def read_distance_mat(self, similarities, use_threads=True): + print(similarities) df = pd.read_feather(similarities, use_threads=use_threads) mat = np.array(df.drop('_subreddit',1)) n = mat.shape[0] @@ -95,6 +104,38 @@ class clustering_job: return cluster_data +class twoway_clustering_job(clustering_job): + def __init__(self, infile, outpath, name, call1, call2, args1, args2): + self.outpath = Path(outpath) + self.call1 = call1 + self.args1 = args1 + self.call2 = call2 + self.args2 = args2 + self.infile = Path(infile) + self.name = name + self.hasrun = False + self.args = args1|args2 + + def run(self): + self.subreddits, self.mat = self.read_distance_mat(self.infile) + self.step1 = self.call1(self.mat, **self.args1) + self.clustering = self.call2(self.mat, self.step1, **self.args2) + self.cluster_data = self.process_clustering(self.clustering, self.subreddits) + self.hasrun = True + self.after_run() + self.cleanup() + + def after_run(): + self.score = self.silhouette() + self.outpath.mkdir(parents=True, exist_ok=True) + print(self.outpath/(self.name+".feather")) + self.cluster_data.to_feather(self.outpath/(self.name + ".feather")) + + + def cleanup(self): + super().cleanup() + self.step1 = None + @dataclass class clustering_result: outpath:Path diff --git a/clustering/grid_sweep.py b/clustering/grid_sweep.py index c0365d0..f021515 100644 --- a/clustering/grid_sweep.py +++ b/clustering/grid_sweep.py @@ -31,3 +31,19 @@ class grid_sweep: outcsv = Path(outcsv) outcsv.parent.mkdir(parents=True, exist_ok=True) self.infos.to_csv(outcsv) + + +class twoway_grid_sweep(grid_sweep): + def __init__(self, jobtype, inpath, outpath, namer, args1, args2, *args, **kwargs): + self.jobtype = jobtype + self.namer = namer + prod1 = product(* args1.values()) + prod2 = product(* args2.values()) + grid1 = [dict(zip(args1.keys(), pargs)) for pargs in prod1] + grid2 = [dict(zip(args2.keys(), pargs)) for pargs in prod2] + grid = product(grid1, grid2) + inpath = Path(inpath) + outpath = Path(outpath) + self.hasrun = False + self.grid = [(inpath,outpath,namer(**(g[0] | g[1])), g[0], g[1], *args) for g in grid] + self.jobs = [jobtype(*g) for g in self.grid] diff --git a/clustering/lsi_base.py b/clustering/lsi_base.py index 80b7101..14bbfc5 100644 --- a/clustering/lsi_base.py +++ b/clustering/lsi_base.py @@ -1,5 +1,5 @@ from clustering_base import clustering_job, clustering_result -from grid_sweep import grid_sweep +from grid_sweep import grid_sweep, twoway_grid_sweep from dataclasses import dataclass from itertools import chain from pathlib import Path @@ -27,3 +27,18 @@ class lsi_grid_sweep(grid_sweep): self.hasrun = False self.subgrids = [self.subsweep(lsi_path, outpath, lsi_dim, *args, **kwargs) for lsi_dim, lsi_path in zip(lsi_nums, lsi_paths)] self.jobs = list(chain(*map(lambda gs: gs.jobs, self.subgrids))) + +class twoway_lsi_grid_sweep(twoway_grid_sweep): + def __init__(self, jobtype, subsweep, inpath, lsi_dimensions, outpath, args1, args2, save_step1): + self.jobtype = jobtype + self.subsweep = subsweep + inpath = Path(inpath) + if lsi_dimensions == 'all': + lsi_paths = list(inpath.glob("*.feather")) + else: + lsi_paths = [inpath / (str(dim) + '.feather') for dim in lsi_dimensions] + + lsi_nums = [int(p.stem) for p in lsi_paths] + self.hasrun = False + self.subgrids = [self.subsweep(lsi_path, outpath, lsi_dim, args1, args2, save_step1) for lsi_dim, lsi_path in zip(lsi_nums, lsi_paths)] + self.jobs = list(chain(*map(lambda gs: gs.jobs, self.subgrids))) diff --git a/clustering/umap_hdbscan_clustering.py b/clustering/umap_hdbscan_clustering.py new file mode 100644 index 0000000..6a4d2a1 --- /dev/null +++ b/clustering/umap_hdbscan_clustering.py @@ -0,0 +1,221 @@ +from clustering_base import clustering_result, clustering_job, twoway_clustering_job +from hdbscan_clustering import hdbscan_clustering_result +import umap +from grid_sweep import twoway_grid_sweep +from dataclasses import dataclass +import hdbscan +from sklearn.neighbors import NearestNeighbors +import plotnine as pn +import numpy as np +from itertools import product, starmap, chain +import pandas as pd +from multiprocessing import cpu_count +import fire + +def test_select_hdbscan_clustering(): + # select_hdbscan_clustering("/gscratch/comdata/output/reddit_similarity/subreddit_comment_authors-tf_30k_LSI", + # "test_hdbscan_author30k", + # min_cluster_sizes=[2], + # min_samples=[1,2], + # cluster_selection_epsilons=[0,0.05,0.1,0.15], + # cluster_selection_methods=['eom','leaf'], + # lsi_dimensions='all') + inpath = "/gscratch/comdata/output/reddit_similarity/subreddit_comment_authors-tf_10k_LSI" + outpath = "test_umap_hdbscan_lsi" + min_cluster_sizes=[2,3,4] + min_samples=[1,2,3] + cluster_selection_epsilons=[0,0.1,0.3,0.5] + cluster_selection_methods=[1] + lsi_dimensions='all' + n_neighbors = [5,10,15,25,35,70,100] + learning_rate = [0.1,0.5,1,2] + min_dist = [0.5,1,1.5,2] + local_connectivity = [1,2,3,4,5] + + hdbscan_params = {"min_cluster_sizes":min_cluster_sizes, "min_samples":min_samples, "cluster_selection_epsilons":cluster_selection_epsilons, "cluster_selection_methods":cluster_selection_methods} + umap_params = {"n_neighbors":n_neighbors, "learning_rate":learning_rate, "min_dist":min_dist, "local_connectivity":local_connectivity} + gs = umap_hdbscan_grid_sweep(inpath, "all", outpath, hdbscan_params,umap_params) + + # gs.run(20) + # gs.save("test_hdbscan/lsi_sweep.csv") + + + # job1 = hdbscan_lsi_job(infile=inpath, outpath=outpath, name="test", lsi_dims=500, min_cluster_size=2, min_samples=1,cluster_selection_epsilon=0,cluster_selection_method='eom') + # job1.run() + # print(job1.get_info()) + + # df = pd.read_csv("test_hdbscan/selection_data.csv") + # test_select_hdbscan_clustering() + # check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather") + # silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather") + # c = check_clusters.merge(silscores,on='subreddit')# fire.Fire(select_hdbscan_clustering) +class umap_hdbscan_grid_sweep(twoway_grid_sweep): + def __init__(self, + inpath, + outpath, + umap_params, + hdbscan_params): + + super().__init__(umap_hdbscan_job, inpath, outpath, self.namer, umap_params, hdbscan_params) + + def namer(self, + min_cluster_size, + min_samples, + cluster_selection_epsilon, + cluster_selection_method, + n_neighbors, + learning_rate, + min_dist, + local_connectivity + ): + return f"mcs-{min_cluster_size}_ms-{min_samples}_cse-{cluster_selection_epsilon}_csm-{cluster_selection_method}_nn-{n_neighbors}_lr-{learning_rate}_md-{min_dist}_lc-{local_connectivity}" + +@dataclass +class umap_hdbscan_clustering_result(hdbscan_clustering_result): + n_neighbors:int + learning_rate:float + min_dist:float + local_connectivity:int + +class umap_hdbscan_job(twoway_clustering_job): + def __init__(self, infile, outpath, name, + umap_args = {"n_neighbors":15, "learning_rate":1, "min_dist":1, "local_connectivity":1}, + hdbscan_args = {"min_cluster_size":2, "min_samples":1, "cluster_selection_epsilon":0, "cluster_selection_method":'eom'}, + save_step1 = False, + *args, + **kwargs): + super().__init__(infile, + outpath, + name, + call1=umap_hdbscan_job._umap_embedding, + call2=umap_hdbscan_job._hdbscan_clustering, + args1=umap_args, + args2=hdbscan_args, + save_step1=save_step1, + *args, + **kwargs + ) + + self.n_neighbors = umap_args['n_neighbors'] + self.learning_rate = umap_args['learning_rate'] + self.min_dist = umap_args['min_dist'] + self.local_connectivity = umap_args['local_connectivity'] + self.min_cluster_size = hdbscan_args['min_cluster_size'] + self.min_samples = hdbscan_args['min_samples'] + self.cluster_selection_epsilon = hdbscan_args['cluster_selection_epsilon'] + self.cluster_selection_method = hdbscan_args['cluster_selection_method'] + + def after_run(self): + coords = self.step1.emedding_ + self.cluster_data['x'] = coords[:,0] + self.cluster_data['y'] = coords[:,1] + super().after_run() + + + def _umap_embedding(mat, **umap_args): + print(f"running umap embedding. umap_args:{umap_args}") + umapmodel = umap.UMAP(metric='precomputed', **umap_args) + umapmodel = umapmodel.fit(mat) + return umapmodel + + def _hdbscan_clustering(mat, umapmodel, **hdbscan_args): + print(f"running hdbascan clustering. hdbscan_args:{hdbscan_args}") + + umap_coords = umapmodel.transform(mat) + + clusterer = hdbscan.HDBSCAN(metric='euclidean', + core_dist_n_jobs=cpu_count(), + **hdbscan_args + ) + + clustering = clusterer.fit(umap_coords) + + return(clustering) + + def get_info(self): + result = super().get_info() + self.result = umap_hdbscan_clustering_result(**result.__dict__, + min_cluster_size=self.min_cluster_size, + min_samples=self.min_samples, + cluster_selection_epsilon=self.cluster_selection_epsilon, + cluster_selection_method=self.cluster_selection_method, + n_neighbors = self.n_neighbors, + learning_rate = self.learning_rate, + min_dist = self.min_dist, + local_connectivity=self.local_connectivity + ) + return self.result + +def run_umap_hdbscan_grid_sweep(savefile, inpath, outpath, n_neighbors = [15], learning_rate=[1], min_dist=[1], local_connectivity=[1], + min_cluster_sizes=[2], min_samples=[1], cluster_selection_epsilons=[0], cluster_selection_methods=['eom']): + """Run umap + hdbscan clustering once or more with different parameters. + + Usage: + umap_hdbscan_clustering.py --savefile=SAVEFILE --inpath=INPATH --outpath=OUTPATH --n_neighbors= --learning_rate= --min_dist= --local_connectivity= --min_cluster_sizes= --min_samples= --cluster_selection_epsilons= --cluster_selection_methods= + + Keword arguments: + savefile: path to save the metadata and diagnostics + inpath: path to feather data containing a labeled matrix of subreddit similarities. + outpath: path to output fit kmeans clusterings. + n_neighbors: umap parameter takes integers greater than 1 + learning_rate: umap parameter takes positive real values + min_dist: umap parameter takes positive real values + local_connectivity: umap parameter takes positive integers + min_cluster_sizes: one or more integers indicating the minumum cluster size + min_samples: one ore more integers indicating the minimum number of samples used in the algorithm + cluster_selection_epsilon: one or more similarity thresholds for transition from dbscan to hdbscan + cluster_selection_method: "eom" or "leaf" eom gives larger clusters. + """ + + umap_args = {'n_neighbors':list(map(int, n_neighbors)), + 'learning_rate':list(map(float,learning_rate)), + 'min_dist':list(map(float,min_dist)), + 'local_connectivity':list(map(int,local_connectivity)), + } + + hdbscan_args = {'min_cluster_size':list(map(int,min_cluster_sizes)), + 'min_samples':list(map(int,min_samples)), + 'cluster_selection_epsilon':list(map(float,cluster_selection_epsilons)), + 'cluster_selection_method':cluster_selection_methods} + + obj = umap_hdbscan_grid_sweep(inpath, + outpath, + umap_args, + hdbscan_args) + obj.run(cores=10) + obj.save(savefile) + + +def KNN_distances_plot(mat,outname,k=2): + nbrs = NearestNeighbors(n_neighbors=k,algorithm='auto',metric='precomputed').fit(mat) + distances, indices = nbrs.kneighbors(mat) + d2 = distances[:,-1] + df = pd.DataFrame({'dist':d2}) + df = df.sort_values("dist",ascending=False) + df['idx'] = np.arange(0,d2.shape[0]) + 1 + p = pn.qplot(x='idx',y='dist',data=df,geom='line') + pn.scales.scale_y_continuous(minor_breaks = np.arange(0,50)/50, + breaks = np.arange(0,10)/10) + p.save(outname,width=16,height=10) + +def make_KNN_plots(): + similarities = "/gscratch/comdata/output/reddit_similarity/subreddit_comment_terms_10k.feather" + subreddits, mat = read_similarity_mat(similarities) + mat = sim_to_dist(mat) + + KNN_distances_plot(mat,k=2,outname='terms_knn_dist2.png') + + similarities = "/gscratch/comdata/output/reddit_similarity/subreddit_comment_authors_10k.feather" + subreddits, mat = read_similarity_mat(similarities) + mat = sim_to_dist(mat) + KNN_distances_plot(mat,k=2,outname='authors_knn_dist2.png') + + similarities = "/gscratch/comdata/output/reddit_similarity/subreddit_comment_authors-tf_10k.feather" + subreddits, mat = read_similarity_mat(similarities) + mat = sim_to_dist(mat) + KNN_distances_plot(mat,k=2,outname='authors-tf_knn_dist2.png') + +if __name__ == "__main__": + fire.Fire(run_umap_hdbscan_grid_sweep) + +# test_select_hdbscan_clustering() + #fire.Fire(select_hdbscan_clustering) diff --git a/clustering/umap_hdbscan_clustering_lsi.py b/clustering/umap_hdbscan_clustering_lsi.py new file mode 100644 index 0000000..09b3630 --- /dev/null +++ b/clustering/umap_hdbscan_clustering_lsi.py @@ -0,0 +1,114 @@ +from umap_hdbscan_clustering import umap_hdbscan_job, umap_hdbscan_grid_sweep, umap_hdbscan_clustering_result +from lsi_base import twoway_lsi_grid_sweep, lsi_mixin, lsi_result_mixin +from grid_sweep import twoway_grid_sweep +import fire +from dataclasses import dataclass + +@dataclass +class umap_hdbscan_clustering_result_lsi(umap_hdbscan_clustering_result, lsi_result_mixin): + pass + +class umap_hdbscan_lsi_job(umap_hdbscan_job, lsi_mixin): + def __init__(self, infile, outpath, name, umap_args, hdbscan_args, lsi_dims, save_step1=False): + super().__init__( + infile, + outpath, + name, + umap_args, + hdbscan_args, + save_step1 + ) + super().set_lsi_dims(lsi_dims) + + def get_info(self): + partial_result = super().get_info() + self.result = umap_hdbscan_clustering_result_lsi(**partial_result.__dict__, + lsi_dimensions=self.lsi_dims) + return self.result + +class umap_hdbscan_lsi_grid_sweep(twoway_lsi_grid_sweep): + def __init__(self, + inpath, + lsi_dims, + outpath, + umap_args, + hdbscan_args, + save_step1 + ): + + super().__init__(umap_hdbscan_lsi_job, + _umap_hdbscan_lsi_grid_sweep, + inpath, + lsi_dims, + outpath, + umap_args, + hdbscan_args, + save_step1 + ) + + + +class _umap_hdbscan_lsi_grid_sweep(twoway_grid_sweep): + def __init__(self, + inpath, + outpath, + lsi_dim, + umap_args, + hdbscan_args, + save_step1): + + self.lsi_dim = lsi_dim + self.jobtype = umap_hdbscan_lsi_job + super().__init__(self.jobtype, inpath, outpath, self.namer, umap_args, hdbscan_args, save_step1, lsi_dim) + + + def namer(self, *args, **kwargs): + s = umap_hdbscan_grid_sweep.namer(self, *args, **kwargs) + s += f"_lsi-{self.lsi_dim}" + return s + +def run_umap_hdbscan_lsi_grid_sweep(savefile, inpath, outpath, n_neighbors = [15], learning_rate=[1], min_dist=[1], local_connectivity=[1], + min_cluster_sizes=[2], min_samples=[1], cluster_selection_epsilons=[0], cluster_selection_methods=['eom'], lsi_dimensions='all', save_step1 = False): + """Run hdbscan clustering once or more with different parameters. + + Usage: + hdbscan_clustering_lsi --savefile=SAVEFILE --inpath=INPATH --outpath=OUTPATH --min_cluster_sizes= --min_samples= --cluster_selection_epsilons= --cluster_selection_methods=[eom]> --lsi_dimensions: either "all" or one or more available lsi similarity dimensions at INPATH. + + Keword arguments: + savefile: path to save the metadata and diagnostics + inpath: path to folder containing feather files with LSI similarity labeled matrices of subreddit similarities. + outpath: path to output fit clusterings. + min_cluster_sizes: one or more integers indicating the minumum cluster size + min_samples: one ore more integers indicating the minimum number of samples used in the algorithm + cluster_selection_epsilons: one or more similarity thresholds for transition from dbscan to hdbscan + cluster_selection_methods: one or more of "eom" or "leaf" eom gives larger clusters. + lsi_dimensions: either "all" or one or more available lsi similarity dimensions at INPATH. + """ + + + umap_args = {'n_neighbors':list(map(int, n_neighbors)), + 'learning_rate':list(map(float,learning_rate)), + 'min_dist':list(map(float,min_dist)), + 'local_connectivity':list(map(int,local_connectivity)), + } + + hdbscan_args = {'min_cluster_size':list(map(int,min_cluster_sizes)), + 'min_samples':list(map(int,min_samples)), + 'cluster_selection_epsilon':list(map(float,cluster_selection_epsilons)), + 'cluster_selection_method':cluster_selection_methods} + + obj = umap_hdbscan_lsi_grid_sweep(inpath, + lsi_dimensions, + outpath, + umap_args, + hdbscan_args, + save_step1 + ) + + + obj.run(10) + obj.save(savefile) + + +if __name__ == "__main__": + fire.Fire(run_umap_hdbscan_lsi_grid_sweep)