grid sweep selection for clustering hyperparameters
This commit is contained in:
Nate E TeBlunthuis
parent
628a70734b
commit
01a4c35358
@ -1,32 +1,52 @@
|
||||
#srun_cdsc='srun -p comdata-int -A comdata --time=300:00:00 --time-min=00:15:00 --mem=100G --ntasks=1 --cpus-per-task=28'
|
||||
all:/gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather /gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather /gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather
|
||||
#all:/gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather /gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather /gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_clustering/wang_similarity_10000.feather /gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather /gscratch/comdata/output/reddit_tsne/wang_similarity_10000.feather /gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather
|
||||
srun_singularity=source /gscratch/comdata/users/nathante/cdsc_reddit/bin/activate && srun_singularity.sh
|
||||
similarity_data=/gscratch/comdata/output/reddit_similarity
|
||||
clustering_data=/gscratch/comdata/output/reddit_clustering
|
||||
selection_grid="--max_iter=10000 --convergence_iter=15,30,100 --preference_quantile=0.85 --damping=0.5,0.6,0.7,0.8,0.85,0.9,0.95,0.97,0.99, --preference_quantile=0.1,0.3,0.5,0.7,0.9"
|
||||
all:$(clustering_data)/subreddit_comment_authors_30k.feather $(clustering_data)/subreddit_authors-tf_similarities_30k.feather $(clustering_data)/subreddit_comment_authors_10k.feather $(clustering_data)/subreddit_authors-tf_similarities_10k.feather $(clustering_data)/subreddit_comment_terms_30k.feather $(clustering_data)/subreddit_comment_terms_10k.feather
|
||||
|
||||
/gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather
|
||||
# $srun_cdsc python3
|
||||
start_spark_and_run.sh 1 clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather /gscratch/comdata/output/reddit_clustering/comment_authors_10000.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.85 --damping=0.85
|
||||
$(clustering_data)/subreddit_comment_authors_10k.feather:selection.py $(similarity_data)/subreddit_comment_authors_10k.feather clustering.py
|
||||
$(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors_10k.feather $(clustering_data)/subreddit_comment_authors_10k $(selection_grid) -J 20
|
||||
|
||||
/gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_terms_10000.feather
|
||||
# $srun_cdsc python3
|
||||
start_spark_and_run.sh 1 clustering.py /gscratch/comdata/output/reddit_similarity/comment_terms_10000.feather /gscratch/comdata/output/reddit_clustering/comment_terms_10000.feather ---max_iter=1000 --convergence_iter=15 --preference_quantile=0.9 --damping=0.5
|
||||
$(clustering_data)/subreddit_comment_terms_10k.feather:selection.py $(similarity_data)/subreddit_comment_terms_10k.feather clustering.py
|
||||
$(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_terms_10k.feather $(clustering_data)/subreddit_comment_terms_10k $(selection_grid) -J 20
|
||||
|
||||
$(clustering_data)/subreddit_authors-tf_similarities_10k.feather:clustering.py $(similarity_data)/subreddit_comment_authors-tf_10k.feather
|
||||
$(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors-tf_10k.feather $(clustering_data)/subreddit_comment_authors-tf_10k $(selection_grid) -J 20
|
||||
|
||||
$(clustering_data)/subreddit_comment_authors_30k.feather:selection.py $(similarity_data)/subreddit_comment_authors_30k.feather clustering.py
|
||||
$(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors_30k.feather $(clustering_data)/subreddit_comment_authors_30k $(selection_grid) -J 10
|
||||
|
||||
$(clustering_data)/subreddit_comment_terms_30k.feather:selection.py $(similarity_data)/subreddit_comment_terms_30k.feather clustering.py
|
||||
$(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_terms_30k.feather $(clustering_data)/subreddit_comment_terms_30k $(selection_grid) -J 10
|
||||
|
||||
$(clustering_data)/subreddit_authors-tf_similarities_30k.feather:clustering.py $(similarity_data)/subreddit_comment_authors-tf_30k.feather
|
||||
$(srun_singularity) python3 selection.py $(similarity_data)/subreddit_comment_authors-tf_30k.feather $(clustering_data)/subreddit_comment_authors-tf_30k $(selection_grid) -J 8
|
||||
|
||||
|
||||
# $(clustering_data)/subreddit_comment_authors_100k.feather:clustering.py $(similarity_data)/subreddit_comment_authors_100k.feather
|
||||
# $(srun_singularity) python3 clustering.py $(similarity_data)/subreddit_comment_authors_100k.feather $(clustering_data)/subreddit_comment_authors_100k.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.85 --damping=0.85
|
||||
|
||||
# $(clustering_data)/comment_terms_100k.feather:clustering.py $(similarity_data)/subreddit_comment_terms_100k.feather
|
||||
# $(srun_singularity) python3 clustering.py $(similarity_data)/comment_terms_10000.feather $(clustering_data)/comment_terms_10000.feather ---max_iter=1000 --convergence_iter=15 --preference_quantile=0.9 --damping=0.5
|
||||
|
||||
# $(clustering_data)/subreddit_comment_author-tf_100k.feather:clustering.py $(similarity_data)/subreddit_comment_author-tf_100k.feather
|
||||
# $(srun_singularity) python3 clustering.py $(similarity_data)/subreddit_comment_author-tf_100k.parquet $(clustering_data)/subreddit_comment_author-tf_100k.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.5 --damping=0.85
|
||||
|
||||
/gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet
|
||||
# $srun_cdsc
|
||||
start_spark_and_run.sh 1 clustering.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet /gscratch/comdata/output/reddit_clustering/subreddit_author_tf_similarities_10000.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.5 --damping=0.85
|
||||
|
||||
# it's pretty difficult to get a result that isn't one huge megacluster. A sign that it's bullcrap
|
||||
# /gscratch/comdata/output/reddit_clustering/wang_similarity_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather
|
||||
# ./clustering.py /gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather /gscratch/comdata/output/reddit_clustering/wang_similarity_10000.feather ---max_iter=400 --convergence_iter=15 --preference_quantile=0.9 --damping=0.85
|
||||
|
||||
/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather:fit_tsne.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet
|
||||
# /gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather:fit_tsne.py /gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet
|
||||
|
||||
start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet --output=/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather
|
||||
# start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/subreddit_author_tf_similarities_10000.parquet --output=/gscratch/comdata/output/reddit_tsne/subreddit_author_tf_similarities_10000.feather
|
||||
|
||||
|
||||
# /gscratch/comdata/output/reddit_tsne/wang_similarity_10000.feather:fit_tsne.py /gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather
|
||||
|
||||
# python3 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/wang_similarity_10000.feather --output=/gscratch/comdata/output/reddit_tsne/wang_similarity_10000.feather
|
||||
|
||||
/gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather
|
||||
# $srun_cdsc python3
|
||||
start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather --output=/gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather
|
||||
# /gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather:clustering.py /gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather
|
||||
# # $srun_cdsc python3
|
||||
# start_spark_and_run.sh 1 fit_tsne.py --similarities=/gscratch/comdata/output/reddit_similarity/comment_authors_10000.feather --output=/gscratch/comdata/output/reddit_tsne/comment_authors_10000.feather
|
||||
|
||||
@ -1,29 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# TODO: replace prints with logging.
|
||||
import sys
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from sklearn.cluster import AffinityPropagation
|
||||
import fire
|
||||
from pathlib import Path
|
||||
|
||||
def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True):
|
||||
def read_similarity_mat(similarities, use_threads=True):
|
||||
df = pd.read_feather(similarities, use_threads=use_threads)
|
||||
mat = np.array(df.drop('_subreddit',1))
|
||||
n = mat.shape[0]
|
||||
mat[range(n),range(n)] = 1
|
||||
return (df._subreddit,mat)
|
||||
|
||||
def affinity_clustering(similarities, *args, **kwargs):
|
||||
subreddits, mat = read_similarity_mat(similarities)
|
||||
return _affinity_clustering(mat, subreddits, *args, **kwargs)
|
||||
|
||||
def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True):
|
||||
'''
|
||||
similarities: feather file with a dataframe of similarity scores
|
||||
preference_quantile: parameter controlling how many clusters to make. higher values = more clusters. 0.85 is a good value with 3000 subreddits.
|
||||
damping: parameter controlling how iterations are merged. Higher values make convergence faster and more dependable. 0.85 is a good value for the 10000 subreddits by author.
|
||||
'''
|
||||
|
||||
df = pd.read_feather(similarities)
|
||||
n = df.shape[0]
|
||||
mat = np.array(df.drop('_subreddit',1))
|
||||
mat[range(n),range(n)] = 1
|
||||
assert(all(np.diag(mat)==1))
|
||||
print(f"damping:{damping}; convergenceIter:{convergence_iter}; preferenceQuantile:{preference_quantilne}")
|
||||
|
||||
preference = np.quantile(mat,preference_quantile)
|
||||
|
||||
print(f"preference is {preference}")
|
||||
|
||||
print("data loaded")
|
||||
|
||||
sys.stdout.flush()
|
||||
clustering = AffinityPropagation(damping=damping,
|
||||
max_iter=max_iter,
|
||||
convergence_iter=convergence_iter,
|
||||
@ -39,7 +46,7 @@ def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, conv
|
||||
|
||||
print(f"found {len(set(clusters))} clusters")
|
||||
|
||||
cluster_data = pd.DataFrame({'subreddit': df.subreddit,'cluster':clustering.labels_})
|
||||
cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_})
|
||||
|
||||
cluster_sizes = cluster_data.groupby("cluster").count()
|
||||
print(f"the largest cluster has {cluster_sizes.subreddit.max()} members")
|
||||
@ -48,7 +55,10 @@ def affinity_clustering(similarities, output, damping=0.9, max_iter=100000, conv
|
||||
|
||||
print(f"{(cluster_sizes.subreddit==1).sum()} clusters have 1 member")
|
||||
|
||||
sys.stdout.flush()
|
||||
cluster_data.to_feather(output)
|
||||
print(f"saved {output}")
|
||||
return clustering
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(affinity_clustering)
|
||||
|
||||
87
clustering/selection.py
Normal file
87
clustering/selection.py
Normal file
@ -0,0 +1,87 @@
|
||||
from sklearn.metrics import silhouette_score
|
||||
from sklearn.cluster import AffinityPropagation
|
||||
from functools import partial
|
||||
from clustering import _affinity_clustering, read_similarity_mat
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing import Pool, cpu_count, Array, Process
|
||||
from pathlib import Path
|
||||
from itertools import product, starmap
|
||||
import pandas as pd
|
||||
import fire
|
||||
import sys
|
||||
|
||||
# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying.
|
||||
|
||||
@dataclass
|
||||
class clustering_result:
|
||||
outpath:Path
|
||||
damping:float
|
||||
max_iter:int
|
||||
convergence_iter:int
|
||||
preference_quantile:float
|
||||
silhouette_score:float
|
||||
alt_silhouette_score:float
|
||||
name:str
|
||||
|
||||
def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat):
|
||||
if name is None:
|
||||
name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{convergence_iter}"
|
||||
print(name)
|
||||
sys.stdout.flush()
|
||||
outpath = outdir / (str(name) + ".feather")
|
||||
print(outpath)
|
||||
clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
|
||||
score = silhouette_score(clustering.affinity_matrix_, clustering.labels_, metric='precomputed')
|
||||
alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
|
||||
|
||||
res = clustering_result(outpath=outpath,
|
||||
damping=damping,
|
||||
max_iter=max_iter,
|
||||
convergence_iter=convergence_iter,
|
||||
preference_quantile=preference_quantile,
|
||||
silhouette_score=score,
|
||||
alt_silhouette_score=score,
|
||||
name=str(name))
|
||||
|
||||
return res
|
||||
|
||||
# alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
|
||||
|
||||
def select_affinity_clustering(similarities, outdir, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
|
||||
|
||||
damping = list(map(float,damping))
|
||||
convergence_iter = convergence_iter = list(map(int,convergence_iter))
|
||||
preference_quantile = list(map(float,preference_quantile))
|
||||
|
||||
if type(outdir) is str:
|
||||
outdir = Path(outdir)
|
||||
|
||||
outdir.mkdir(parents=True,exist_ok=True)
|
||||
|
||||
subreddits, mat = read_similarity_mat(similarities,use_threads=True)
|
||||
|
||||
if alt_similarities is not None:
|
||||
alt_mat = read_similarity_mat(alt_similarities,use_threads=True)
|
||||
else:
|
||||
alt_mat = None
|
||||
|
||||
if J is None:
|
||||
J = cpu_count()
|
||||
pool = Pool(J)
|
||||
|
||||
# get list of tuples: the combinations of hyperparameters
|
||||
hyper_grid = product(damping, convergence_iter, preference_quantile)
|
||||
hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
|
||||
|
||||
_do_clustering = partial(do_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
|
||||
|
||||
# similarities = Array('d', mat)
|
||||
# call pool.starmap
|
||||
print("running clustering selection")
|
||||
clustering_data = pool.starmap(_do_clustering, hyper_grid)
|
||||
clustering_data = pd.DataFrame(list(clustering_data))
|
||||
return clustering_data
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fire.Fire(select_affinity_clustering)
|
||||
Loading…
Reference in New Issue
Block a user