refactor clustering.py into method-specific files.
This commit is contained in:
parent
e1c9d9af6f
commit
8d1df5b26e
@ -18,7 +18,44 @@ class affinity_clustering_result(clustering_result):
|
|||||||
convergence_iter:int
|
convergence_iter:int
|
||||||
preference_quantile:float
|
preference_quantile:float
|
||||||
|
|
||||||
def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
|
def affinity_clustering(similarities, output, *args, **kwargs):
|
||||||
|
subreddits, mat = read_similarity_mat(similarities)
|
||||||
|
clustering = _affinity_clustering(mat, *args, **kwargs)
|
||||||
|
cluster_data = process_clustering_result(clustering, subreddits)
|
||||||
|
cluster_data['algorithm'] = 'affinity'
|
||||||
|
return(cluster_data)
|
||||||
|
|
||||||
|
def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000, convergence_iter=30, preference_quantile=0.5, random_state=1968, verbose=True):
|
||||||
|
'''
|
||||||
|
similarities: matrix of similarity scores
|
||||||
|
preference_quantile: parameter controlling how many clusters to make. higher values = more clusters. 0.85 is a good value with 3000 subreddits.
|
||||||
|
damping: parameter controlling how iterations are merged. Higher values make convergence faster and more dependable. 0.85 is a good value for the 10000 subreddits by author.
|
||||||
|
'''
|
||||||
|
print(f"damping:{damping}; convergenceIter:{convergence_iter}; preferenceQuantile:{preference_quantile}")
|
||||||
|
|
||||||
|
preference = np.quantile(mat,preference_quantile)
|
||||||
|
|
||||||
|
print(f"preference is {preference}")
|
||||||
|
print("data loaded")
|
||||||
|
sys.stdout.flush()
|
||||||
|
clustering = AffinityPropagation(damping=damping,
|
||||||
|
max_iter=max_iter,
|
||||||
|
convergence_iter=convergence_iter,
|
||||||
|
copy=False,
|
||||||
|
preference=preference,
|
||||||
|
affinity='precomputed',
|
||||||
|
verbose=verbose,
|
||||||
|
random_state=random_state).fit(mat)
|
||||||
|
|
||||||
|
cluster_data = process_clustering_result(clustering, subreddits)
|
||||||
|
output = Path(output)
|
||||||
|
output.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
cluster_data.to_feather(output)
|
||||||
|
print(f"saved {output}")
|
||||||
|
return clustering
|
||||||
|
|
||||||
|
|
||||||
|
def do_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
|
||||||
if name is None:
|
if name is None:
|
||||||
name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
|
name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
|
||||||
print(name)
|
print(name)
|
||||||
@ -53,41 +90,6 @@ def do_affinity_clustering(damping, convergence_iter, preference_quantile, name,
|
|||||||
|
|
||||||
return res
|
return res
|
||||||
|
|
||||||
def do_affinity_clustering(damping, convergence_iter, preference_quantile, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
|
|
||||||
if name is None:
|
|
||||||
name = f"damping-{damping}_convergenceIter-{convergence_iter}_preferenceQuantile-{preference_quantile}"
|
|
||||||
print(name)
|
|
||||||
sys.stdout.flush()
|
|
||||||
outpath = outdir / (str(name) + ".feather")
|
|
||||||
outpath.parent.mkdir(parents=True,exist_ok=True)
|
|
||||||
print(outpath)
|
|
||||||
clustering = _affinity_clustering(mat, subreddits, outpath, damping, max_iter, convergence_iter, preference_quantile, random_state, verbose)
|
|
||||||
mat = sim_to_dist(clustering.affinity_matrix_)
|
|
||||||
|
|
||||||
try:
|
|
||||||
score = silhouette_score(mat, clustering.labels_, metric='precomputed')
|
|
||||||
except ValueError:
|
|
||||||
score = None
|
|
||||||
|
|
||||||
if alt_mat is not None:
|
|
||||||
alt_distances = sim_to_dist(alt_mat)
|
|
||||||
try:
|
|
||||||
alt_score = silhouette_score(alt_mat, clustering.labels_, metric='precomputed')
|
|
||||||
except ValueError:
|
|
||||||
alt_score = None
|
|
||||||
|
|
||||||
res = clustering_result(outpath=outpath,
|
|
||||||
damping=damping,
|
|
||||||
max_iter=max_iter,
|
|
||||||
convergence_iter=convergence_iter,
|
|
||||||
preference_quantile=preference_quantile,
|
|
||||||
silhouette_score=score,
|
|
||||||
alt_silhouette_score=score,
|
|
||||||
name=str(name))
|
|
||||||
|
|
||||||
return res
|
|
||||||
|
|
||||||
|
|
||||||
# alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
|
# alt similiarities is for checking the silhouette coefficient of an alternative measure of similarity (e.g., topic similarities for user clustering).
|
||||||
|
|
||||||
def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
|
def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max_iter=100000, convergence_iter=[30], preference_quantile=[0.5], random_state=1968, verbose=True, alt_similarities=None, J=None):
|
||||||
@ -116,7 +118,7 @@ def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max
|
|||||||
hyper_grid = product(damping, convergence_iter, preference_quantile)
|
hyper_grid = product(damping, convergence_iter, preference_quantile)
|
||||||
hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
|
hyper_grid = (t + (str(i),) for i, t in enumerate(hyper_grid))
|
||||||
|
|
||||||
_do_clustering = partial(do_affinity_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
|
_do_clustering = partial(do_clustering, mat=mat, subreddits=subreddits, outdir=outdir, max_iter=max_iter, random_state=random_state, verbose=verbose, alt_mat=alt_mat)
|
||||||
|
|
||||||
# similarities = Array('d', mat)
|
# similarities = Array('d', mat)
|
||||||
# call pool.starmap
|
# call pool.starmap
|
||||||
@ -124,8 +126,6 @@ def select_affinity_clustering(similarities, outdir, outinfo, damping=[0.9], max
|
|||||||
clustering_data = pool.starmap(_do_clustering, hyper_grid)
|
clustering_data = pool.starmap(_do_clustering, hyper_grid)
|
||||||
clustering_data = pd.DataFrame(list(clustering_data))
|
clustering_data = pd.DataFrame(list(clustering_data))
|
||||||
clustering_data.to_csv(outinfo)
|
clustering_data.to_csv(outinfo)
|
||||||
|
|
||||||
|
|
||||||
return clustering_data
|
return clustering_data
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
@ -3,7 +3,7 @@
|
|||||||
import sys
|
import sys
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from sklearn.cluster import AffinityPropagation, KMeans
|
from sklearn.cluster import AffinityPropagation
|
||||||
import fire
|
import fire
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from multiprocessing import cpu_count
|
from multiprocessing import cpu_count
|
||||||
@ -46,24 +46,6 @@ def _affinity_clustering(mat, subreddits, output, damping=0.9, max_iter=100000,
|
|||||||
print(f"saved {output}")
|
print(f"saved {output}")
|
||||||
return clustering
|
return clustering
|
||||||
|
|
||||||
def kmeans_clustering(similarities, *args, **kwargs):
|
|
||||||
subreddits, mat = read_similarity_mat(similarities)
|
|
||||||
mat = sim_to_dist(mat)
|
|
||||||
clustering = _kmeans_clustering(mat, *args, **kwargs)
|
|
||||||
cluster_data = process_clustering_result(clustering, subreddits)
|
|
||||||
return(cluster_data)
|
|
||||||
|
|
||||||
def _kmeans_clustering(mat, output, n_clusters, n_init=10, max_iter=100000, random_state=1968, verbose=True):
|
|
||||||
|
|
||||||
clustering = KMeans(n_clusters=n_clusters,
|
|
||||||
n_init=n_init,
|
|
||||||
max_iter=max_iter,
|
|
||||||
random_state=random_state,
|
|
||||||
verbose=verbose
|
|
||||||
).fit(mat)
|
|
||||||
|
|
||||||
return clustering
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
@ -28,6 +28,13 @@ def test_select_hdbscan_clustering():
|
|||||||
cluster_selection_methods=['eom'];
|
cluster_selection_methods=['eom'];
|
||||||
lsi_dimensions='all'
|
lsi_dimensions='all'
|
||||||
|
|
||||||
|
df = pd.read_csv("test_hdbscan/selection_data.csv")
|
||||||
|
test_select_hdbscan_clustering()
|
||||||
|
check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather")
|
||||||
|
silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather")
|
||||||
|
c = check_clusters.merge(silscores,on='subreddit')# fire.Fire(select_hdbscan_clustering)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class hdbscan_clustering_result(clustering_result):
|
class hdbscan_clustering_result(clustering_result):
|
||||||
min_cluster_size:int
|
min_cluster_size:int
|
||||||
@ -165,8 +172,4 @@ def make_KNN_plots():
|
|||||||
KNN_distances_plot(mat,k=2,outname='authors-tf_knn_dist2.png')
|
KNN_distances_plot(mat,k=2,outname='authors-tf_knn_dist2.png')
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
df = pd.read_csv("test_hdbscan/selection_data.csv")
|
fire.Fire(select_hdbscan_clustering)
|
||||||
test_select_hdbscan_clustering()
|
|
||||||
check_clusters = pd.read_feather("test_hdbscan/500_2_2_0.1_eom.feather")
|
|
||||||
silscores = pd.read_feather("test_hdbscan/silhouette_samples500_2_2_0.1_eom.feather")
|
|
||||||
c = check_clusters.merge(silscores,on='subreddit')# fire.Fire(select_hdbscan_clustering)
|
|
||||||
|
@ -1,23 +1,32 @@
|
|||||||
from sklearn.metrics import silhouette_score
|
from sklearn.cluster import KMeans
|
||||||
from sklearn.cluster import AffinityPropagation
|
|
||||||
from functools import partial
|
|
||||||
from clustering import _kmeans_clustering, read_similarity_mat, sim_to_dist, process_clustering_result, clustering_result
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from multiprocessing import Pool, cpu_count, Array, Process
|
|
||||||
from pathlib import Path
|
|
||||||
from itertools import product, starmap
|
|
||||||
import numpy as np
|
|
||||||
import pandas as pd
|
|
||||||
import fire
|
import fire
|
||||||
import sys
|
from pathlib import Path
|
||||||
|
from multiprocessing import cpu_count
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from clustering_base import sim_to_dist, process_clustering_result, clustering_result, read_similarity_mat
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class kmeans_clustering_result(clustering_result):
|
class kmeans_clustering_result(clustering_result):
|
||||||
n_clusters:int
|
n_clusters:int
|
||||||
n_init:int
|
n_init:int
|
||||||
|
|
||||||
|
def kmeans_clustering(similarities, *args, **kwargs):
|
||||||
|
subreddits, mat = read_similarity_mat(similarities)
|
||||||
|
mat = sim_to_dist(mat)
|
||||||
|
clustering = _kmeans_clustering(mat, *args, **kwargs)
|
||||||
|
cluster_data = process_clustering_result(clustering, subreddits)
|
||||||
|
return(cluster_data)
|
||||||
|
|
||||||
# silhouette is the only one that doesn't need the feature matrix. So it's probably the only one that's worth trying.
|
def _kmeans_clustering(mat, output, n_clusters, n_init=10, max_iter=100000, random_state=1968, verbose=True):
|
||||||
|
|
||||||
|
clustering = KMeans(n_clusters=n_clusters,
|
||||||
|
n_init=n_init,
|
||||||
|
max_iter=max_iter,
|
||||||
|
random_state=random_state,
|
||||||
|
verbose=verbose
|
||||||
|
).fit(mat)
|
||||||
|
|
||||||
|
return clustering
|
||||||
|
|
||||||
def do_clustering(n_clusters, n_init, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
|
def do_clustering(n_clusters, n_init, name, mat, subreddits, max_iter, outdir:Path, random_state, verbose, alt_mat, overwrite=False):
|
||||||
if name is None:
|
if name is None:
|
Loading…
Reference in New Issue
Block a user