50 lines
1.4 KiB
Python
50 lines
1.4 KiB
Python
|
from pathlib import Path
|
||
|
|
import numpy as np
|
||
|
|
import pandas as pd
|
||
|
|
from dataclasses import dataclass
|
||
|
|
|
||
|
|
def sim_to_dist(mat):
|
||
|
|
dist = 1-mat
|
||
|
|
dist[dist < 0] = 0
|
||
|
|
np.fill_diagonal(dist,0)
|
||
|
|
return dist
|
||
|
|
|
||
|
|
def process_clustering_result(clustering, subreddits):
|
||
|
|
|
||
|
|
if hasattr(clustering,'n_iter_'):
|
||
|
|
print(f"clustering took {clustering.n_iter_} iterations")
|
||
|
|
|
||
|
|
clusters = clustering.labels_
|
||
|
|
|
||
|
|
print(f"found {len(set(clusters))} clusters")
|
||
|
|
|
||
|
|
cluster_data = pd.DataFrame({'subreddit': subreddits,'cluster':clustering.labels_})
|
||
|
|
|
||
|
|
cluster_sizes = cluster_data.groupby("cluster").count().reset_index()
|
||
|
|
print(f"the largest cluster has {cluster_sizes.loc[cluster_sizes.cluster!=-1].subreddit.max()} members")
|
||
|
|
|
||
|
|
print(f"the median cluster has {cluster_sizes.subreddit.median()} members")
|
||
|
|
|
||
|
|
print(f"{(cluster_sizes.subreddit==1).sum()} clusters have 1 member")
|
||
|
|
|
||
|
|
print(f"{(cluster_sizes.loc[cluster_sizes.cluster==-1,['subreddit']])} subreddits are in cluster -1",flush=True)
|
||
|
|
|
||
|
|
return cluster_data
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class clustering_result:
|
||
|
|
outpath:Path
|
||
|
|
max_iter:int
|
||
|
|
silhouette_score:float
|
||
|
|
alt_silhouette_score:float
|
||
|
|
name:str
|
||
|
|
n_clusters:int
|
||
|
|
|
||
|
|
def read_similarity_mat(similarities, use_threads=True):
|
||
|
|
df = pd.read_feather(similarities, use_threads=use_threads)
|
||
|
|
mat = np.array(df.drop('_subreddit',1))
|
||
|
|
n = mat.shape[0]
|
||
|
|
mat[range(n),range(n)] = 1
|
||
|
|
return (df._subreddit,mat)
|