collective/cdsc_reddit
13
0
Fork 0
Code

split fitting and plotting tsne.

Browse Source
This commit is contained in:
Nate E TeBlunthuis 2020-11-11 16:38:22 -08:00
parent db53c0138a
commit 4447c60265
3 changed files with 37 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
fit_tsne.py Normal file
View File

@ -0,0 +1,35 @@
import pyarrow
import pandas as pd
from numpy import random
import numpy as np
from sklearn.manifold import TSNE
df = pd.read_feather("reddit_term_similarity_3000.feather")
df = df.sort_values(['i','j'])
n = max(df.i.max(),df.j.max())
def zero_pad(grp):
p = grp.shape[0]
grp = grp.sort_values('j')
return np.concatenate([np.zeros(n-p),np.ones(1),np.array(grp.value)])
col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates()
first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0]
col_names = [first_name] + list(col_names.subreddit_j)
mat = df.groupby('i').apply(zero_pad)
mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)])
mat = np.stack(mat)
mat = mat + np.tril(mat.transpose(),k=-1)
dist = 2*np.arccos(mat)/np.pi
tsne_model = TSNE(2,learning_rate=200,perplexity=40,n_iter=5000,metric='precomputed')
tsne_fit_model = tsne_model.fit(dist)
tsne_fit_whole = tsne_fit_model.fit_transform(mat)
plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names})
plot_data.to_feather("tsne_subreddit_fit.feather")

1
tsne_subreddit_fit.feather Symbolic link
View File

@ -0,0 +1 @@
.git/annex/objects/1M/PF/SHA256E-s60874--5fe93033b4fcac562cb235e85134d2bd330a0aecd6d0afc151f9b9c028b0ebe5/SHA256E-s60874--5fe93033b4fcac562cb235e85134d2bd330a0aecd6d0afc151f9b9c028b0ebe5

28
visualization/tsne_vis.py
View File

@ -7,33 +7,7 @@ from numpy import random
import numpy as np
from sklearn.manifold import TSNE
df = pd.read_csv("reddit_term_similarity_3000.csv")
df = df.sort_values(['i','j'])
n = max(df.i.max(),df.j.max())
def zero_pad(grp):
p = grp.shape[0]
grp = grp.sort_values('j')
return np.concatenate([np.zeros(n-p),np.zeros(1),np.array(grp.value)])
col_names = df.sort_values('j').loc[:,['subreddit_j']].drop_duplicates()
first_name = list(set(df.subreddit_i) - set(df.subreddit_j))[0]
col_names = [first_name] + list(col_names.subreddit_j)
mat = df.groupby('i').apply(zero_pad)
mat.loc[n] = np.concatenate([np.zeros(n),np.ones(1)])
mat = np.stack(mat)
# plot the matrix using the first and second eigenvalues
mat = mat + np.tril(mat.transpose(),k=-1)
tsne_model = TSNE(2,learning_rate=500,perplexity=40,n_iter=2000)
tsne_fit_model = tsne_model.fit(mat)
tsne_fit_whole = tsne_fit_model.fit_transform(mat)
plot_data = pd.DataFrame({'x':tsne_fit_whole[:,0],'y':tsne_fit_whole[:,1], 'subreddit':col_names})
plot_data.to_feather("tsne_subreddit_fit.feather")
pd.read_feather("tsne_subreddit_fit.feather")
slider = alt.binding_range(min=1,max=100,step=1,name='zoom: ')
selector = alt.selection_single(name='zoomselect',fields=['zoom'],bind='scales',init={'zoom':1})