Break code into more files.

code/recc/__init__.py

@@ -0,0 +1,2 @@
+from .build_suggestion import ReccModel, sim_from_tag_index
+from .federated_design import TagData

code/recc/build_suggestion.py

@@ -1,11 +1,10 @@
 from federated_design import *
-from sklearn.cluster import AffinityPropagation
-from sklearn.decomposition import TruncatedSVD
 from scipy.sparse.linalg import svds
 from sklearn.preprocessing import normalize
 import polars as pl
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.metrics.pairwise import euclidean_distances
+from sklearn.decomposition import PCA
 import json
 
 class ReccModel:
@@ -33,10 +32,13 @@ class ReccModel:
   def top_tags(self):
     u, s, v = self.svd(k=25)
     tag_stuff = np.diag(s) @ v
+    pca = PCA(n_components=2)
+    tag_pca = pca.fit_transform(tag_stuff.T)
+    print(tag_pca[:, 0])
     return pl.DataFrame({
       "tag": self.tag_names,
-      "x": tag_stuff[-1],
+      "x": tag_pca[:, 0],
-      "y": tag_stuff[-2],
+      "y": tag_pca[:, 1],
       "variance": np.var(tag_stuff, axis=0),
       "count": self.tag_use_counts[self.has_info].tolist(),
       "index": np.arange(len(self.tag_names))
@@ -52,8 +54,8 @@ class ReccModel:
     })
 
 # This one seem pretty good!
-def sim_from_tag_index(index=1000):
+def sim_from_tag_index(rm: ReccModel, index=1000):
-  u, s, v = rm.svd(k=50, norm_axis=0)
+  u, s, v = rm.svd(k=25, norm_axis=0)
   m = (np.diag(s) @ v).T
   pos = m[index]
   server_matrix = u @ np.diag(s)
@@ -64,8 +66,9 @@ if __name__ == "__main__":
   rm = ReccModel()
   rm.top_tags().write_ipc("data/scratch/tag_svd.feather")
   rm.top_servers().write_ipc("data/scratch/server_svd.feather")
-  u, s, v = rm.svd(k=100, norm_axis=None)
+  u, s, v = rm.svd(k=50, norm_axis=None)
-  pos_m = v.T#(v.T @ np.diag(s))#v.T#
+  #pos_m = v.T#(v.T @ np.diag(s))#v.T#
+  pos_m = v.T @ np.diag(s)
   server_matrix = u#u @ np.diag(s)#u#
   with open("recommender/data/positions.json", "w") as f:
     f.write(json.dumps(pos_m.tolist()))
@@ -76,4 +79,22 @@ if __name__ == "__main__":
   with open("recommender/data/tag_names.json", "w") as f:
     f.write(json.dumps(rm.tag_names.tolist()))
 
+
+  top_server_tags_df = rm.tfidf.sort(pl.col("tf_idf"), descending=True).with_columns(pl.lit(1).alias("counter")).with_columns(
+    pl.col("counter").cum_sum().over("host").alias("running_count")
+  ).filter(pl.col("running_count") <= 5).drop("counter", "running_count").select(["host", "tags", "idf", "tf_idf"]).filter(
+    pl.col("tf_idf") > 4
+  )
+  top_server_tags = {}
+  for row in top_server_tags_df.iter_rows(named=True):
+    if row["host"] not in top_server_tags:
+      top_server_tags[row["host"]] = []
+    top_server_tags[row["host"]].append(row["tags"])
+  with open("recommender/data/server_top_tags.json", "w") as f:
+    f.write(json.dumps(top_server_tags))
+
+  # group by host and add descending value
+  #rm.tfidf.sort(pl.col("tf_idf"), descending=True).group_by("host").with_row_index()
+
+
 #rm.server_names[np.argsort(-cosine_similarity(pos_m[779].reshape(1, -1), server_matrix))].tolist()[0][0:10]

code/recc/federated_design.py

@@ -2,8 +2,8 @@ import polars as pl
 from scipy.sparse import lil_matrix
 from sklearn.metrics.pairwise import cosine_similarity
 import numpy as np
-import textdistance
+#import textdistance
-from scipy.stats import kendalltau
+#from scipy.stats import kendalltau
 import rbo
 import scipy
 

recommender/app/page.tsx

@@ -0,0 +1,11 @@
+"use client";
+import React from 'react';
+import { TagSelector } from '../components/TagSelector';
+
+export default function Home() {
+  return (
+    <main className="flex min-h-screen flex-col items-center justify-between p-24">
+      <TagSelector />
+    </main>
+  );
+}

recommender/app/witch/page.tsx

@@ -1,193 +0,0 @@
-"use client";
-import React, { useState, useEffect } from 'react';
-//import top_tags from "@/data/top_tags.json";
-import positions from "@/data/positions.json";
-import server_matrix from "@/data/server_matrix.json";
-import server_names from "@/data/server_names.json";
-import tag_names from "@/data/tag_names.json";
-
-interface Tag {
-  index: number;
-  tag: string;
-}
-// Read from data/top_tags.json FILE
-//const tags: Tag[] = 
-
-const selected_tags = [
-  "politics", "gardening", "art", "nature", "pride", "cycling", "climate",
-  "programming", "dogs", "privacy", "cats", "gaming", "education",
-  "science", "music", "movies", "food", "books", "lgbtq", "python",
-  "emacs", "gay", "trans", "furry", "photography", "cooking", "literature",
-  "television"
-];
-
-
-function dotProduct(a: number[], b: number[]): number {
-  return a.map((x, i) => x * b[i]).reduce((sum, current) => sum + current, 0);
-}
-
-function magnitude(arr: number[]): number {
-  return Math.sqrt(arr.map(x => x * x).reduce((sum, current) => sum + current, 0));
-}
-
-function cosineSimilarity(arr1: number[], arr2: number[]): number {
-  if (arr1.length !== arr2.length) {
-      throw new Error("Arrays must have the same length");
-  }
-
-  const dotProd = dotProduct(arr1, arr2);
-  const magnitudeProd = magnitude(arr1) * magnitude(arr2);
-  if (magnitudeProd === 0) {
-      return 0;
-  }
-  return dotProd / magnitudeProd;
-}
-
-function averageOfArrays(arr: number[][]): number[] {
-  // Get the length of the first sub-array
-  const length = arr[0].length;
-
-  // Initialize an array to store the sums
-  const sums = Array(length).fill(0);
-
-  // Loop over each sub-array
-  for (let i = 0; i < arr.length; i++) {
-    // Loop over each element in the sub-array
-    for (let j = 0; j < arr[i].length; j++) {
-      // Add the element to the corresponding sum
-      sums[j] += arr[i][j];
-    }
-  }
-
-  // Divide each sum by the number of arrays to get the average
-  const averages = sums.map(sum => sum / arr.length);
-
-  return averages;
-}
-
-
-
-const TagSelector: React.FC = () => {
-  //const top_tags: Tag[] = [];
-  const all_tags: Tag[] = tag_names.map((tag: string, index: number) => ({index, tag}))
-  const tags: Tag[] = all_tags.filter((tag: Tag) => selected_tags.includes(tag.tag));
-  //top_tags.filter((tag) => selected_tags.includes(tag.tag));
-  // State to keep track of selected tag IDs.
-  const [selectedTagIds, setSelectedTagIds] = useState<number[]>([]);
-
-  const [suggestedTagIds, setSuggestedTagIds] = useState<number[]>([]);
-
-  const [topServerIds, setTopServerIds] = useState<number[]>([]);
-
-  // Function to handle tag selection toggling.
-  const toggleTag = (tagId: number) => {
-    setSelectedTagIds((currentTagIds) =>
-      currentTagIds.includes(tagId)
-        ? currentTagIds.filter((id) => id !== tagId)
-        : [...currentTagIds, tagId],
-    )
-  };
-
-  function find_most_similar_tags(all_tags: Tag[], selectedTagIds: number[], positions: number[][]) {
-    let most_similar: Tag[] = [];
-    // get the average position of all selected tags
-    if (selectedTagIds.length > 0) {
-      // loop through all selected tags and get their positions
-      const selected_positions = selectedTagIds.map((tagId) => positions[tagId]);
-      for (let i = 0; i < selected_positions.length; i++) {
-        let tag_similarity = positions.map((row) => cosineSimilarity(selected_positions[i], row));
-        let tag_rank = tag_similarity.map((similarity, index) => ({index, similarity})).sort((a, b) => b.similarity - a.similarity).map((item, index) => ({index: item.index, similarity: item.similarity, name: tag_names[item.index]}));
-        console.log(tag_rank.slice(0, 10));
-        let topServerIds = tag_rank.slice(0, 10).map((item) => item.index);
-        for (let tagId of all_tags.filter((tag) => topServerIds.includes(tag.index))) {
-          if (!selectedTagIds.includes(tagId.index) && !most_similar.includes(tagId)) {
-            most_similar.push(tagId);
-          }
-        }
-      }
-    }
-    return most_similar;
-  }
-
-  useEffect(() => {
-    // get the average position of all selected tags
-    if (selectedTagIds.length > 0) {
-      let selected_positions = selectedTagIds.map((tagId) => positions[tagId]);
-      // get the average of selected positions
-      let average_position = averageOfArrays(selected_positions);
-      // loop through each row of the server_matrix and calculate the cosine similarity
-      const server_similarity = server_matrix.map((row) => cosineSimilarity(average_position, row));
-      const server_rank = server_similarity.map((similarity, index) => ({index, similarity})).sort((a, b) => b.similarity - a.similarity).map((item, index) => ({index: item.index, similarity: item.similarity, name: server_names[item.index]}));
-      setTopServerIds(server_rank.slice(0, 10).map((item) => item.index));
-
-      // Find the most similar tags among all tags
-      const tag_similarity = all_tags.map((tag) => ({t: tag, sim: cosineSimilarity(average_position, positions[tag.index])})).sort((a, b) => b.sim - a.sim);
-      const most_similar = find_most_similar_tags(all_tags, selectedTagIds, positions);
-      //tag_similarity.slice(0, 50).map((item) => item.t);
-      console.log(most_similar);
-      setSuggestedTagIds(most_similar.map((tag) => tag.index));
-      console.log(suggestedTagIds);
-    }
-  }, [selectedTagIds]);
-
-  return (
-    <div>
-      <div className="flex flex-wrap gap-2">
-        <div>
-          <h3>Selected</h3>
-          {all_tags.filter((tag) => selectedTagIds.includes(tag.index)).map((tag) => (
-            <button
-              key={tag.index}
-              onClick={() => toggleTag(tag.index)}
-              className={`px-3 py-1 rounded-full text-sm ${
-                selectedTagIds.includes(tag.index) ? 'bg-blue-500 text-white' : 'bg-gray-200 text-gray-800'
-              } transition-colors duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50`}
-            >
-              {tag.tag}
-            </button>
-          ))}
-        </div>
-        <div>
-          <h3>Categories</h3>
-          {tags.filter((tag) => !selectedTagIds.includes(tag.index)).map((tag) => (
-              <button
-                key={tag.index}
-                onClick={() => toggleTag(tag.index)}
-                className={`px-3 py-1 rounded-full text-sm bg-gray-200 text-gray-800 transition-colors duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50`}
-              >
-                {tag.tag}
-              </button>
-            ))}
-        </div>
-        <div>
-          <h3>Suggested Tags ({suggestedTagIds.filter((tag) => !selectedTagIds.includes(tag)).length})</h3>
-          {all_tags.filter((tag) => suggestedTagIds.includes(tag.index)).filter((tag) => !selectedTagIds.includes(tag.index)).map((tag) => (
-              <button
-                key={tag.index}
-                onClick={() => toggleTag(tag.index)}
-                className={`px-3 py-1 rounded-full text-sm bg-gray-200 text-gray-800 transition-colors duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50`}
-              >
-                {tag.tag}
-              </button>
-            ))}
-        </div>
-      </div>
-      <div>
-        <ul>
-          {topServerIds.map((id) => (
-            <li key={id}>{server_names[id]}</li>
-          ))}
-        </ul>
-      </div>
-    </div>
-  );
-};
-
-
-export default function Home() {
-  return (
-    <main className="flex min-h-screen flex-col items-center justify-between p-24">
-      <TagSelector />
-    </main>
-  );
-}

recommender/components/SensativeSelector.tsx

@@ -0,0 +1,25 @@
+import { Checkbox, Field, Label } from '@headlessui/react';
+
+interface SensativeSelectorProps {
+  showSensitiveTags: boolean;
+  setShowSensitiveTags: (checked: boolean) => void;
+}
+
+export const SensativeSelector: React.FC<SensativeSelectorProps> = ({ showSensitiveTags, setShowSensitiveTags }) => {
+  return (
+    <div>
+      <Field className="flex items-center gap-2">
+        <Checkbox
+          checked={showSensitiveTags}
+          onChange={setShowSensitiveTags}
+          className="group block size-4 rounded border bg-white data-[checked]:bg-blue-500"
+        >
+          <svg className="stroke-white opacity-0 group-data-[checked]:opacity-100" viewBox="0 0 14 14" fill="none">
+            <path d="M3 8L6 11L11 3.5" strokeWidth={2} strokeLinecap="round" strokeLinejoin="round" />
+          </svg>
+        </Checkbox>
+        <Label>Show sensitive tags?</Label>
+      </Field>
+    </div>
+  )
+}

recommender/components/TagSelector.tsx

@@ -0,0 +1,130 @@
+"use client";
+import React, { useState, useEffect } from 'react';
+import positions from "@/data/positions.json";
+import server_matrix from "@/data/server_matrix.json";
+import server_names from "@/data/server_names.json";
+import tag_names from "@/data/tag_names.json";
+import { cosineSimilarity, averageOfArrays } from "@/utils/math";
+import { ServerCards } from "@/components/server_cards";
+import type { Tag } from "@/utils/types";
+import { isSensitive } from "@/utils/types";
+//import { selected_tags, find_most_similar_tags } from '@/app/page';
+import { selected_tags, find_most_similar_tags } from '@/utils/types';
+import { SensativeSelector } from './SensativeSelector';
+import { TagSearch } from './tag_search';
+
+export const TagSelector: React.FC = () => {
+  //const top_tags: Tag[] = [];
+  const all_tags: Tag[] = tag_names.map((tag: string, index: number) => ({ index, tag }));
+  const tags: Tag[] = all_tags.filter((tag: Tag) => selected_tags.includes(tag.tag));
+
+  // state
+  const [selectedTagIds, setSelectedTagIds] = useState<number[]>([]);
+  const [suggestedTagIds, setSuggestedTagIds] = useState<number[]>([]);
+  const [topServerIds, setTopServerIds] = useState<number[]>([]);
+  const [showSensitiveTags, setShowSensitiveTags] = useState<boolean>(false);
+
+  // Function to handle tag selection toggling.
+  const toggleTag = (tagId: number) => {
+    setSelectedTagIds((currentTagIds) => currentTagIds.includes(tagId)
+      ? currentTagIds.filter((id) => id !== tagId)
+      : [...currentTagIds, tagId]
+    );
+  };
+
+  useEffect(() => {
+    // get the average position of all selected tags
+    if (selectedTagIds.length == 0) {
+      setTopServerIds([]);
+      setSuggestedTagIds([]);
+    } else {
+      let selected_positions = selectedTagIds.map((tagId) => positions[tagId]);
+      // get the average of selected positions
+      let average_position = averageOfArrays(selected_positions);
+      // loop through each row of the server_matrix and calculate the cosine similarity
+      const server_similarity = averageOfArrays(selected_positions.map((pos) => server_matrix.map((row) => cosineSimilarity(pos, row))));
+      //console.log(server_similarity);
+      const server_rank = server_similarity.map((similarity, index) => ({ index, similarity })).sort((a, b) => b.similarity - a.similarity).map((item, index) => ({ index: item.index, similarity: item.similarity, name: server_names[item.index] }));
+      //console.log(server_rank);
+      setTopServerIds(server_rank.slice(0, 25).map((item) => item.index));
+      // Find the most similar tags among all tags
+      const tag_similarity = all_tags.map((tag) => ({ t: tag, sim: cosineSimilarity(average_position, positions[tag.index]) })).sort((a, b) => b.sim - a.sim);
+      const most_similar = find_most_similar_tags(all_tags, tag_names, selectedTagIds, positions);
+      setSuggestedTagIds(most_similar.map((tag) => tag.index));
+    }
+  }, [selectedTagIds, all_tags]);
+
+  return (
+    <div>
+      <div className="flex flex-wrap gap-2">
+        <div>
+          <h1 className='text-3xl font-bold'>Mastodon Server Reccomender</h1>
+          <div className='grid grid-flow-row-dense grid-cols-3'>
+            <div className='mr-3 mb-2 p-3 bg-gray-100 col-span-2'>
+              <h3>Search</h3>
+              <TagSearch all_tags={all_tags} selectedTagIds={selectedTagIds} onToggle={toggleTag} />
+            </div>
+            <div className='mr-3 mb-2 p-3'>
+              <h3>Settings</h3>
+              <SensativeSelector 
+                showSensitiveTags={showSensitiveTags}
+                setShowSensitiveTags={setShowSensitiveTags}
+              />
+            </div>
+          </div>
+          {/*<div>
+              <p>Languages</p>
+              <select value={"en"}>
+                {locales.map((locale) => (
+                  <option key={locale.code} value={locale.code}>{locale.language}</option>
+                ))}
+              </select>
+              </div>*/}
+          <h3>Selected</h3>
+          {all_tags.filter((tag) => selectedTagIds.includes(tag.index)).map((tag) => (
+            <button
+              key={tag.index}
+              onClick={() => toggleTag(tag.index)}
+              className={`px-3 py-1 m-1 rounded-full text-sm ${selectedTagIds.includes(tag.index) ? 'bg-blue-500 text-white' : 'bg-gray-200 text-gray-800'} transition-colors duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50`}
+            >
+              {tag.tag}
+            </button>
+          ))}
+        </div>
+        <div>
+          <h3>Categories</h3>
+          {tags.filter((tag) => !selectedTagIds.includes(tag.index)).map((tag) => (
+            <button
+              key={tag.index}
+              onClick={() => toggleTag(tag.index)}
+              className={`px-3 py-1 m-1 rounded-full text-sm bg-gray-200 text-gray-800 transition-colors duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50`}
+            >
+              {tag.tag}
+            </button>
+          ))}
+        </div>
+        <div>
+          <h3>Suggested Tags ({suggestedTagIds.filter((tag) => !selectedTagIds.includes(tag)).length})</h3>
+          {all_tags.filter((tag) => suggestedTagIds.includes(tag.index)).filter(
+            (tag) => !selectedTagIds.includes(tag.index)
+          ).filter(
+            (tag) => showSensitiveTags || !isSensitive(tag)
+          ).map((tag) => (
+            <button
+              key={tag.index}
+              onClick={() => toggleTag(tag.index)}
+              className={`px-3 py-1 m-1 rounded-full text-sm ${isSensitive(tag) ? 'bg-red-200' : 'bg-gray-200'} text-gray-800 transition-colors duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-blue-400 focus:ring-opacity-50`}
+            >
+              {isSensitive(tag) ? tag.tag.split('_')[0] : tag.tag}
+            </button>
+          ))}
+        </div>
+      </div>
+      <div>
+        <ul>
+          <ServerCards topServerIds={topServerIds} tagCount={selectedTagIds.length} />
+        </ul>
+      </div>
+    </div>
+  );
+};

recommender/components/server_cards.tsx

@@ -0,0 +1,77 @@
+import server_names from "@/data/server_names.json";
+import servers from "@/data/servers.json";
+
+import top_server_tags_data from "@/data/server_top_tags.json";
+import { locales } from '@/utils/locales';
+
+const top_server_tags: {[server: string]: string[]} = top_server_tags_data;
+
+//const ServerCards: React.FC = (topServerIds: number[]) => {
+interface Server {
+    domain: string;
+    description: string;
+    proxied_thumbnail: string;
+    top_tags: string[];
+    languages: string[];
+}
+
+// map { code: "be", language: "Беларуская" } to dictionary
+const localeDictionary = locales.reduce((acc, locale) => {
+  acc[locale.code] = locale.language;
+  return acc;
+}, {} as { [key: string]: string });
+
+console.log(localeDictionary);
+    
+const server_data: {[id: number]: Server} = {};
+for (let i = 0; i < servers.length; i++) {
+    let s_name: string = servers[i].domain;
+    let index = server_names.indexOf(s_name);
+    if (index > -1) {
+        server_data[index] = {
+            domain: servers[i].domain,
+            description: servers[i].description,
+            proxied_thumbnail: servers[i].proxied_thumbnail,
+            top_tags: [],
+            languages: servers[i].languages
+        };
+        if (top_server_tags[s_name] !== undefined) {
+            server_data[index].top_tags = top_server_tags[s_name];
+        }
+    };
+};
+
+export function ServerCards(props: {topServerIds: number[], tagCount: number}) {
+  let server_info = props.topServerIds.map((id) => server_data[id]);
+  return (
+    <div>
+    {props.tagCount >= 2 ? (
+      <div className='col-span-4 md:col-start-4 md:col-end-13'>
+        <div className='grid gap-gutter sm:grid-cols-2 md:grid-cols-2 xl:grid-cols-4'>
+          {server_info.filter((data) => data !== undefined).map((data) => (
+            <div key={data.domain} className='grid grid-rows-[auto_1fr_auto] rounded-md border border-gray-3 p-4'>
+              <div>
+                <img src={data.proxied_thumbnail} decoding='async' />
+              </div>
+              <div className='pb-5'>
+                <p className='b1 !font-700 font-bold mb-2 text-lg'>{data.domain}</p>
+                <p>{data.languages.map((lang) => (
+                  <span className="mr-1 text-xs" key={lang}>{localeDictionary[lang]}</span>
+                ))}</p>
+                <p className='b3 line-clamp-5 [unicode-bidi:plaintext] text-sm'>{data.description}</p>
+                <ul>
+                  {data.top_tags.map((tag) => (
+                    <li key={tag} className={`inline-block ${tag.endsWith("_sensitive") ? 'bg-red-100' : 'bg-gray-100'} text-gray-800 px-2 py-1 text-xs mr-2 mb-2 uppercase font-mono`}>{tag.split('_')[0]}</li>
+                  ))}
+                </ul>
+              </div>
+            </div>
+          ))}
+        </div>
+      </div>
+    ) : (
+      <div>Please select at least two tags.</div>
+    )}
+    </div>
+    );
+}

recommender/components/tag_search.tsx

@@ -0,0 +1,51 @@
+import React, { useState, useEffect } from 'react';
+import { Combobox, ComboboxInput, ComboboxOption, ComboboxOptions } from '@headlessui/react';
+import type { Tag } from '@/utils/types';
+
+interface TagSearchProps {
+  all_tags: Tag[],
+  selectedTagIds: number[];
+  onToggle: (tag: number) => void;
+}
+
+export const TagSearch: React.FC<TagSearchProps> = ({ all_tags, selectedTagIds, onToggle }) => {
+
+  const [tagQuery, setTagQuery] = useState('')
+
+  const tag_options = all_tags;
+
+  const filteredTags =
+    (tagQuery === '' || tagQuery.length < 2) ?
+    [] :
+    tag_options.filter((tag) => (
+      tag.tag.toLowerCase().startsWith(tagQuery.toLowerCase())
+    ));
+
+
+  return (
+    <Combobox
+      value={selectedTagIds}
+      /*virtual={{ options: filteredTags }}*/
+      onChange={(tag_id) => {
+        if (typeof tag_id === 'number') {
+          onToggle(tag_id);
+        }
+      }}
+      onClose={() => setTagQuery('')}
+    >
+      <ComboboxInput
+        aria-label="Assignee"
+        /*displayValue={(tag: Tag) => tag?.tag}*/
+        className='border data-[hover]:shadow data-[focus]:bg-blue-100 w-full'
+        onChange={(event) => setTagQuery(event.target.value)}
+      />
+      <ComboboxOptions anchor="bottom" className="w-52 empty:hidden border-2">
+        {filteredTags.map((tag) => (
+          <ComboboxOption key={tag.index} value={tag.index} className="group flex gap-2 bg-white data-[focus]:bg-blue-100">
+            {tag.tag}
+          </ComboboxOption>
+        ))}
+      </ComboboxOptions>
+    </Combobox>
+  )
+};

recommender/next.config.mjs

@@ -1,7 +1,7 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: 'export',
-  basePath: '/files/jsdemo'
+  basePath: '/demos/deweb2024'
 };
 
 export default nextConfig;

recommender/package-lock.json

@@ -8,6 +8,7 @@
       "name": "recommender",
       "version": "0.1.0",
       "dependencies": {
+        "@headlessui/react": "^2.0.3",
         "next": "14.1.3",
         "react": "^18",
         "react-dom": "^18"
@@ -113,6 +114,72 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
+    "node_modules/@floating-ui/core": {
+      "version": "1.6.2",
+      "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.6.2.tgz",
+      "integrity": "sha512-+2XpQV9LLZeanU4ZevzRnGFg2neDeKHgFLjP6YLW+tly0IvrhqT4u8enLGjLH3qeh85g19xY5rsAusfwTdn5lg==",
+      "dependencies": {
+        "@floating-ui/utils": "^0.2.0"
+      }
+    },
+    "node_modules/@floating-ui/dom": {
+      "version": "1.6.5",
+      "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.6.5.tgz",
+      "integrity": "sha512-Nsdud2X65Dz+1RHjAIP0t8z5e2ff/IRbei6BqFrl1urT8sDVzM1HMQ+R0XcU5ceRfyO3I6ayeqIfh+6Wb8LGTw==",
+      "dependencies": {
+        "@floating-ui/core": "^1.0.0",
+        "@floating-ui/utils": "^0.2.0"
+      }
+    },
+    "node_modules/@floating-ui/react": {
+      "version": "0.26.15",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.15.tgz",
+      "integrity": "sha512-WKmfLkxTwCm09Dxq4LpjL3EPbZVSp5wvnap1jmculsfnzg2Ag/pCkP+OPyjE5dFMXqX97hsLIqJehboZ5XAHXw==",
+      "dependencies": {
+        "@floating-ui/react-dom": "^2.0.0",
+        "@floating-ui/utils": "^0.2.0",
+        "tabbable": "^6.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0",
+        "react-dom": ">=16.8.0"
+      }
+    },
+    "node_modules/@floating-ui/react-dom": {
+      "version": "2.0.9",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.0.9.tgz",
+      "integrity": "sha512-q0umO0+LQK4+p6aGyvzASqKbKOJcAHJ7ycE9CuUvfx3s9zTHWmGJTPOIlM/hmSBfUfg/XfY5YhLBLR/LHwShQQ==",
+      "dependencies": {
+        "@floating-ui/dom": "^1.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0",
+        "react-dom": ">=16.8.0"
+      }
+    },
+    "node_modules/@floating-ui/utils": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.2.tgz",
+      "integrity": "sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw=="
+    },
+    "node_modules/@headlessui/react": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-2.0.3.tgz",
+      "integrity": "sha512-Xd1h0YZgfhxZ7W1w4TvK0/TZ1c4qaX4liYVUkAXqW1HCLcXSqnMeYAUGJS/BBroBAUL9HErjyFcRpCWRQZ/0lA==",
+      "dependencies": {
+        "@floating-ui/react": "^0.26.13",
+        "@react-aria/focus": "^3.16.2",
+        "@react-aria/interactions": "^3.21.1",
+        "@tanstack/react-virtual": "3.5.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "react": "^18",
+        "react-dom": "^18"
+      }
+    },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.14",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz",
@@ -432,6 +499,83 @@
         "node": ">=14"
       }
     },
+    "node_modules/@react-aria/focus": {
+      "version": "3.17.0",
+      "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.17.0.tgz",
+      "integrity": "sha512-aRzBw1WTUkcIV3xFrqPA6aB8ZVt3XyGpTaSHAypU0Pgoy2wRq9YeJYpbunsKj9CJmskuffvTqXwAjTcaQish1Q==",
+      "dependencies": {
+        "@react-aria/interactions": "^3.21.2",
+        "@react-aria/utils": "^3.24.0",
+        "@react-types/shared": "^3.23.0",
+        "@swc/helpers": "^0.5.0",
+        "clsx": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0"
+      }
+    },
+    "node_modules/@react-aria/interactions": {
+      "version": "3.21.2",
+      "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.21.2.tgz",
+      "integrity": "sha512-Ju706DtoEmI/2vsfu9DCEIjDqsRBVLm/wmt2fr0xKbBca7PtmK8daajxFWz+eTq+EJakvYfLr7gWgLau9HyWXg==",
+      "dependencies": {
+        "@react-aria/ssr": "^3.9.3",
+        "@react-aria/utils": "^3.24.0",
+        "@react-types/shared": "^3.23.0",
+        "@swc/helpers": "^0.5.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0"
+      }
+    },
+    "node_modules/@react-aria/ssr": {
+      "version": "3.9.3",
+      "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.3.tgz",
+      "integrity": "sha512-5bUZ93dmvHFcmfUcEN7qzYe8yQQ8JY+nHN6m9/iSDCQ/QmCiE0kWXYwhurjw5ch6I8WokQzx66xKIMHBAa4NNA==",
+      "dependencies": {
+        "@swc/helpers": "^0.5.0"
+      },
+      "engines": {
+        "node": ">= 12"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0"
+      }
+    },
+    "node_modules/@react-aria/utils": {
+      "version": "3.24.0",
+      "resolved": "https://registry.npmjs.org/@react-aria/utils/-/utils-3.24.0.tgz",
+      "integrity": "sha512-JAxkPhK5fCvFVNY2YG3TW3m1nTzwRcbz7iyTSkUzLFat4N4LZ7Kzh7NMHsgeE/oMOxd8zLY+XsUxMu/E/2GujA==",
+      "dependencies": {
+        "@react-aria/ssr": "^3.9.3",
+        "@react-stately/utils": "^3.10.0",
+        "@react-types/shared": "^3.23.0",
+        "@swc/helpers": "^0.5.0",
+        "clsx": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0"
+      }
+    },
+    "node_modules/@react-stately/utils": {
+      "version": "3.10.0",
+      "resolved": "https://registry.npmjs.org/@react-stately/utils/-/utils-3.10.0.tgz",
+      "integrity": "sha512-nji2i9fTYg65ZWx/3r11zR1F2tGya+mBubRCbMTwHyRnsSLFZaeq/W6lmrOyIy1uMJKBNKLJpqfmpT4x7rw6pg==",
+      "dependencies": {
+        "@swc/helpers": "^0.5.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0"
+      }
+    },
+    "node_modules/@react-types/shared": {
+      "version": "3.23.0",
+      "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.23.0.tgz",
+      "integrity": "sha512-GQm/iPiii3ikcaMNR4WdVkJ4w0mKtV3mLqeSfSqzdqbPr6vONkqXbh3RhPlPmAJs1b4QHnexd/wZQP3U9DHOwQ==",
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0"
+      }
+    },
     "node_modules/@rushstack/eslint-patch": {
       "version": "1.7.2",
       "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.7.2.tgz",
@@ -446,6 +590,31 @@
         "tslib": "^2.4.0"
       }
     },
+    "node_modules/@tanstack/react-virtual": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.5.0.tgz",
+      "integrity": "sha512-rtvo7KwuIvqK9zb0VZ5IL7fiJAEnG+0EiFZz8FUOs+2mhGqdGmjKIaT1XU7Zq0eFqL0jonLlhbayJI/J2SA/Bw==",
+      "dependencies": {
+        "@tanstack/virtual-core": "3.5.0"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
+    "node_modules/@tanstack/virtual-core": {
+      "version": "3.5.0",
+      "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.5.0.tgz",
+      "integrity": "sha512-KnPRCkQTyqhanNC0K63GBG3wA8I+D1fQuVnAvcBF8f13akOKeQp1gSbu6f77zCxhEk727iV5oQnbHLYzHrECLg==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      }
+    },
     "node_modules/@types/json5": {
       "version": "0.0.29",
       "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
@@ -1189,6 +1358,14 @@
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
       "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
     },
+    "node_modules/clsx": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
+      "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/color-convert": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -4373,6 +4550,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/tabbable": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz",
+      "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="
+    },
     "node_modules/tailwindcss": {
       "version": "3.4.1",
       "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-3.4.1.tgz",

recommender/package.json

@@ -9,6 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@headlessui/react": "^2.0.3",
     "next": "14.1.3",
     "react": "^18",
     "react-dom": "^18"

recommender/tailwind.config.ts

@@ -4,15 +4,11 @@ const config: Config = {
   content: [
     "./pages/**/*.{js,ts,jsx,tsx,mdx}",
     "./components/**/*.{js,ts,jsx,tsx,mdx}",
+    "./app/*.{js,ts,jsx,tsx,mdx}",
     "./app/**/*.{js,ts,jsx,tsx,mdx}",
   ],
   theme: {
     extend: {
-      backgroundImage: {
-        "gradient-radial": "radial-gradient(var(--tw-gradient-stops))",
-        "gradient-conic":
-          "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))",
-      },
     },
   },
   plugins: [],

recommender/unused-tailwind.config.js

@@ -1,6 +1,7 @@
 /** @type {import('tailwindcss').Config} */
 module.exports = {
   content: [
+    "./app/*.{js,ts,jsx,tsx,mdx}",
     "./app/**/*.{js,ts,jsx,tsx,mdx}",
     "./pages/**/*.{js,ts,jsx,tsx,mdx}",
     "./components/**/*.{js,ts,jsx,tsx,mdx}"

recommender/utils/locales.tsx

@@ -0,0 +1,43 @@
+export const locales = [
+    { code: "ar", language: "العربية", dir: "rtl" },
+    { code: "be", language: "Беларуская" },
+    { code: "bn", language: "বাংলা" },
+    { code: "ca", language: "Català" },
+    { code: "cs", language: "Čeština" },
+    { code: "cy", language: "Cymraeg" },
+    { code: "de", language: "Deutsch" },
+    { code: "en", language: "English" },
+    { code: "eo", language: "Esperanto" },
+    { code: "es", language: "Español" },
+    { code: "eu", language: "Euskara" },
+    { code: "fa", language: "فارسی", dir: "rtl" },
+    { code: "fi", language: "Suomi" },
+    { code: "fr", language: "Français" },
+    { code: "fy", language: "Frysk" },
+    { code: "ga", language: "Gaeilge" },
+    { code: "gd", language: "Gàidhlig" },
+    { code: "gl", language: "Galego" },
+    { code: "he", language: "עברית", dir: "rtl" },
+    { code: "is", language: "Íslenska" },
+    { code: "it", language: "Italiano" },
+    { code: "ja", language: "日本語" },
+    { code: "kab", language: "Taqbaylit" },
+    { code: "ko", language: "한국어" },
+    { code: "ku", language: "Kurmancî" },
+    { code: "nl-NL", language: "Nederlands" },
+    { code: "no", language: "Norsk" },
+    { code: "pl", language: "Polski" },
+    { code: "pt-BR", language: "Português" },
+    { code: "ru", language: "Русский" },
+    { code: "si", language: "සිංහල" },
+    { code: "sk", language: "Slovenčina" },
+    { code: "sl", language: "Slovenščina" },
+    { code: "sq", language: "Shqip" },
+    { code: "sv", language: "Svenska" },
+    { code: "th", language: "ภาษาไทย" },
+    { code: "tr", language: "Türkçe" },
+    { code: "uk", language: "Українська" },
+    { code: "vi", language: "Tiếng Việt" },
+    { code: "zh", language: "中文" },
+    { code: "zh-TW", language: "繁體中文" },
+  ];

recommender/utils/math.tsx

@@ -0,0 +1,44 @@
+function dotProduct(a: number[], b: number[]): number {
+    return a.map((x, i) => x * b[i]).reduce((sum, current) => sum + current, 0);
+  }  
+  
+  function magnitude(arr: number[]): number {
+    return Math.sqrt(arr.map(x => x * x).reduce((sum, current) => sum + current, 0));
+  }
+  
+  function cosineSimilarity(arr1: number[], arr2: number[]): number {
+    if (arr1.length !== arr2.length) {
+        throw new Error("Arrays must have the same length");
+    }
+  
+    const dotProd = dotProduct(arr1, arr2);
+    const magnitudeProd = magnitude(arr1) * magnitude(arr2);
+    if (magnitudeProd === 0) {
+        return 0;
+    }
+    return dotProd / magnitudeProd;
+  }
+  
+  function averageOfArrays(arr: number[][]): number[] {
+    // Get the length of the first sub-array
+    const length = arr[0].length;
+  
+    // Initialize an array to store the sums
+    const sums = Array(length).fill(0);
+  
+    // Loop over each sub-array
+    for (let i = 0; i < arr.length; i++) {
+      // Loop over each element in the sub-array
+      for (let j = 0; j < arr[i].length; j++) {
+        // Add the element to the corresponding sum
+        sums[j] += arr[i][j];
+      }
+    }
+  
+    // Divide each sum by the number of arrays to get the average
+    const averages = sums.map(sum => sum / arr.length);
+  
+    return averages;
+  }
+
+ export { cosineSimilarity, averageOfArrays };

recommender/utils/other.tsx

@@ -0,0 +1,2 @@
+
+

recommender/utils/types.tsx

@@ -0,0 +1,41 @@
+import { cosineSimilarity } from "@/utils/math";
+
+interface Tag {
+    index: number;
+    tag: string;
+}
+
+export function isSensitive(tag: Tag) {
+  return tag.tag.endsWith("_sensitive")
+}
+
+export const selected_tags = [
+    "politics", "gardening", "art", "nature", "pride", "cycling", "climate",
+    "programming", "dogs", "privacy", "cats", "gaming", "education",
+    "science", "music", "movies", "food", "books", "lgbtq", "python",
+    "emacs", "gay", "trans", "furry", "photography", "cooking", "literature",
+    "television"
+  ];
+  
+  export function find_most_similar_tags(all_tags: Tag[], tag_names: string[], selectedTagIds: number[], positions: number[][]) {
+    let most_similar: Tag[] = [];
+    // get the average position of all selected tags
+    if (selectedTagIds.length > 0) {
+      // loop through all selected tags and get their positions
+      const selected_positions = selectedTagIds.map((tagId) => positions[tagId]);
+      for (let i = 0; i < selected_positions.length; i++) {
+        let tag_similarity = positions.map((row) => cosineSimilarity(selected_positions[i], row));
+        let tag_rank = tag_similarity.map((similarity, index) => ({index, similarity})).sort((a, b) => b.similarity - a.similarity).map((item, index) => ({index: item.index, similarity: item.similarity, name: tag_names[item.index]}));
+        console.log(tag_rank.slice(0, 10));
+        let topServerIds = tag_rank.slice(0, 10).map((item) => item.index);
+        for (let tagId of all_tags.filter((tag) => topServerIds.includes(tag.index))) {
+          if (!selectedTagIds.includes(tagId.index) && !most_similar.includes(tagId)) {
+            most_similar.push(tagId);
+          }
+        }
+      }
+    }
+    return most_similar;
+  }
+
+export type { Tag };