{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "18977cd3-de19-43e0-87a7-79ddc0fda973",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gscratch/scrubbed/mjilg/jupyter-notebook/lib/python3.9/site-packages/torch/cuda/__init__.py:734: UserWarning: Can't initialize NVML\n",
      "  warnings.warn(\"Can't initialize NVML\")\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd \n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "afe53935-8175-4dee-bb27-0bc87fbd5d5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "70a77468-aba1-4042-9900-d3c96d4c9f88",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize\n",
    "#nltk.download('punkt_tab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8ab815f2-b63b-429b-9036-3870aae54fad",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fa6a7cea-1375-4153-a388-1847dfa5b257",
   "metadata": {},
   "outputs": [],
   "source": [
    "phab_path = \"/mmfs1/gscratch/comdata/users/mjilg/mw-repo-lifecycles/case1/0228_ve_phab_comments.csv\"\n",
    "phab_df = pd.read_csv(phab_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "812ab4c8-2561-466b-bc57-defc93f5c893",
   "metadata": {},
   "outputs": [],
   "source": [
    "#cleaning df \n",
    "phab_df['has_ref'] = phab_df['comment_text'].apply(lambda x: bool(re.search(r\"\\b(visualeditor|VE|ve|VisualEditor)\\b\", str(x), re.IGNORECASE)))\n",
    "phab_df['has_bot_ref'] = phab_df['comment_text'].apply(lambda x: bool(re.search(r\"\\b(bots|scripts|gadgets)\\b\", str(x), re.IGNORECASE)))\n",
    "phab_df['timestamp'] = pd.to_datetime(phab_df['date_created'], unit='s', origin='unix', utc=True)\n",
    "phab_df['comment_id'] = phab_df.index + 1\n",
    "# after 11-1-2012 before 11-1-2013\n",
    "filtered_phab_df = phab_df[(phab_df['date_created'] < 1383264000) & (phab_df['date_created'] > 1351728000)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "60bcef32-67be-44f5-a51a-84e6e63d29ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_urls(text):\n",
    "    return re.sub(r'http\\S+', '', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7babf07b-4f91-4e48-88a9-4fe10f8b668d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#turning df into sentences \n",
    "sentence_level_data = filtered_phab_df.explode(\n",
    "    'comment_text').reset_index(drop=True)\n",
    "\n",
    "sentence_level_data['sentence'] = sentence_level_data['comment_text'].apply(\n",
    "    lambda x: sent_tokenize(str(x)))\n",
    "\n",
    "sentence_level_data = sentence_level_data.explode('sentence').reset_index(drop=True)\n",
    "\n",
    "sentence_level_data['sentence_id'] = sentence_level_data.groupby('comment_id').cumcount() + 1\n",
    "\n",
    "sentence_level_data = sentence_level_data.drop(columns=['has_bot_ref', 'has_ref'], errors='ignore')\n",
    "\n",
    "sentence_level_data['has_ref'] = sentence_level_data['sentence'].apply(\n",
    "    lambda x: bool(re.search(r'\\b(visualeditor|VE|ve|VisualEditor)\\b', x)))\n",
    "\n",
    "sentence_level_data['has_bot_ref'] = sentence_level_data['sentence'].apply(\n",
    "    lambda x: bool(re.search(r'\\b(bots|scripts|gadgets)\\b', x)))\n",
    "\n",
    "sentence_level_data = sentence_level_data.reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "558d1638-abe9-4fc2-896e-6fc1bc396ca3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task_title</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>date_created</th>\n",
       "      <th>AuthorPHID</th>\n",
       "      <th>WMFaffil</th>\n",
       "      <th>TaskPHID</th>\n",
       "      <th>comment_type</th>\n",
       "      <th>status</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>comment_id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>has_ref</th>\n",
       "      <th>has_bot_ref</th>\n",
       "      <th>sentence_no_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>VisualEditor: [Regression] \"More\" menu gets sh...</td>\n",
       "      <td>Tested on both the Italian and the English Wik...</td>\n",
       "      <td>1380976920</td>\n",
       "      <td>PHID-USER-wil4b5lylrvf3krixlkl</td>\n",
       "      <td>True</td>\n",
       "      <td>PHID-TASK-64s56xzrc22ustp2z7wx</td>\n",
       "      <td>task_description</td>\n",
       "      <td>resolved</td>\n",
       "      <td>2013-10-05 12:42:00+00:00</td>\n",
       "      <td>709</td>\n",
       "      <td>Tested on both the Italian and the English Wik...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Tested on both the Italian and the English Wik...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>VisualEditor: [Regression] \"More\" menu gets sh...</td>\n",
       "      <td>Tested on both the Italian and the English Wik...</td>\n",
       "      <td>1380976920</td>\n",
       "      <td>PHID-USER-wil4b5lylrvf3krixlkl</td>\n",
       "      <td>True</td>\n",
       "      <td>PHID-TASK-64s56xzrc22ustp2z7wx</td>\n",
       "      <td>task_description</td>\n",
       "      <td>resolved</td>\n",
       "      <td>2013-10-05 12:42:00+00:00</td>\n",
       "      <td>709</td>\n",
       "      <td>--------------------------\\n**Version**: unspe...</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>--------------------------\\n**Version**: unspe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>VisualEditor: [Regression] \"More\" menu gets sh...</td>\n",
       "      <td>Note that this is fixed and has been deployed ...</td>\n",
       "      <td>1381281033</td>\n",
       "      <td>PHID-USER-ydswvwhh5pm4lshahjje</td>\n",
       "      <td>True</td>\n",
       "      <td>PHID-TASK-64s56xzrc22ustp2z7wx</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-10-09 01:10:33+00:00</td>\n",
       "      <td>710</td>\n",
       "      <td>Note that this is fixed and has been deployed ...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Note that this is fixed and has been deployed ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>VisualEditor: [Regression] \"More\" menu gets sh...</td>\n",
       "      <td>Note that this is fixed and has been deployed ...</td>\n",
       "      <td>1381281033</td>\n",
       "      <td>PHID-USER-ydswvwhh5pm4lshahjje</td>\n",
       "      <td>True</td>\n",
       "      <td>PHID-TASK-64s56xzrc22ustp2z7wx</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-10-09 01:10:33+00:00</td>\n",
       "      <td>710</td>\n",
       "      <td>Please re-open if it still occurs for you afte...</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Please re-open if it still occurs for you afte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>VisualEditor: [Regression] \"More\" menu gets sh...</td>\n",
       "      <td>Change 88226 merged by Robmoen:\\nThe amazing m...</td>\n",
       "      <td>1381274919</td>\n",
       "      <td>PHID-USER-idceizaw6elwiwm5xshb</td>\n",
       "      <td>True</td>\n",
       "      <td>PHID-TASK-64s56xzrc22ustp2z7wx</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-10-08 23:28:39+00:00</td>\n",
       "      <td>711</td>\n",
       "      <td>Change 88226 merged by Robmoen:\\nThe amazing m...</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Change 88226 merged by Robmoen:\\nThe amazing m...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32483</th>\n",
       "      <td>Implement minification for SVG files in Resour...</td>\n",
       "      <td>I think we can apply the same as PNG files.\\n\\...</td>\n",
       "      <td>1362551377</td>\n",
       "      <td>PHID-USER-sai77mtxmpqnm6pycyvz</td>\n",
       "      <td>False</td>\n",
       "      <td>PHID-TASK-s5at4xy2rbu4kz3beaqh</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-03-06 06:29:37+00:00</td>\n",
       "      <td>32338</td>\n",
       "      <td>I know SVG is text where PNG is not, but unles...</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>I know SVG is text where PNG is not, but unles...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32484</th>\n",
       "      <td>Implement minification for SVG files in Resour...</td>\n",
       "      <td>I think we can apply the same as PNG files.\\n\\...</td>\n",
       "      <td>1362551377</td>\n",
       "      <td>PHID-USER-sai77mtxmpqnm6pycyvz</td>\n",
       "      <td>False</td>\n",
       "      <td>PHID-TASK-s5at4xy2rbu4kz3beaqh</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-03-06 06:29:37+00:00</td>\n",
       "      <td>32338</td>\n",
       "      <td>As for solving this bug, if we want to minify ...</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>As for solving this bug, if we want to minify ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32485</th>\n",
       "      <td>Implement minification for SVG files in Resour...</td>\n",
       "      <td>I think we can apply the same as PNG files.\\n\\...</td>\n",
       "      <td>1362551377</td>\n",
       "      <td>PHID-USER-sai77mtxmpqnm6pycyvz</td>\n",
       "      <td>False</td>\n",
       "      <td>PHID-TASK-s5at4xy2rbu4kz3beaqh</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-03-06 06:29:37+00:00</td>\n",
       "      <td>32338</td>\n",
       "      <td>fonts), that would get quite tricky.</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>fonts), that would get quite tricky.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32486</th>\n",
       "      <td>Implement minification for SVG files in Resour...</td>\n",
       "      <td>I think we can apply the same as PNG files.\\n\\...</td>\n",
       "      <td>1362551377</td>\n",
       "      <td>PHID-USER-sai77mtxmpqnm6pycyvz</td>\n",
       "      <td>False</td>\n",
       "      <td>PHID-TASK-s5at4xy2rbu4kz3beaqh</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-03-06 06:29:37+00:00</td>\n",
       "      <td>32338</td>\n",
       "      <td>They'd need a url to be served from.</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>They'd need a url to be served from.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32487</th>\n",
       "      <td>Implement minification for SVG files in Resour...</td>\n",
       "      <td>I think we can apply the same as PNG files.\\n\\...</td>\n",
       "      <td>1362551377</td>\n",
       "      <td>PHID-USER-sai77mtxmpqnm6pycyvz</td>\n",
       "      <td>False</td>\n",
       "      <td>PHID-TASK-s5at4xy2rbu4kz3beaqh</td>\n",
       "      <td>task_subcomment</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013-03-06 06:29:37+00:00</td>\n",
       "      <td>32338</td>\n",
       "      <td>As for SVGs we embed in CSS (SVGs for icons li...</td>\n",
       "      <td>7</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>As for SVGs we embed in CSS (SVGs for icons li...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32488 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              task_title  \\\n",
       "0      VisualEditor: [Regression] \"More\" menu gets sh...   \n",
       "1      VisualEditor: [Regression] \"More\" menu gets sh...   \n",
       "2      VisualEditor: [Regression] \"More\" menu gets sh...   \n",
       "3      VisualEditor: [Regression] \"More\" menu gets sh...   \n",
       "4      VisualEditor: [Regression] \"More\" menu gets sh...   \n",
       "...                                                  ...   \n",
       "32483  Implement minification for SVG files in Resour...   \n",
       "32484  Implement minification for SVG files in Resour...   \n",
       "32485  Implement minification for SVG files in Resour...   \n",
       "32486  Implement minification for SVG files in Resour...   \n",
       "32487  Implement minification for SVG files in Resour...   \n",
       "\n",
       "                                            comment_text  date_created  \\\n",
       "0      Tested on both the Italian and the English Wik...    1380976920   \n",
       "1      Tested on both the Italian and the English Wik...    1380976920   \n",
       "2      Note that this is fixed and has been deployed ...    1381281033   \n",
       "3      Note that this is fixed and has been deployed ...    1381281033   \n",
       "4      Change 88226 merged by Robmoen:\\nThe amazing m...    1381274919   \n",
       "...                                                  ...           ...   \n",
       "32483  I think we can apply the same as PNG files.\\n\\...    1362551377   \n",
       "32484  I think we can apply the same as PNG files.\\n\\...    1362551377   \n",
       "32485  I think we can apply the same as PNG files.\\n\\...    1362551377   \n",
       "32486  I think we can apply the same as PNG files.\\n\\...    1362551377   \n",
       "32487  I think we can apply the same as PNG files.\\n\\...    1362551377   \n",
       "\n",
       "                           AuthorPHID  WMFaffil  \\\n",
       "0      PHID-USER-wil4b5lylrvf3krixlkl      True   \n",
       "1      PHID-USER-wil4b5lylrvf3krixlkl      True   \n",
       "2      PHID-USER-ydswvwhh5pm4lshahjje      True   \n",
       "3      PHID-USER-ydswvwhh5pm4lshahjje      True   \n",
       "4      PHID-USER-idceizaw6elwiwm5xshb      True   \n",
       "...                               ...       ...   \n",
       "32483  PHID-USER-sai77mtxmpqnm6pycyvz     False   \n",
       "32484  PHID-USER-sai77mtxmpqnm6pycyvz     False   \n",
       "32485  PHID-USER-sai77mtxmpqnm6pycyvz     False   \n",
       "32486  PHID-USER-sai77mtxmpqnm6pycyvz     False   \n",
       "32487  PHID-USER-sai77mtxmpqnm6pycyvz     False   \n",
       "\n",
       "                             TaskPHID      comment_type    status  \\\n",
       "0      PHID-TASK-64s56xzrc22ustp2z7wx  task_description  resolved   \n",
       "1      PHID-TASK-64s56xzrc22ustp2z7wx  task_description  resolved   \n",
       "2      PHID-TASK-64s56xzrc22ustp2z7wx   task_subcomment       NaN   \n",
       "3      PHID-TASK-64s56xzrc22ustp2z7wx   task_subcomment       NaN   \n",
       "4      PHID-TASK-64s56xzrc22ustp2z7wx   task_subcomment       NaN   \n",
       "...                               ...               ...       ...   \n",
       "32483  PHID-TASK-s5at4xy2rbu4kz3beaqh   task_subcomment       NaN   \n",
       "32484  PHID-TASK-s5at4xy2rbu4kz3beaqh   task_subcomment       NaN   \n",
       "32485  PHID-TASK-s5at4xy2rbu4kz3beaqh   task_subcomment       NaN   \n",
       "32486  PHID-TASK-s5at4xy2rbu4kz3beaqh   task_subcomment       NaN   \n",
       "32487  PHID-TASK-s5at4xy2rbu4kz3beaqh   task_subcomment       NaN   \n",
       "\n",
       "                      timestamp  comment_id  \\\n",
       "0     2013-10-05 12:42:00+00:00         709   \n",
       "1     2013-10-05 12:42:00+00:00         709   \n",
       "2     2013-10-09 01:10:33+00:00         710   \n",
       "3     2013-10-09 01:10:33+00:00         710   \n",
       "4     2013-10-08 23:28:39+00:00         711   \n",
       "...                         ...         ...   \n",
       "32483 2013-03-06 06:29:37+00:00       32338   \n",
       "32484 2013-03-06 06:29:37+00:00       32338   \n",
       "32485 2013-03-06 06:29:37+00:00       32338   \n",
       "32486 2013-03-06 06:29:37+00:00       32338   \n",
       "32487 2013-03-06 06:29:37+00:00       32338   \n",
       "\n",
       "                                                sentence  sentence_id  \\\n",
       "0      Tested on both the Italian and the English Wik...            1   \n",
       "1      --------------------------\\n**Version**: unspe...            2   \n",
       "2      Note that this is fixed and has been deployed ...            1   \n",
       "3      Please re-open if it still occurs for you afte...            2   \n",
       "4      Change 88226 merged by Robmoen:\\nThe amazing m...            1   \n",
       "...                                                  ...          ...   \n",
       "32483  I know SVG is text where PNG is not, but unles...            3   \n",
       "32484  As for solving this bug, if we want to minify ...            4   \n",
       "32485               fonts), that would get quite tricky.            5   \n",
       "32486               They'd need a url to be served from.            6   \n",
       "32487  As for SVGs we embed in CSS (SVGs for icons li...            7   \n",
       "\n",
       "       has_ref  has_bot_ref                                    sentence_no_url  \n",
       "0        False        False  Tested on both the Italian and the English Wik...  \n",
       "1        False        False  --------------------------\\n**Version**: unspe...  \n",
       "2        False        False  Note that this is fixed and has been deployed ...  \n",
       "3        False        False  Please re-open if it still occurs for you afte...  \n",
       "4        False        False  Change 88226 merged by Robmoen:\\nThe amazing m...  \n",
       "...        ...          ...                                                ...  \n",
       "32483    False        False  I know SVG is text where PNG is not, but unles...  \n",
       "32484    False        False  As for solving this bug, if we want to minify ...  \n",
       "32485    False        False               fonts), that would get quite tricky.  \n",
       "32486    False        False               They'd need a url to be served from.  \n",
       "32487     True        False  As for SVGs we embed in CSS (SVGs for icons li...  \n",
       "\n",
       "[32488 rows x 15 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_level_data['sentence_no_url'] = sentence_level_data['sentence'].apply(\n",
    "    lambda x: remove_urls(str(x)) if x else []\n",
    ")\n",
    "sentence_level_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "acb87a1a-c3e0-4d3f-8450-e2af96150e94",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_dependency_tree(sentence):\n",
    "    doc = nlp(sentence)\n",
    "    return [(token.text, token.dep_, token.head.text, token.ancestors, token.subtree, token.children) for token in doc]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b67c136e-16c4-4002-a2d6-f92c88252baf",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_sentence_level_data = sentence_level_data[sentence_level_data['has_ref'] == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f749706a-f2bb-42e3-aae5-3876b00c48ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19414/2706376531.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  filtered_sentence_level_data['dependency_tree'] = filtered_sentence_level_data['sentence'].apply(extract_dependency_tree)\n"
     ]
    }
   ],
   "source": [
    "filtered_sentence_level_data['dependency_tree'] = filtered_sentence_level_data['sentence'].apply(extract_dependency_tree)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "82c48463-5a90-4105-9ee9-5763d0b1e35b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dependency_relations = []\n",
    "\n",
    "for index, row in filtered_sentence_level_data.iterrows():\n",
    "    sentence = row['sentence']\n",
    "    timestamp = row['timestamp']\n",
    "    comment_id = row['comment_id']\n",
    "    sentence_id = row['sentence_id']\n",
    "    WMFaffil = row['WMFaffil']\n",
    "    \n",
    "    for token, dep, head, ancestors, subtree, children in row['dependency_tree']:\n",
    "        if re.search(r'\\b(visualeditor|VE|ve|VisualEditor)\\b', token, re.IGNORECASE):\n",
    "            dependency_relations.append({\n",
    "                'sentence_id': sentence_id,\n",
    "                'comment_id': comment_id,\n",
    "                'timestamp': timestamp,\n",
    "                'wmfAffil':WMFaffil,\n",
    "                'token': token,\n",
    "                'dependency': dep,\n",
    "                'head': head,\n",
    "                'depth': len(list(ancestors)), \n",
    "                'children': len(list(children)) \n",
    "            })\n",
    "            \n",
    "dependency_relations_df = pd.DataFrame(dependency_relations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "60d7d808-0211-41b3-8dfa-0143dbba94bd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>comment_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>wmfAffil</th>\n",
       "      <th>token</th>\n",
       "      <th>dependency</th>\n",
       "      <th>head</th>\n",
       "      <th>depth</th>\n",
       "      <th>children</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>723</td>\n",
       "      <td>2013-10-06 10:36:55+00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>’ve</td>\n",
       "      <td>aux</td>\n",
       "      <td>attached</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>725</td>\n",
       "      <td>2013-10-03 20:15:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>ve.base</td>\n",
       "      <td>pobj</td>\n",
       "      <td>for</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>725</td>\n",
       "      <td>2013-10-03 20:15:00+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>VisualEditor</td>\n",
       "      <td>nsubjpass</td>\n",
       "      <td>broken</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>730</td>\n",
       "      <td>2013-10-03 20:32:16+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>ve.base</td>\n",
       "      <td>compound</td>\n",
       "      <td>request</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>730</td>\n",
       "      <td>2013-10-03 20:32:16+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>ve.core</td>\n",
       "      <td>compound</td>\n",
       "      <td>request</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5430</th>\n",
       "      <td>2</td>\n",
       "      <td>32259</td>\n",
       "      <td>2013-10-06 23:34:50+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>VE</td>\n",
       "      <td>pobj</td>\n",
       "      <td>in</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5431</th>\n",
       "      <td>1</td>\n",
       "      <td>32324</td>\n",
       "      <td>2012-12-06 18:16:58+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>'ve</td>\n",
       "      <td>aux</td>\n",
       "      <td>determined</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5432</th>\n",
       "      <td>1</td>\n",
       "      <td>32324</td>\n",
       "      <td>2012-12-06 18:16:58+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>ve.ce</td>\n",
       "      <td>pobj</td>\n",
       "      <td>within</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5433</th>\n",
       "      <td>2</td>\n",
       "      <td>32330</td>\n",
       "      <td>2013-04-28 17:25:19+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>VE</td>\n",
       "      <td>dobj</td>\n",
       "      <td>enabling</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5434</th>\n",
       "      <td>7</td>\n",
       "      <td>32338</td>\n",
       "      <td>2013-03-06 06:29:37+00:00</td>\n",
       "      <td>False</td>\n",
       "      <td>VisualEditor</td>\n",
       "      <td>pobj</td>\n",
       "      <td>in</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5435 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      sentence_id  comment_id                 timestamp  wmfAffil  \\\n",
       "0               2         723 2013-10-06 10:36:55+00:00      True   \n",
       "1               1         725 2013-10-03 20:15:00+00:00     False   \n",
       "2               1         725 2013-10-03 20:15:00+00:00     False   \n",
       "3               1         730 2013-10-03 20:32:16+00:00     False   \n",
       "4               2         730 2013-10-03 20:32:16+00:00     False   \n",
       "...           ...         ...                       ...       ...   \n",
       "5430            2       32259 2013-10-06 23:34:50+00:00     False   \n",
       "5431            1       32324 2012-12-06 18:16:58+00:00     False   \n",
       "5432            1       32324 2012-12-06 18:16:58+00:00     False   \n",
       "5433            2       32330 2013-04-28 17:25:19+00:00     False   \n",
       "5434            7       32338 2013-03-06 06:29:37+00:00     False   \n",
       "\n",
       "             token dependency        head  depth  children  \n",
       "0              ’ve        aux    attached      1         0  \n",
       "1          ve.base       pobj         for      3         1  \n",
       "2     VisualEditor  nsubjpass      broken      1         0  \n",
       "3          ve.base   compound     request      2         0  \n",
       "4          ve.core   compound     request      3         0  \n",
       "...            ...        ...         ...    ...       ...  \n",
       "5430            VE       pobj          in      5         0  \n",
       "5431           've        aux  determined      1         0  \n",
       "5432         ve.ce       pobj      within      3         0  \n",
       "5433            VE       dobj    enabling      6         0  \n",
       "5434  VisualEditor       pobj          in      4         0  \n",
       "\n",
       "[5435 rows x 9 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dependency_relations_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "70178cd5-c3f9-43e4-b859-99f655b07048",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19414/3477839074.py:2: UserWarning: This pattern is interpreted as a regular expression, and has match groups. To actually get the groups, use str.extract.\n",
      "  filtered_dependencies = dependency_relations_df[dependency_relations_df['token'].str.contains(pattern, regex=True)]\n"
     ]
    }
   ],
   "source": [
    "pattern = r'\\b(ve|VE|visualeditor|VisualEditor)\\b'\n",
    "filtered_dependencies = dependency_relations_df[dependency_relations_df['token'].str.contains(pattern, regex=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "37e1e207-5462-46f3-ae56-85b1f231f44c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from matplotlib.gridspec import GridSpec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80f2d8b9-6363-429c-bbe8-4cc4bafa39f5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bdb36a7b-a9b1-4b99-adc4-65cefd78db7c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_22147/2479837711.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
      "/tmp/ipykernel_22147/2479837711.py:17: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_22147/2479837711.py:17: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_22147/2479837711.py:23: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 380700 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 8))\n",
    "gs = GridSpec(2, 1, height_ratios=[5, 1])\n",
    "\n",
    "# Main plot: Dependency groups by timestamp\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.scatterplot(data=filtered_dependencies, x='timestamp', y='dependency', hue='wmfAffil', style='dependency', markers=True, s=100, ax=ax0)\n",
    "ax0.set_title('Dependency Group for \"VisualEditor\" by Timestamp (w/o URLs)')\n",
    "ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
    "#ax0.set_xticklabels([])\n",
    "ax0.set_xlabel('')\n",
    "ax0.set_ylabel('Dependency')\n",
    "#ax0.legend(title='Dependency', bbox_to_anchor=(1, 1), loc='upper left')\n",
    "ax0.legend().set_visible(False)\n",
    "\n",
    "# Secondary plot: Overall count of sentences over time\n",
    "ax1 = plt.subplot(gs[1], sharex=ax0)\n",
    "filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n",
    "weekly_comment_counts = filtered_dependencies.groupby('week')['comment_id'].nunique()\n",
    "sns.lineplot(x=weekly_comment_counts.index, y=weekly_comment_counts.values, ax=ax1, marker='o')\n",
    "ax1.set_title('Count of Sentences Discussing VE (Grouped by Week)')\n",
    "ax1.set_xlabel('Timestamp')\n",
    "ax1.set_ylabel('Count')\n",
    "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dae8ebc0-05f3-48c1-946f-fb70e074e5ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_19414/3977280642.py:3: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  task_phab_df['week'] = task_phab_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_19414/3977280642.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  task_phab_df['week'] = task_phab_df['timestamp'].dt.to_period('W').dt.start_time\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 120124 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "9816"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "task_phab_df = filtered_phab_df[filtered_phab_df['comment_type']==\"task_description\"]\n",
    "task_phab_df['week'] = task_phab_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "unique_taskPHIDs = task_phab_df.groupby('week')['TaskPHID'].nunique()\n",
    "\n",
    "wmf_task_phab_df = task_phab_df[task_phab_df['WMFaffil'] == True]\n",
    "wmf_tasks = wmf_task_phab_df.groupby('week')['TaskPHID'].nunique()\n",
    "\n",
    "other_task_phab_df = task_phab_df[task_phab_df['WMFaffil'] != True]\n",
    "other_tasks = other_task_phab_df.groupby('week')['TaskPHID'].nunique()\n",
    "\n",
    "sns.lineplot(x=unique_taskPHIDs.index, y=unique_taskPHIDs.values, color='black', marker='o')\n",
    "sns.lineplot(x=wmf_tasks.index, y=wmf_tasks.values, color='#c7756a', label='WMF-affiliated authors', marker='x')\n",
    "sns.lineplot(x=other_tasks.index, y=other_tasks.values, color='#5da2d8', label='Nonaffiliated authors', marker='x')\n",
    "plt.title('Task Descriptions (Task head)')\n",
    "plt.xlabel('Timestamp')\n",
    "plt.ylabel('Unique taskPHIDs')\n",
    "plt.xticks(rotation=45)\n",
    "plt.grid(True)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "#len(filtered_phab_df[filtered_phab_df['TaskPHID'].isin(task_phab_df['TaskPHID'])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1869ecbb-7579-43a9-92f3-8fa3a06ee331",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_56151/825633528.py:14: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_56151/825633528.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 208260 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 8))\n",
    "gs = GridSpec(2, 1, height_ratios=[6, 6])\n",
    "\n",
    "# Main plot: Token depth by timestamp\n",
    "'''\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.scatterplot(data=filtered_dependencies, x='timestamp', y='dependency', hue='wmfAffil', style='dependency', markers=True, s=100, ax=ax0)\n",
    "ax0.set_title('VE Depth by Timestamp w/o URLS')\n",
    "ax0.set_xlabel('')\n",
    "ax0.set_ylabel('Dependency Type')\n",
    "ax0.legend().set_visible(False)\n",
    "'''\n",
    "# Calculate the median depth over time\n",
    "filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n",
    "median_depth = filtered_dependencies.groupby('week')['depth'].median().reset_index()\n",
    "\n",
    "wmf_filtered_dependencies = filtered_dependencies[filtered_dependencies['wmfAffil'] == True]\n",
    "wmf_median_depth = wmf_filtered_dependencies.groupby('week')['depth'].median().reset_index()\n",
    "\n",
    "other_filtered_dependencies = filtered_dependencies[filtered_dependencies['wmfAffil'] != True]\n",
    "other_median_depth = other_filtered_dependencies.groupby('week')['depth'].median().reset_index()\n",
    "\n",
    "# Plot the median depth over time\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.lineplot(data=median_depth, x='week', y='depth', ax=ax0, color='black', label='Median Depth', marker='o')\n",
    "sns.lineplot(data=wmf_median_depth, x='week', y='depth', ax=ax0, color='#c7756a', label='WMF-affiliated authors', marker='x')\n",
    "sns.lineplot(data=other_median_depth, x='week', y='depth', ax=ax0, color='#5da2d8', label='Nonaffiliated authors', marker='x')\n",
    "ax0.set_title('Median Depth of VE in Phabricator Sentence Dependency Trees')\n",
    "ax0.set_ylabel('Median Depth')\n",
    "ax0.set_xlabel('')\n",
    "\n",
    "# Secondary plot: Overall count of sentences over time\n",
    "ax2 = plt.subplot(gs[1], sharex=ax0)\n",
    "#filtered_dependencies['week'] = filtered_dependencies['timestamp'].dt.to_period('W').dt.start_time\n",
    "weekly_comment_counts = filtered_dependencies.groupby('week')['comment_id'].nunique()\n",
    "sns.lineplot(x=weekly_comment_counts.index, y=weekly_comment_counts.values, color='black', label='Total', ax=ax2, marker='o')\n",
    "wmf_weekly_comment_counts = wmf_filtered_dependencies.groupby('week')['comment_id'].nunique()\n",
    "sns.lineplot(x=wmf_weekly_comment_counts.index, y=wmf_weekly_comment_counts.values, color='#c7756a', label='WMF-affiliated authors', ax=ax2, marker='x')\n",
    "other_weekly_comment_counts = other_filtered_dependencies.groupby('week')['comment_id'].nunique()\n",
    "sns.lineplot(x=other_weekly_comment_counts.index, y=other_weekly_comment_counts.values, color='#5da2d8', label='Nonaffiliated authors', ax=ax2, marker='x')\n",
    "ax2.set_title('Count of Phabricator Comments Discussing VE (Grouped by Week)')\n",
    "ax2.set_xlabel('Timestamp')\n",
    "ax2.set_ylabel('Count')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "plt.savefig('030525_ve_phab_comments.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "e3fd6f81-7cad-4a2c-b5e8-a1f906d32b87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_97550/2055267856.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  bot_filtered_sentence_level_data['dependency_tree'] = bot_filtered_sentence_level_data['sentence_no_url'].apply(extract_dependency_tree)\n"
     ]
    }
   ],
   "source": [
    "bot_filtered_sentence_level_data = sentence_level_data[sentence_level_data['has_bot_ref'] == True]\n",
    "bot_filtered_sentence_level_data['dependency_tree'] = bot_filtered_sentence_level_data['sentence_no_url'].apply(extract_dependency_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9ef307b2-b200-47b1-ade9-81f8c81847f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "bot_dependency_relations = []\n",
    "\n",
    "for index, row in bot_filtered_sentence_level_data.iterrows():\n",
    "    sentence = row['sentence']\n",
    "    timestamp = row['timestamp']\n",
    "    comment_id = row['comment_id']\n",
    "    sentence_id = row['sentence_id']\n",
    "    \n",
    "    for token, dep, head, ancestors, subtree, children in row['dependency_tree']:\n",
    "        if re.search(r'\\b(bots|scripts|gadgets)\\b', token, re.IGNORECASE):\n",
    "            bot_dependency_relations.append({\n",
    "                'sentence_id': sentence_id,\n",
    "                'comment_id': comment_id,\n",
    "                'timestamp': timestamp,\n",
    "                'token': token,\n",
    "                'dependency': dep,\n",
    "                'head': head, \n",
    "                'depth': len(list(ancestors)), \n",
    "                'children': len(list(children)) \n",
    "            })\n",
    "            \n",
    "bot_dependency_relations_df = pd.DataFrame(bot_dependency_relations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7cb8c88d-d657-4555-8af1-47f9bed2c9ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>comment_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>token</th>\n",
       "      <th>dependency</th>\n",
       "      <th>head</th>\n",
       "      <th>depth</th>\n",
       "      <th>children</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>730</td>\n",
       "      <td>2013-10-03 20:32:16+00:00</td>\n",
       "      <td>scripts</td>\n",
       "      <td>pobj</td>\n",
       "      <td>in</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3948</td>\n",
       "      <td>2013-10-25 23:44:17+00:00</td>\n",
       "      <td>gadgets</td>\n",
       "      <td>dobj</td>\n",
       "      <td>load</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3948</td>\n",
       "      <td>2013-10-25 23:44:17+00:00</td>\n",
       "      <td>gadgets</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>reach</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>3963</td>\n",
       "      <td>2013-10-04 15:10:00+00:00</td>\n",
       "      <td>load.php?debug=false&amp;lang=en&amp;modules=jquery%2C...</td>\n",
       "      <td>compound</td>\n",
       "      <td>Exception</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3963</td>\n",
       "      <td>2013-10-04 15:10:00+00:00</td>\n",
       "      <td>load.php?debug=false&amp;lang=en&amp;modules=jquery%2C...</td>\n",
       "      <td>compound</td>\n",
       "      <td>Exception</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>2</td>\n",
       "      <td>29956</td>\n",
       "      <td>2013-03-15 11:27:23+00:00</td>\n",
       "      <td>bots</td>\n",
       "      <td>nsubjpass</td>\n",
       "      <td>voiced</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>6</td>\n",
       "      <td>30140</td>\n",
       "      <td>2013-06-20 20:09:44+00:00</td>\n",
       "      <td>scripts</td>\n",
       "      <td>pobj</td>\n",
       "      <td>For</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>3</td>\n",
       "      <td>31749</td>\n",
       "      <td>2013-08-07 13:35:56+00:00</td>\n",
       "      <td>scripts</td>\n",
       "      <td>attr</td>\n",
       "      <td>be</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>6</td>\n",
       "      <td>31907</td>\n",
       "      <td>2013-06-30 19:20:13+00:00</td>\n",
       "      <td>gadgets</td>\n",
       "      <td>dobj</td>\n",
       "      <td>support</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>8</td>\n",
       "      <td>31907</td>\n",
       "      <td>2013-06-30 19:20:13+00:00</td>\n",
       "      <td>gadgets</td>\n",
       "      <td>dobj</td>\n",
       "      <td>support</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>180 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sentence_id  comment_id                 timestamp  \\\n",
       "0              5         730 2013-10-03 20:32:16+00:00   \n",
       "1              1        3948 2013-10-25 23:44:17+00:00   \n",
       "2              2        3948 2013-10-25 23:44:17+00:00   \n",
       "3              4        3963 2013-10-04 15:10:00+00:00   \n",
       "4              4        3963 2013-10-04 15:10:00+00:00   \n",
       "..           ...         ...                       ...   \n",
       "175            2       29956 2013-03-15 11:27:23+00:00   \n",
       "176            6       30140 2013-06-20 20:09:44+00:00   \n",
       "177            3       31749 2013-08-07 13:35:56+00:00   \n",
       "178            6       31907 2013-06-30 19:20:13+00:00   \n",
       "179            8       31907 2013-06-30 19:20:13+00:00   \n",
       "\n",
       "                                                 token dependency       head  \\\n",
       "0                                              scripts       pobj         in   \n",
       "1                                              gadgets       dobj       load   \n",
       "2                                              gadgets      nsubj      reach   \n",
       "3    load.php?debug=false&lang=en&modules=jquery%2C...   compound  Exception   \n",
       "4    load.php?debug=false&lang=en&modules=jquery%2C...   compound  Exception   \n",
       "..                                                 ...        ...        ...   \n",
       "175                                               bots  nsubjpass     voiced   \n",
       "176                                            scripts       pobj        For   \n",
       "177                                            scripts       attr         be   \n",
       "178                                            gadgets       dobj    support   \n",
       "179                                            gadgets       dobj    support   \n",
       "\n",
       "     depth  children  \n",
       "0        3         2  \n",
       "1        3         1  \n",
       "2        1         0  \n",
       "3        4         1  \n",
       "4        4         2  \n",
       "..     ...       ...  \n",
       "175      1         2  \n",
       "176      2         2  \n",
       "177      3         4  \n",
       "178      3         0  \n",
       "179      3         0  \n",
       "\n",
       "[180 rows x 8 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bot_dependency_relations_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c8b10c32-9c28-4151-af29-a1504eb29b76",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_83107/353490988.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
      "/tmp/ipykernel_83107/353490988.py:17: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  bot_dependency_relations_df['week'] = bot_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_83107/353490988.py:23: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 127628 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 8))\n",
    "gs = GridSpec(2, 1, height_ratios=[5, 1])\n",
    "\n",
    "# Main plot: Dependency groups by timestamp\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.scatterplot(data=bot_dependency_relations_df, x='timestamp', y='dependency', hue='dependency', style='dependency', markers=True, s=100, ax=ax0)\n",
    "ax0.set_title('Dependency Group for \"bots|gadgets|scripts\" by Timestamp (w/o URLS)')\n",
    "ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
    "#ax0.set_xticklabels([])\n",
    "ax0.set_xlabel('')\n",
    "ax0.set_ylabel('Dependency')\n",
    "#ax0.legend(title='Dependency', bbox_to_anchor=(1, 1), loc='upper left')\n",
    "ax0.legend().set_visible(False)\n",
    "\n",
    "# Secondary plot: Overall count of sentences over time\n",
    "ax1 = plt.subplot(gs[1], sharex=ax0)\n",
    "bot_dependency_relations_df['week'] = bot_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "weekly_counts = bot_dependency_relations_df['week'].value_counts().sort_index()\n",
    "sns.lineplot(x=weekly_counts.index, y=weekly_counts.values, ax=ax1, marker='o')\n",
    "ax1.set_title('Count of Sentences Discussing bots|gadgets|scripts (Grouped by Week)')\n",
    "ax1.set_xlabel('Timestamp')\n",
    "ax1.set_ylabel('Count')\n",
    "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4808a67b-0ab1-4c02-924e-0f81bcacd64e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_97550/331759571.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
      "/tmp/ipykernel_97550/331759571.py:14: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  bot_dependency_relations_df['week'] = bot_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_97550/331759571.py:23: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  bot_dependency_relations_df['week'] = bot_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_97550/331759571.py:29: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 165244 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 8))\n",
    "gs = GridSpec(2, 1, height_ratios=[5, 1])\n",
    "\n",
    "# Main plot: Token depth by timestamp\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.scatterplot(data=bot_dependency_relations_df, x='timestamp', y='depth', hue='dependency', style='dependency', markers=True, s=100, ax=ax0)\n",
    "ax0.set_title('Bot Depth by Timestamp w/o URLS')\n",
    "ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
    "ax0.set_xlabel('')\n",
    "ax0.set_ylabel('Depth')\n",
    "#ax0.legend().set_visible(False)\n",
    "\n",
    "# Calculate the median depth over time\n",
    "bot_dependency_relations_df['week'] = bot_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "median_depth = bot_dependency_relations_df.groupby('week')['depth'].median().reset_index()\n",
    "\n",
    "# Plot the median depth over time\n",
    "sns.lineplot(data=median_depth, x='week', y='depth', ax=ax0, color='black', label='Median Depth', marker='o')\n",
    "ax0.legend(title='Legend', bbox_to_anchor=(1.05, 1), loc='upper left')\n",
    "\n",
    "# Secondary plot: Overall count of sentences over time\n",
    "ax1 = plt.subplot(gs[1], sharex=ax0)\n",
    "bot_dependency_relations_df['week'] = bot_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "weekly_counts = bot_dependency_relations_df['week'].value_counts().sort_index()\n",
    "sns.lineplot(x=weekly_counts.index, y=weekly_counts.values, ax=ax1, marker='o')\n",
    "ax1.set_title('Count of Sentences Discussing Bots (Grouped by Week)')\n",
    "ax1.set_xlabel('Timestamp')\n",
    "ax1.set_ylabel('Count')\n",
    "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a1c1a934-f764-4e22-a7c1-d2b10e237db8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_level_data['dependency_tree'] = sentence_level_data['sentence_no_url'].apply(extract_dependency_tree)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "df6960b5-8bab-49c7-b101-a19864472a63",
   "metadata": {},
   "outputs": [],
   "source": [
    "wmf_dependency_relations = []\n",
    "\n",
    "for index, row in sentence_level_data.iterrows():\n",
    "    sentence = row['sentence']\n",
    "    timestamp = row['timestamp']\n",
    "    comment_id = row['comment_id']\n",
    "    sentence_id = row['sentence_id']\n",
    "    \n",
    "    for token, dep, head, ancestors, subtree, children in row['dependency_tree']:\n",
    "        if re.search(r'\\b(wmf|Wikimedia|Foundation|staff| wikimedia foundation)\\b', token, re.IGNORECASE):\n",
    "            wmf_dependency_relations.append({\n",
    "                'sentence_id': sentence_id,\n",
    "                'comment_id': comment_id,\n",
    "                'timestamp': timestamp,\n",
    "                'token': token,\n",
    "                'dependency': dep,\n",
    "                'head': head, \n",
    "                'depth': len(list(ancestors)), \n",
    "                'children': len(list(children)) \n",
    "            })\n",
    "            \n",
    "wmf_dependency_relations_df = pd.DataFrame(wmf_dependency_relations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "402ce764-2332-4cb1-8c36-e4d88cebae64",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>comment_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>token</th>\n",
       "      <th>dependency</th>\n",
       "      <th>head</th>\n",
       "      <th>depth</th>\n",
       "      <th>children</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>747</td>\n",
       "      <td>2013-09-28 15:44:00+00:00</td>\n",
       "      <td>WMF</td>\n",
       "      <td>nsubj</td>\n",
       "      <td>comes</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>750</td>\n",
       "      <td>2013-09-30 23:29:57+00:00</td>\n",
       "      <td>Wikimedia</td>\n",
       "      <td>compound</td>\n",
       "      <td>wiki</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>752</td>\n",
       "      <td>2013-09-30 07:16:14+00:00</td>\n",
       "      <td>WMF</td>\n",
       "      <td>appos</td>\n",
       "      <td>You</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>753</td>\n",
       "      <td>2013-09-28 19:11:06+00:00</td>\n",
       "      <td>Wikimedia</td>\n",
       "      <td>compound</td>\n",
       "      <td>wiki</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1050</td>\n",
       "      <td>2013-06-27 00:46:02+00:00</td>\n",
       "      <td>WMF</td>\n",
       "      <td>pobj</td>\n",
       "      <td>at</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>3</td>\n",
       "      <td>32254</td>\n",
       "      <td>2013-10-07 02:30:30+00:00</td>\n",
       "      <td>Wikimedia</td>\n",
       "      <td>compound</td>\n",
       "      <td>wikis</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432</th>\n",
       "      <td>1</td>\n",
       "      <td>32256</td>\n",
       "      <td>2013-10-07 00:58:28+00:00</td>\n",
       "      <td>wmf</td>\n",
       "      <td>compound</td>\n",
       "      <td>wiki</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>433</th>\n",
       "      <td>1</td>\n",
       "      <td>32257</td>\n",
       "      <td>2013-10-07 00:44:40+00:00</td>\n",
       "      <td>WMF</td>\n",
       "      <td>compound</td>\n",
       "      <td>wiki</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>2</td>\n",
       "      <td>32258</td>\n",
       "      <td>2013-10-07 00:38:54+00:00</td>\n",
       "      <td>WMF</td>\n",
       "      <td>amod</td>\n",
       "      <td>wikis</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>2</td>\n",
       "      <td>32258</td>\n",
       "      <td>2013-10-07 00:38:54+00:00</td>\n",
       "      <td>WMF</td>\n",
       "      <td>compound</td>\n",
       "      <td>wiki</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>436 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     sentence_id  comment_id                 timestamp      token dependency  \\\n",
       "0              2         747 2013-09-28 15:44:00+00:00        WMF      nsubj   \n",
       "1              5         750 2013-09-30 23:29:57+00:00  Wikimedia   compound   \n",
       "2              2         752 2013-09-30 07:16:14+00:00        WMF      appos   \n",
       "3              2         753 2013-09-28 19:11:06+00:00  Wikimedia   compound   \n",
       "4              2        1050 2013-06-27 00:46:02+00:00        WMF       pobj   \n",
       "..           ...         ...                       ...        ...        ...   \n",
       "431            3       32254 2013-10-07 02:30:30+00:00  Wikimedia   compound   \n",
       "432            1       32256 2013-10-07 00:58:28+00:00        wmf   compound   \n",
       "433            1       32257 2013-10-07 00:44:40+00:00        WMF   compound   \n",
       "434            2       32258 2013-10-07 00:38:54+00:00        WMF       amod   \n",
       "435            2       32258 2013-10-07 00:38:54+00:00        WMF   compound   \n",
       "\n",
       "      head  depth  children  \n",
       "0    comes      1         1  \n",
       "1     wiki      4         0  \n",
       "2      You      4         0  \n",
       "3     wiki      3         0  \n",
       "4       at      2         0  \n",
       "..     ...    ...       ...  \n",
       "431  wikis      8         0  \n",
       "432   wiki      5         0  \n",
       "433   wiki      3         0  \n",
       "434  wikis      3         0  \n",
       "435   wiki      3         0  \n",
       "\n",
       "[436 rows x 8 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wmf_dependency_relations_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "5c305993-a0c3-403f-a971-f0e527eceb5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_83107/998074097.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
      "/tmp/ipykernel_83107/998074097.py:17: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  wmf_dependency_relations_df['week'] = wmf_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_83107/998074097.py:23: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 147776 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 8))\n",
    "gs = GridSpec(2, 1, height_ratios=[5, 1])\n",
    "\n",
    "# Main plot: Dependency groups by timestamp\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.scatterplot(data=wmf_dependency_relations_df, x='timestamp', y='dependency', hue='dependency', style='dependency', markers=True, s=100, ax=ax0)\n",
    "ax0.set_title('Dependency Group for \"WMF\" by Timestamp (w/o URLS)')\n",
    "ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
    "#ax0.set_xticklabels([])\n",
    "ax0.set_xlabel('')\n",
    "ax0.set_ylabel('Dependency')\n",
    "#ax0.legend(title='Dependency', bbox_to_anchor=(1, 1), loc='upper left')\n",
    "ax0.legend().set_visible(False)\n",
    "\n",
    "# Secondary plot: Overall count of sentences over time\n",
    "ax1 = plt.subplot(gs[1], sharex=ax0)\n",
    "wmf_dependency_relations_df['week'] = wmf_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "weekly_counts = wmf_dependency_relations_df['week'].value_counts().sort_index()\n",
    "sns.lineplot(x=weekly_counts.index, y=weekly_counts.values, ax=ax1, marker='o')\n",
    "ax1.set_title('Count of Sentences Discussing \"WMF\" (Grouped by Week)')\n",
    "ax1.set_xlabel('Timestamp')\n",
    "ax1.set_ylabel('Count')\n",
    "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1fa1f33e-0a38-426e-b75b-42f2fd2cad36",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_97550/4168321775.py:8: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
      "/tmp/ipykernel_97550/4168321775.py:14: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  wmf_dependency_relations_df['week'] = wmf_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_97550/4168321775.py:23: UserWarning: Converting to PeriodArray/Index representation will drop timezone information.\n",
      "  wmf_dependency_relations_df['week'] = wmf_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
      "/tmp/ipykernel_97550/4168321775.py:29: UserWarning: set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.\n",
      "  ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 213160 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12, 8))\n",
    "gs = GridSpec(2, 1, height_ratios=[5, 1])\n",
    "\n",
    "# Main plot: Token depth by timestamp\n",
    "ax0 = plt.subplot(gs[0])\n",
    "sns.scatterplot(data=wmf_dependency_relations_df, x='timestamp', y='depth', hue='dependency', style='dependency', markers=True, s=100, ax=ax0)\n",
    "ax0.set_title('WMF Depth by Timestamp w/o URLS')\n",
    "ax0.set_xticklabels(ax0.get_xticklabels(), rotation=45)\n",
    "ax0.set_xlabel('')\n",
    "ax0.set_ylabel('Depth')\n",
    "#ax0.legend().set_visible(False)\n",
    "\n",
    "# Calculate the median depth over time\n",
    "wmf_dependency_relations_df['week'] = wmf_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "median_depth = wmf_dependency_relations_df.groupby('week')['depth'].median().reset_index()\n",
    "\n",
    "# Plot the median depth over time\n",
    "sns.lineplot(data=median_depth, x='week', y='depth', ax=ax0, color='black', label='Median Depth', marker='o')\n",
    "ax0.legend(title='Legend', bbox_to_anchor=(1.05, 1), loc='upper left')\n",
    "\n",
    "# Secondary plot: Overall count of sentences over time\n",
    "ax1 = plt.subplot(gs[1], sharex=ax0)\n",
    "wmf_dependency_relations_df['week'] = wmf_dependency_relations_df['timestamp'].dt.to_period('W').dt.start_time\n",
    "weekly_counts = wmf_dependency_relations_df['week'].value_counts().sort_index()\n",
    "sns.lineplot(x=weekly_counts.index, y=weekly_counts.values, ax=ax1, marker='o')\n",
    "ax1.set_title('Count of Sentences Discussing WMF (Grouped by Week)')\n",
    "ax1.set_xlabel('Timestamp')\n",
    "ax1.set_ylabel('Count')\n",
    "ax1.set_xticklabels(ax1.get_xticklabels(), rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.21"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
