{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_manifest = pd.read_csv('0207_readme_merged_manifest.csv')\n",
    "topic_distributions = pd.read_csv('020725_README_file_topic_distributions.csv')\n",
    "readability_scores = pd.read_csv('020725_README_readability.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4226"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_merge = readability_scores.merge(topic_distributions, on=['filename'],how=\"inner\")\n",
    "#primary_merge = first_merge.merge(readability_scores, )\n",
    "first_merge['new_filepath'] = first_merge['filename']\n",
    "#len(first_merge)\n",
    "second_merge = first_merge.merge(merged_manifest, on=['new_filepath'], how=\"inner\")\n",
    "len(second_merge)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>flesch_reading_ease</th>\n",
       "      <th>flesch_kincaid_grade</th>\n",
       "      <th>linsear_write_formula</th>\n",
       "      <th>dale_chall_readability_score</th>\n",
       "      <th>mcalpine_eflaw</th>\n",
       "      <th>reading_time</th>\n",
       "      <th>char_count</th>\n",
       "      <th>word_count</th>\n",
       "      <th>t0</th>\n",
       "      <th>...</th>\n",
       "      <th>effect</th>\n",
       "      <th>group</th>\n",
       "      <th>level</th>\n",
       "      <th>term</th>\n",
       "      <th>estimate</th>\n",
       "      <th>std.error</th>\n",
       "      <th>conf.low</th>\n",
       "      <th>conf.high</th>\n",
       "      <th>ranef_grouping</th>\n",
       "      <th>rank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>libevent_libevent_hullabaloo_README</td>\n",
       "      <td>50.36</td>\n",
       "      <td>11.4</td>\n",
       "      <td>14.625000</td>\n",
       "      <td>12.47</td>\n",
       "      <td>26.3</td>\n",
       "      <td>7.37</td>\n",
       "      <td>502</td>\n",
       "      <td>83</td>\n",
       "      <td>0.001976</td>\n",
       "      <td>...</td>\n",
       "      <td>ran_vals</td>\n",
       "      <td>project_id</td>\n",
       "      <td>libevent_libevent</td>\n",
       "      <td>before_after:week_index</td>\n",
       "      <td>0.186522</td>\n",
       "      <td>0.478091</td>\n",
       "      <td>-0.750519</td>\n",
       "      <td>1.123563</td>\n",
       "      <td>1</td>\n",
       "      <td>3229.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zaach_jison_hullabaloo_README.md</td>\n",
       "      <td>44.78</td>\n",
       "      <td>13.5</td>\n",
       "      <td>7.714286</td>\n",
       "      <td>10.70</td>\n",
       "      <td>37.2</td>\n",
       "      <td>33.05</td>\n",
       "      <td>2250</td>\n",
       "      <td>395</td>\n",
       "      <td>0.000489</td>\n",
       "      <td>...</td>\n",
       "      <td>ran_vals</td>\n",
       "      <td>project_id</td>\n",
       "      <td>zaach_jison</td>\n",
       "      <td>before_after:week_index</td>\n",
       "      <td>1.001879</td>\n",
       "      <td>0.217063</td>\n",
       "      <td>0.576443</td>\n",
       "      <td>1.427315</td>\n",
       "      <td>2</td>\n",
       "      <td>4068.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>lincolnloop_python-qrcode.git_hullabaloo_READM...</td>\n",
       "      <td>58.28</td>\n",
       "      <td>8.4</td>\n",
       "      <td>7.583333</td>\n",
       "      <td>10.76</td>\n",
       "      <td>18.3</td>\n",
       "      <td>6.82</td>\n",
       "      <td>464</td>\n",
       "      <td>78</td>\n",
       "      <td>0.002597</td>\n",
       "      <td>...</td>\n",
       "      <td>ran_vals</td>\n",
       "      <td>project_id</td>\n",
       "      <td>lincolnloop_python-qrcode.git</td>\n",
       "      <td>before_after:week_index</td>\n",
       "      <td>-0.220654</td>\n",
       "      <td>0.679910</td>\n",
       "      <td>-1.553254</td>\n",
       "      <td>1.111946</td>\n",
       "      <td>1</td>\n",
       "      <td>1467.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mati75_evilwm.git_hullabaloo_README</td>\n",
       "      <td>66.74</td>\n",
       "      <td>7.2</td>\n",
       "      <td>5.888889</td>\n",
       "      <td>8.77</td>\n",
       "      <td>18.1</td>\n",
       "      <td>56.76</td>\n",
       "      <td>3864</td>\n",
       "      <td>742</td>\n",
       "      <td>0.000266</td>\n",
       "      <td>...</td>\n",
       "      <td>ran_vals</td>\n",
       "      <td>project_id</td>\n",
       "      <td>mati75_evilwm.git</td>\n",
       "      <td>before_after:week_index</td>\n",
       "      <td>-0.293846</td>\n",
       "      <td>0.697316</td>\n",
       "      <td>-1.660560</td>\n",
       "      <td>1.072868</td>\n",
       "      <td>1</td>\n",
       "      <td>1184.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>markdown-it_markdown-it_hullabaloo_README.md</td>\n",
       "      <td>45.76</td>\n",
       "      <td>11.1</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>12.84</td>\n",
       "      <td>24.0</td>\n",
       "      <td>1.51</td>\n",
       "      <td>103</td>\n",
       "      <td>17</td>\n",
       "      <td>0.006993</td>\n",
       "      <td>...</td>\n",
       "      <td>ran_vals</td>\n",
       "      <td>project_id</td>\n",
       "      <td>markdown-it_markdown-it</td>\n",
       "      <td>before_after:week_index</td>\n",
       "      <td>0.238615</td>\n",
       "      <td>0.602856</td>\n",
       "      <td>-0.942961</td>\n",
       "      <td>1.420190</td>\n",
       "      <td>1</td>\n",
       "      <td>3459.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            filename  flesch_reading_ease  \\\n",
       "0                libevent_libevent_hullabaloo_README                50.36   \n",
       "1                   zaach_jison_hullabaloo_README.md                44.78   \n",
       "2  lincolnloop_python-qrcode.git_hullabaloo_READM...                58.28   \n",
       "3                mati75_evilwm.git_hullabaloo_README                66.74   \n",
       "4       markdown-it_markdown-it_hullabaloo_README.md                45.76   \n",
       "\n",
       "   flesch_kincaid_grade  linsear_write_formula  dale_chall_readability_score  \\\n",
       "0                  11.4              14.625000                         12.47   \n",
       "1                  13.5               7.714286                         10.70   \n",
       "2                   8.4               7.583333                         10.76   \n",
       "3                   7.2               5.888889                          8.77   \n",
       "4                  11.1              12.000000                         12.84   \n",
       "\n",
       "   mcalpine_eflaw  reading_time  char_count  word_count        t0  ...  \\\n",
       "0            26.3          7.37         502          83  0.001976  ...   \n",
       "1            37.2         33.05        2250         395  0.000489  ...   \n",
       "2            18.3          6.82         464          78  0.002597  ...   \n",
       "3            18.1         56.76        3864         742  0.000266  ...   \n",
       "4            24.0          1.51         103          17  0.006993  ...   \n",
       "\n",
       "     effect       group                          level  \\\n",
       "0  ran_vals  project_id              libevent_libevent   \n",
       "1  ran_vals  project_id                    zaach_jison   \n",
       "2  ran_vals  project_id  lincolnloop_python-qrcode.git   \n",
       "3  ran_vals  project_id              mati75_evilwm.git   \n",
       "4  ran_vals  project_id        markdown-it_markdown-it   \n",
       "\n",
       "                      term  estimate  std.error  conf.low  conf.high  \\\n",
       "0  before_after:week_index  0.186522   0.478091 -0.750519   1.123563   \n",
       "1  before_after:week_index  1.001879   0.217063  0.576443   1.427315   \n",
       "2  before_after:week_index -0.220654   0.679910 -1.553254   1.111946   \n",
       "3  before_after:week_index -0.293846   0.697316 -1.660560   1.072868   \n",
       "4  before_after:week_index  0.238615   0.602856 -0.942961   1.420190   \n",
       "\n",
       "   ranef_grouping    rank  \n",
       "0               1  3229.0  \n",
       "1               2  4068.0  \n",
       "2               1  1467.0  \n",
       "3               1  1184.0  \n",
       "4               1  3459.0  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "second_merge.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">flesch_reading_ease</th>\n",
       "      <th colspan=\"2\" halign=\"left\">reading_time</th>\n",
       "      <th colspan=\"2\" halign=\"left\">word_count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ranef_grouping</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36.425600</td>\n",
       "      <td>49.920</td>\n",
       "      <td>27.601600</td>\n",
       "      <td>10.06</td>\n",
       "      <td>294.320000</td>\n",
       "      <td>99.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47.580425</td>\n",
       "      <td>50.330</td>\n",
       "      <td>27.562204</td>\n",
       "      <td>15.09</td>\n",
       "      <td>294.790940</td>\n",
       "      <td>147.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>54.819748</td>\n",
       "      <td>52.835</td>\n",
       "      <td>31.478753</td>\n",
       "      <td>14.04</td>\n",
       "      <td>341.339329</td>\n",
       "      <td>145.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               flesch_reading_ease         reading_time         word_count  \\\n",
       "                              mean  median         mean median        mean   \n",
       "ranef_grouping                                                               \n",
       "0                        36.425600  49.920    27.601600  10.06  294.320000   \n",
       "1                        47.580425  50.330    27.562204  15.09  294.790940   \n",
       "2                        54.819748  52.835    31.478753  14.04  341.339329   \n",
       "\n",
       "                       \n",
       "               median  \n",
       "ranef_grouping         \n",
       "0                99.0  \n",
       "1               147.0  \n",
       "2               145.5  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "readability_aggregate = second_merge.groupby('ranef_grouping').agg({\n",
    "    'flesch_reading_ease' : ['mean', 'median'],\n",
    "    'reading_time' : ['mean', 'median'],\n",
    "    'word_count' : ['mean', 'median'],\n",
    "})\n",
    "\n",
    "readability_aggregate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>t0</th>\n",
       "      <th>t1</th>\n",
       "      <th>t2</th>\n",
       "      <th>t3</th>\n",
       "      <th>t4</th>\n",
       "      <th>t5</th>\n",
       "      <th>t6</th>\n",
       "      <th>t7</th>\n",
       "      <th>t8</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ranef_grouping</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.072362</td>\n",
       "      <td>0.117356</td>\n",
       "      <td>0.094708</td>\n",
       "      <td>0.055965</td>\n",
       "      <td>0.147862</td>\n",
       "      <td>0.068689</td>\n",
       "      <td>0.124238</td>\n",
       "      <td>0.056790</td>\n",
       "      <td>0.262029</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.073022</td>\n",
       "      <td>0.093757</td>\n",
       "      <td>0.102363</td>\n",
       "      <td>0.083688</td>\n",
       "      <td>0.144348</td>\n",
       "      <td>0.076305</td>\n",
       "      <td>0.093023</td>\n",
       "      <td>0.074407</td>\n",
       "      <td>0.259087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.073784</td>\n",
       "      <td>0.111356</td>\n",
       "      <td>0.076307</td>\n",
       "      <td>0.050178</td>\n",
       "      <td>0.130441</td>\n",
       "      <td>0.059123</td>\n",
       "      <td>0.093142</td>\n",
       "      <td>0.067075</td>\n",
       "      <td>0.338593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      t0        t1        t2        t3        t4        t5  \\\n",
       "                    mean      mean      mean      mean      mean      mean   \n",
       "ranef_grouping                                                               \n",
       "0               0.072362  0.117356  0.094708  0.055965  0.147862  0.068689   \n",
       "1               0.073022  0.093757  0.102363  0.083688  0.144348  0.076305   \n",
       "2               0.073784  0.111356  0.076307  0.050178  0.130441  0.059123   \n",
       "\n",
       "                      t6        t7        t8  \n",
       "                    mean      mean      mean  \n",
       "ranef_grouping                                \n",
       "0               0.124238  0.056790  0.262029  \n",
       "1               0.093023  0.074407  0.259087  \n",
       "2               0.093142  0.067075  0.338593  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topic_aggregate = second_merge.groupby('ranef_grouping').agg({\n",
    "    't0' : ['mean'],\n",
    "    't1' : ['mean'],\n",
    "    't2' : ['mean'],\n",
    "    't3' : ['mean'],\n",
    "    't4' : ['mean'],\n",
    "    't5' : ['mean'],\n",
    "    't6' : ['mean'],\n",
    "    't7' : ['mean'],\n",
    "    't8' : ['mean']\n",
    "})\n",
    "\n",
    "topic_aggregate"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}