{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"merged_manifest = pd.read_csv('0207_readme_merged_manifest.csv')\n",
"topic_distributions = pd.read_csv('020725_README_file_topic_distributions.csv')\n",
"readability_scores = pd.read_csv('020725_README_readability.csv')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4226"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"first_merge = readability_scores.merge(topic_distributions, on=['filename'],how=\"inner\")\n",
"#primary_merge = first_merge.merge(readability_scores, )\n",
"first_merge['new_filepath'] = first_merge['filename']\n",
"#len(first_merge)\n",
"second_merge = first_merge.merge(merged_manifest, on=['new_filepath'], how=\"inner\")\n",
"len(second_merge)\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>filename</th>\n",
" <th>flesch_reading_ease</th>\n",
" <th>flesch_kincaid_grade</th>\n",
" <th>linsear_write_formula</th>\n",
" <th>dale_chall_readability_score</th>\n",
" <th>mcalpine_eflaw</th>\n",
" <th>reading_time</th>\n",
" <th>char_count</th>\n",
" <th>word_count</th>\n",
" <th>t0</th>\n",
" <th>...</th>\n",
" <th>effect</th>\n",
" <th>group</th>\n",
" <th>level</th>\n",
" <th>term</th>\n",
" <th>estimate</th>\n",
" <th>std.error</th>\n",
" <th>conf.low</th>\n",
" <th>conf.high</th>\n",
" <th>ranef_grouping</th>\n",
" <th>rank</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>libevent_libevent_hullabaloo_README</td>\n",
" <td>50.36</td>\n",
" <td>11.4</td>\n",
" <td>14.625000</td>\n",
" <td>12.47</td>\n",
" <td>26.3</td>\n",
" <td>7.37</td>\n",
" <td>502</td>\n",
" <td>83</td>\n",
" <td>0.001976</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>libevent_libevent</td>\n",
" <td>before_after:week_index</td>\n",
" <td>0.186522</td>\n",
" <td>0.478091</td>\n",
" <td>-0.750519</td>\n",
" <td>1.123563</td>\n",
" <td>1</td>\n",
" <td>3229.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>zaach_jison_hullabaloo_README.md</td>\n",
" <td>44.78</td>\n",
" <td>13.5</td>\n",
" <td>7.714286</td>\n",
" <td>10.70</td>\n",
" <td>37.2</td>\n",
" <td>33.05</td>\n",
" <td>2250</td>\n",
" <td>395</td>\n",
" <td>0.000489</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>zaach_jison</td>\n",
" <td>before_after:week_index</td>\n",
" <td>1.001879</td>\n",
" <td>0.217063</td>\n",
" <td>0.576443</td>\n",
" <td>1.427315</td>\n",
" <td>2</td>\n",
" <td>4068.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>lincolnloop_python-qrcode.git_hullabaloo_READM...</td>\n",
" <td>58.28</td>\n",
" <td>8.4</td>\n",
" <td>7.583333</td>\n",
" <td>10.76</td>\n",
" <td>18.3</td>\n",
" <td>6.82</td>\n",
" <td>464</td>\n",
" <td>78</td>\n",
" <td>0.002597</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>lincolnloop_python-qrcode.git</td>\n",
" <td>before_after:week_index</td>\n",
" <td>-0.220654</td>\n",
" <td>0.679910</td>\n",
" <td>-1.553254</td>\n",
" <td>1.111946</td>\n",
" <td>1</td>\n",
" <td>1467.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>mati75_evilwm.git_hullabaloo_README</td>\n",
" <td>66.74</td>\n",
" <td>7.2</td>\n",
" <td>5.888889</td>\n",
" <td>8.77</td>\n",
" <td>18.1</td>\n",
" <td>56.76</td>\n",
" <td>3864</td>\n",
" <td>742</td>\n",
" <td>0.000266</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>mati75_evilwm.git</td>\n",
" <td>before_after:week_index</td>\n",
" <td>-0.293846</td>\n",
" <td>0.697316</td>\n",
" <td>-1.660560</td>\n",
" <td>1.072868</td>\n",
" <td>1</td>\n",
" <td>1184.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>markdown-it_markdown-it_hullabaloo_README.md</td>\n",
" <td>45.76</td>\n",
" <td>11.1</td>\n",
" <td>12.000000</td>\n",
" <td>12.84</td>\n",
" <td>24.0</td>\n",
" <td>1.51</td>\n",
" <td>103</td>\n",
" <td>17</td>\n",
" <td>0.006993</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>markdown-it_markdown-it</td>\n",
" <td>before_after:week_index</td>\n",
" <td>0.238615</td>\n",
" <td>0.602856</td>\n",
" <td>-0.942961</td>\n",
" <td>1.420190</td>\n",
" <td>1</td>\n",
" <td>3459.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 33 columns</p>\n",
"</div>"
],
"text/plain": [
" filename flesch_reading_ease \\\n",
"0 libevent_libevent_hullabaloo_README 50.36 \n",
"1 zaach_jison_hullabaloo_README.md 44.78 \n",
"2 lincolnloop_python-qrcode.git_hullabaloo_READM... 58.28 \n",
"3 mati75_evilwm.git_hullabaloo_README 66.74 \n",
"4 markdown-it_markdown-it_hullabaloo_README.md 45.76 \n",
"\n",
" flesch_kincaid_grade linsear_write_formula dale_chall_readability_score \\\n",
"0 11.4 14.625000 12.47 \n",
"1 13.5 7.714286 10.70 \n",
"2 8.4 7.583333 10.76 \n",
"3 7.2 5.888889 8.77 \n",
"4 11.1 12.000000 12.84 \n",
"\n",
" mcalpine_eflaw reading_time char_count word_count t0 ... \\\n",
"0 26.3 7.37 502 83 0.001976 ... \n",
"1 37.2 33.05 2250 395 0.000489 ... \n",
"2 18.3 6.82 464 78 0.002597 ... \n",
"3 18.1 56.76 3864 742 0.000266 ... \n",
"4 24.0 1.51 103 17 0.006993 ... \n",
"\n",
" effect group level \\\n",
"0 ran_vals project_id libevent_libevent \n",
"1 ran_vals project_id zaach_jison \n",
"2 ran_vals project_id lincolnloop_python-qrcode.git \n",
"3 ran_vals project_id mati75_evilwm.git \n",
"4 ran_vals project_id markdown-it_markdown-it \n",
"\n",
" term estimate std.error conf.low conf.high \\\n",
"0 before_after:week_index 0.186522 0.478091 -0.750519 1.123563 \n",
"1 before_after:week_index 1.001879 0.217063 0.576443 1.427315 \n",
"2 before_after:week_index -0.220654 0.679910 -1.553254 1.111946 \n",
"3 before_after:week_index -0.293846 0.697316 -1.660560 1.072868 \n",
"4 before_after:week_index 0.238615 0.602856 -0.942961 1.420190 \n",
"\n",
" ranef_grouping rank \n",
"0 1 3229.0 \n",
"1 2 4068.0 \n",
"2 1 1467.0 \n",
"3 1 1184.0 \n",
"4 1 3459.0 \n",
"\n",
"[5 rows x 33 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"second_merge.head()"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th colspan=\"2\" halign=\"left\">flesch_reading_ease</th>\n",
" <th colspan=\"2\" halign=\"left\">reading_time</th>\n",
" <th colspan=\"2\" halign=\"left\">word_count</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>mean</th>\n",
" <th>median</th>\n",
" <th>mean</th>\n",
" <th>median</th>\n",
" <th>mean</th>\n",
" <th>median</th>\n",
" </tr>\n",
" <tr>\n",
" <th>ranef_grouping</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>36.425600</td>\n",
" <td>49.920</td>\n",
" <td>27.601600</td>\n",
" <td>10.06</td>\n",
" <td>294.320000</td>\n",
" <td>99.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>47.580425</td>\n",
" <td>50.330</td>\n",
" <td>27.562204</td>\n",
" <td>15.09</td>\n",
" <td>294.790940</td>\n",
" <td>147.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>54.819748</td>\n",
" <td>52.835</td>\n",
" <td>31.478753</td>\n",
" <td>14.04</td>\n",
" <td>341.339329</td>\n",
" <td>145.5</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" flesch_reading_ease reading_time word_count \\\n",
" mean median mean median mean \n",
"ranef_grouping \n",
"0 36.425600 49.920 27.601600 10.06 294.320000 \n",
"1 47.580425 50.330 27.562204 15.09 294.790940 \n",
"2 54.819748 52.835 31.478753 14.04 341.339329 \n",
"\n",
" \n",
" median \n",
"ranef_grouping \n",
"0 99.0 \n",
"1 147.0 \n",
"2 145.5 "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"readability_aggregate = second_merge.groupby('ranef_grouping').agg({\n",
" 'flesch_reading_ease' : ['mean', 'median'],\n",
" 'reading_time' : ['mean', 'median'],\n",
" 'word_count' : ['mean', 'median'],\n",
"})\n",
"\n",
"readability_aggregate"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th>t0</th>\n",
" <th>t1</th>\n",
" <th>t2</th>\n",
" <th>t3</th>\n",
" <th>t4</th>\n",
" <th>t5</th>\n",
" <th>t6</th>\n",
" <th>t7</th>\n",
" <th>t8</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" </tr>\n",
" <tr>\n",
" <th>ranef_grouping</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>0.072362</td>\n",
" <td>0.117356</td>\n",
" <td>0.094708</td>\n",
" <td>0.055965</td>\n",
" <td>0.147862</td>\n",
" <td>0.068689</td>\n",
" <td>0.124238</td>\n",
" <td>0.056790</td>\n",
" <td>0.262029</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>0.073022</td>\n",
" <td>0.093757</td>\n",
" <td>0.102363</td>\n",
" <td>0.083688</td>\n",
" <td>0.144348</td>\n",
" <td>0.076305</td>\n",
" <td>0.093023</td>\n",
" <td>0.074407</td>\n",
" <td>0.259087</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>0.073784</td>\n",
" <td>0.111356</td>\n",
" <td>0.076307</td>\n",
" <td>0.050178</td>\n",
" <td>0.130441</td>\n",
" <td>0.059123</td>\n",
" <td>0.093142</td>\n",
" <td>0.067075</td>\n",
" <td>0.338593</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" t0 t1 t2 t3 t4 t5 \\\n",
" mean mean mean mean mean mean \n",
"ranef_grouping \n",
"0 0.072362 0.117356 0.094708 0.055965 0.147862 0.068689 \n",
"1 0.073022 0.093757 0.102363 0.083688 0.144348 0.076305 \n",
"2 0.073784 0.111356 0.076307 0.050178 0.130441 0.059123 \n",
"\n",
" t6 t7 t8 \n",
" mean mean mean \n",
"ranef_grouping \n",
"0 0.124238 0.056790 0.262029 \n",
"1 0.093023 0.074407 0.259087 \n",
"2 0.093142 0.067075 0.338593 "
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"topic_aggregate = second_merge.groupby('ranef_grouping').agg({\n",
" 't0' : ['mean'],\n",
" 't1' : ['mean'],\n",
" 't2' : ['mean'],\n",
" 't3' : ['mean'],\n",
" 't4' : ['mean'],\n",
" 't5' : ['mean'],\n",
" 't6' : ['mean'],\n",
" 't7' : ['mean'],\n",
" 't8' : ['mean']\n",
"})\n",
"\n",
"topic_aggregate"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}