{ "cells": [ { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "merged_manifest = pd.read_csv('0207_readme_merged_manifest.csv')\n", "topic_distributions = pd.read_csv('020725_README_file_topic_distributions.csv')\n", "readability_scores = pd.read_csv('020725_README_readability.csv')" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "4226" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "first_merge = readability_scores.merge(topic_distributions, on=['filename'],how=\"inner\")\n", "#primary_merge = first_merge.merge(readability_scores, )\n", "first_merge['new_filepath'] = first_merge['filename']\n", "#len(first_merge)\n", "second_merge = first_merge.merge(merged_manifest, on=['new_filepath'], how=\"inner\")\n", "len(second_merge)\n" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr style=\"text-align: right;\">\n", " <th></th>\n", " <th>filename</th>\n", " <th>flesch_reading_ease</th>\n", " <th>flesch_kincaid_grade</th>\n", " <th>linsear_write_formula</th>\n", " <th>dale_chall_readability_score</th>\n", " <th>mcalpine_eflaw</th>\n", " <th>reading_time</th>\n", " <th>char_count</th>\n", " <th>word_count</th>\n", " <th>t0</th>\n", " <th>...</th>\n", " <th>effect</th>\n", " <th>group</th>\n", " <th>level</th>\n", " <th>term</th>\n", " <th>estimate</th>\n", " <th>std.error</th>\n", " <th>conf.low</th>\n", " <th>conf.high</th>\n", " <th>ranef_grouping</th>\n", " <th>rank</th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>0</th>\n", " <td>libevent_libevent_hullabaloo_README</td>\n", " <td>50.36</td>\n", " <td>11.4</td>\n", " <td>14.625000</td>\n", " <td>12.47</td>\n", " <td>26.3</td>\n", " <td>7.37</td>\n", " <td>502</td>\n", " <td>83</td>\n", " <td>0.001976</td>\n", " <td>...</td>\n", " <td>ran_vals</td>\n", " <td>project_id</td>\n", " <td>libevent_libevent</td>\n", " <td>before_after:week_index</td>\n", " <td>0.186522</td>\n", " <td>0.478091</td>\n", " <td>-0.750519</td>\n", " <td>1.123563</td>\n", " <td>1</td>\n", " <td>3229.0</td>\n", " </tr>\n", " <tr>\n", " <th>1</th>\n", " <td>zaach_jison_hullabaloo_README.md</td>\n", " <td>44.78</td>\n", " <td>13.5</td>\n", " <td>7.714286</td>\n", " <td>10.70</td>\n", " <td>37.2</td>\n", " <td>33.05</td>\n", " <td>2250</td>\n", " <td>395</td>\n", " <td>0.000489</td>\n", " <td>...</td>\n", " <td>ran_vals</td>\n", " <td>project_id</td>\n", " <td>zaach_jison</td>\n", " <td>before_after:week_index</td>\n", " <td>1.001879</td>\n", " <td>0.217063</td>\n", " <td>0.576443</td>\n", " <td>1.427315</td>\n", " <td>2</td>\n", " <td>4068.0</td>\n", " </tr>\n", " <tr>\n", " <th>2</th>\n", " <td>lincolnloop_python-qrcode.git_hullabaloo_READM...</td>\n", " <td>58.28</td>\n", " <td>8.4</td>\n", " <td>7.583333</td>\n", " <td>10.76</td>\n", " <td>18.3</td>\n", " <td>6.82</td>\n", " <td>464</td>\n", " <td>78</td>\n", " <td>0.002597</td>\n", " <td>...</td>\n", " <td>ran_vals</td>\n", " <td>project_id</td>\n", " <td>lincolnloop_python-qrcode.git</td>\n", " <td>before_after:week_index</td>\n", " <td>-0.220654</td>\n", " <td>0.679910</td>\n", " <td>-1.553254</td>\n", " <td>1.111946</td>\n", " <td>1</td>\n", " <td>1467.0</td>\n", " </tr>\n", " <tr>\n", " <th>3</th>\n", " <td>mati75_evilwm.git_hullabaloo_README</td>\n", " <td>66.74</td>\n", " <td>7.2</td>\n", " <td>5.888889</td>\n", " <td>8.77</td>\n", " <td>18.1</td>\n", " <td>56.76</td>\n", " <td>3864</td>\n", " <td>742</td>\n", " <td>0.000266</td>\n", " <td>...</td>\n", " <td>ran_vals</td>\n", " <td>project_id</td>\n", " <td>mati75_evilwm.git</td>\n", " <td>before_after:week_index</td>\n", " <td>-0.293846</td>\n", " <td>0.697316</td>\n", " <td>-1.660560</td>\n", " <td>1.072868</td>\n", " <td>1</td>\n", " <td>1184.0</td>\n", " </tr>\n", " <tr>\n", " <th>4</th>\n", " <td>markdown-it_markdown-it_hullabaloo_README.md</td>\n", " <td>45.76</td>\n", " <td>11.1</td>\n", " <td>12.000000</td>\n", " <td>12.84</td>\n", " <td>24.0</td>\n", " <td>1.51</td>\n", " <td>103</td>\n", " <td>17</td>\n", " <td>0.006993</td>\n", " <td>...</td>\n", " <td>ran_vals</td>\n", " <td>project_id</td>\n", " <td>markdown-it_markdown-it</td>\n", " <td>before_after:week_index</td>\n", " <td>0.238615</td>\n", " <td>0.602856</td>\n", " <td>-0.942961</td>\n", " <td>1.420190</td>\n", " <td>1</td>\n", " <td>3459.0</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "<p>5 rows × 33 columns</p>\n", "</div>" ], "text/plain": [ " filename flesch_reading_ease \\\n", "0 libevent_libevent_hullabaloo_README 50.36 \n", "1 zaach_jison_hullabaloo_README.md 44.78 \n", "2 lincolnloop_python-qrcode.git_hullabaloo_READM... 58.28 \n", "3 mati75_evilwm.git_hullabaloo_README 66.74 \n", "4 markdown-it_markdown-it_hullabaloo_README.md 45.76 \n", "\n", " flesch_kincaid_grade linsear_write_formula dale_chall_readability_score \\\n", "0 11.4 14.625000 12.47 \n", "1 13.5 7.714286 10.70 \n", "2 8.4 7.583333 10.76 \n", "3 7.2 5.888889 8.77 \n", "4 11.1 12.000000 12.84 \n", "\n", " mcalpine_eflaw reading_time char_count word_count t0 ... \\\n", "0 26.3 7.37 502 83 0.001976 ... \n", "1 37.2 33.05 2250 395 0.000489 ... \n", "2 18.3 6.82 464 78 0.002597 ... \n", "3 18.1 56.76 3864 742 0.000266 ... \n", "4 24.0 1.51 103 17 0.006993 ... \n", "\n", " effect group level \\\n", "0 ran_vals project_id libevent_libevent \n", "1 ran_vals project_id zaach_jison \n", "2 ran_vals project_id lincolnloop_python-qrcode.git \n", "3 ran_vals project_id mati75_evilwm.git \n", "4 ran_vals project_id markdown-it_markdown-it \n", "\n", " term estimate std.error conf.low conf.high \\\n", "0 before_after:week_index 0.186522 0.478091 -0.750519 1.123563 \n", "1 before_after:week_index 1.001879 0.217063 0.576443 1.427315 \n", "2 before_after:week_index -0.220654 0.679910 -1.553254 1.111946 \n", "3 before_after:week_index -0.293846 0.697316 -1.660560 1.072868 \n", "4 before_after:week_index 0.238615 0.602856 -0.942961 1.420190 \n", "\n", " ranef_grouping rank \n", "0 1 3229.0 \n", "1 2 4068.0 \n", "2 1 1467.0 \n", "3 1 1184.0 \n", "4 1 3459.0 \n", "\n", "[5 rows x 33 columns]" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "second_merge.head()" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead tr th {\n", " text-align: left;\n", " }\n", "\n", " .dataframe thead tr:last-of-type th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr>\n", " <th></th>\n", " <th colspan=\"2\" halign=\"left\">flesch_reading_ease</th>\n", " <th colspan=\"2\" halign=\"left\">reading_time</th>\n", " <th colspan=\"2\" halign=\"left\">word_count</th>\n", " </tr>\n", " <tr>\n", " <th></th>\n", " <th>mean</th>\n", " <th>median</th>\n", " <th>mean</th>\n", " <th>median</th>\n", " <th>mean</th>\n", " <th>median</th>\n", " </tr>\n", " <tr>\n", " <th>ranef_grouping</th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>0</th>\n", " <td>36.425600</td>\n", " <td>49.920</td>\n", " <td>27.601600</td>\n", " <td>10.06</td>\n", " <td>294.320000</td>\n", " <td>99.0</td>\n", " </tr>\n", " <tr>\n", " <th>1</th>\n", " <td>47.580425</td>\n", " <td>50.330</td>\n", " <td>27.562204</td>\n", " <td>15.09</td>\n", " <td>294.790940</td>\n", " <td>147.0</td>\n", " </tr>\n", " <tr>\n", " <th>2</th>\n", " <td>54.819748</td>\n", " <td>52.835</td>\n", " <td>31.478753</td>\n", " <td>14.04</td>\n", " <td>341.339329</td>\n", " <td>145.5</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "</div>" ], "text/plain": [ " flesch_reading_ease reading_time word_count \\\n", " mean median mean median mean \n", "ranef_grouping \n", "0 36.425600 49.920 27.601600 10.06 294.320000 \n", "1 47.580425 50.330 27.562204 15.09 294.790940 \n", "2 54.819748 52.835 31.478753 14.04 341.339329 \n", "\n", " \n", " median \n", "ranef_grouping \n", "0 99.0 \n", "1 147.0 \n", "2 145.5 " ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "readability_aggregate = second_merge.groupby('ranef_grouping').agg({\n", " 'flesch_reading_ease' : ['mean', 'median'],\n", " 'reading_time' : ['mean', 'median'],\n", " 'word_count' : ['mean', 'median'],\n", "})\n", "\n", "readability_aggregate" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead tr th {\n", " text-align: left;\n", " }\n", "\n", " .dataframe thead tr:last-of-type th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr>\n", " <th></th>\n", " <th>t0</th>\n", " <th>t1</th>\n", " <th>t2</th>\n", " <th>t3</th>\n", " <th>t4</th>\n", " <th>t5</th>\n", " <th>t6</th>\n", " <th>t7</th>\n", " <th>t8</th>\n", " </tr>\n", " <tr>\n", " <th></th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " <th>mean</th>\n", " </tr>\n", " <tr>\n", " <th>ranef_grouping</th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " <th></th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>0</th>\n", " <td>0.072362</td>\n", " <td>0.117356</td>\n", " <td>0.094708</td>\n", " <td>0.055965</td>\n", " <td>0.147862</td>\n", " <td>0.068689</td>\n", " <td>0.124238</td>\n", " <td>0.056790</td>\n", " <td>0.262029</td>\n", " </tr>\n", " <tr>\n", " <th>1</th>\n", " <td>0.073022</td>\n", " <td>0.093757</td>\n", " <td>0.102363</td>\n", " <td>0.083688</td>\n", " <td>0.144348</td>\n", " <td>0.076305</td>\n", " <td>0.093023</td>\n", " <td>0.074407</td>\n", " <td>0.259087</td>\n", " </tr>\n", " <tr>\n", " <th>2</th>\n", " <td>0.073784</td>\n", " <td>0.111356</td>\n", " <td>0.076307</td>\n", " <td>0.050178</td>\n", " <td>0.130441</td>\n", " <td>0.059123</td>\n", " <td>0.093142</td>\n", " <td>0.067075</td>\n", " <td>0.338593</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "</div>" ], "text/plain": [ " t0 t1 t2 t3 t4 t5 \\\n", " mean mean mean mean mean mean \n", "ranef_grouping \n", "0 0.072362 0.117356 0.094708 0.055965 0.147862 0.068689 \n", "1 0.073022 0.093757 0.102363 0.083688 0.144348 0.076305 \n", "2 0.073784 0.111356 0.076307 0.050178 0.130441 0.059123 \n", "\n", " t6 t7 t8 \n", " mean mean mean \n", "ranef_grouping \n", "0 0.124238 0.056790 0.262029 \n", "1 0.093023 0.074407 0.259087 \n", "2 0.093142 0.067075 0.338593 " ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "topic_aggregate = second_merge.groupby('ranef_grouping').agg({\n", " 't0' : ['mean'],\n", " 't1' : ['mean'],\n", " 't2' : ['mean'],\n", " 't3' : ['mean'],\n", " 't4' : ['mean'],\n", " 't5' : ['mean'],\n", " 't6' : ['mean'],\n", " 't7' : ['mean'],\n", " 't8' : ['mean']\n", "})\n", "\n", "topic_aggregate" ] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.2" } }, "nbformat": 4, "nbformat_minor": 2 }