mgaughan/govdoc-cr-analysis
1
0
Fork 0
Code
main
govdoc-cr-analysis/text_analysis/readme_partitioned_analysis.ipynb

547 lines
18 KiB
Plaintext
Raw Permalink Normal View History

preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
{
"cells": [
{
"cell_type": "code",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 22,
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd"
]
},
{
"cell_type": "code",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 23,
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
"metadata": {},
"outputs": [],
"source": [
"merged_manifest = pd.read_csv('0203_readme_merged_manifest.csv')\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"topic_distributions = pd.read_csv('020325_README_file_topic_distributions.csv')\n",
"readability_scores = pd.read_csv('020325_README_readability.csv')"
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
]
},
{
"cell_type": "code",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 26,
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"4247"
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
]
},
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 26,
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"first_merge = readability_scores.merge(topic_distributions, on=['filename'],how=\"inner\")\n",
"#primary_merge = first_merge.merge(readability_scores, )\n",
"first_merge['fvf_filepath'] = first_merge['filename']\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"#len(first_merge)\n",
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
"second_merge = first_merge.merge(merged_manifest, on=['fvf_filepath'], how=\"inner\")\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"len(second_merge)\n"
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
]
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>filename</th>\n",
" <th>flesch_reading_ease</th>\n",
" <th>flesch_kincaid_grade</th>\n",
" <th>linsear_write_formula</th>\n",
" <th>dale_chall_readability_score</th>\n",
" <th>mcalpine_eflaw</th>\n",
" <th>reading_time</th>\n",
" <th>char_count</th>\n",
" <th>word_count</th>\n",
" <th>t0</th>\n",
" <th>...</th>\n",
" <th>effect</th>\n",
" <th>group</th>\n",
" <th>level</th>\n",
" <th>term</th>\n",
" <th>estimate</th>\n",
" <th>std.error</th>\n",
" <th>conf.low</th>\n",
" <th>conf.high</th>\n",
" <th>ranef_grouping</th>\n",
" <th>rank</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>libevent_libevent_hullabaloo_README</td>\n",
" <td>50.36</td>\n",
" <td>11.4</td>\n",
" <td>14.625000</td>\n",
" <td>12.47</td>\n",
" <td>26.3</td>\n",
" <td>7.37</td>\n",
" <td>502</td>\n",
" <td>83</td>\n",
" <td>0.001976</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>libevent_libevent</td>\n",
" <td>before_after:week_index</td>\n",
" <td>0.186522</td>\n",
" <td>0.478091</td>\n",
" <td>-0.750519</td>\n",
" <td>1.123563</td>\n",
" <td>1</td>\n",
" <td>3229.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>zaach_jison_hullabaloo_README.md</td>\n",
" <td>44.78</td>\n",
" <td>13.5</td>\n",
" <td>7.714286</td>\n",
" <td>10.70</td>\n",
" <td>37.2</td>\n",
" <td>33.05</td>\n",
" <td>2250</td>\n",
" <td>395</td>\n",
" <td>0.000489</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>zaach_jison</td>\n",
" <td>before_after:week_index</td>\n",
" <td>1.001879</td>\n",
" <td>0.217063</td>\n",
" <td>0.576443</td>\n",
" <td>1.427315</td>\n",
" <td>2</td>\n",
" <td>4068.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>lincolnloop_python-qrcode.git_hullabaloo_READM...</td>\n",
" <td>58.28</td>\n",
" <td>8.4</td>\n",
" <td>7.583333</td>\n",
" <td>10.76</td>\n",
" <td>18.3</td>\n",
" <td>6.82</td>\n",
" <td>464</td>\n",
" <td>78</td>\n",
" <td>0.002597</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>lincolnloop_python-qrcode.git</td>\n",
" <td>before_after:week_index</td>\n",
" <td>-0.220654</td>\n",
" <td>0.679910</td>\n",
" <td>-1.553254</td>\n",
" <td>1.111946</td>\n",
" <td>1</td>\n",
" <td>1467.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>mati75_evilwm.git_hullabaloo_README</td>\n",
" <td>66.74</td>\n",
" <td>7.2</td>\n",
" <td>5.888889</td>\n",
" <td>8.77</td>\n",
" <td>18.1</td>\n",
" <td>56.76</td>\n",
" <td>3864</td>\n",
" <td>742</td>\n",
" <td>0.000266</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>mati75_evilwm.git</td>\n",
" <td>before_after:week_index</td>\n",
" <td>-0.293846</td>\n",
" <td>0.697316</td>\n",
" <td>-1.660560</td>\n",
" <td>1.072868</td>\n",
" <td>1</td>\n",
" <td>1184.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>markdown-it_markdown-it_hullabaloo_README.md</td>\n",
" <td>45.76</td>\n",
" <td>11.1</td>\n",
" <td>12.000000</td>\n",
" <td>12.84</td>\n",
" <td>24.0</td>\n",
" <td>1.51</td>\n",
" <td>103</td>\n",
" <td>17</td>\n",
" <td>0.006993</td>\n",
" <td>...</td>\n",
" <td>ran_vals</td>\n",
" <td>project_id</td>\n",
" <td>markdown-it_markdown-it</td>\n",
" <td>before_after:week_index</td>\n",
" <td>0.238615</td>\n",
" <td>0.602856</td>\n",
" <td>-0.942961</td>\n",
" <td>1.420190</td>\n",
" <td>1</td>\n",
" <td>3459.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 33 columns</p>\n",
"</div>"
],
"text/plain": [
" filename flesch_reading_ease \\\n",
"0 libevent_libevent_hullabaloo_README 50.36 \n",
"1 zaach_jison_hullabaloo_README.md 44.78 \n",
"2 lincolnloop_python-qrcode.git_hullabaloo_READM... 58.28 \n",
"3 mati75_evilwm.git_hullabaloo_README 66.74 \n",
"4 markdown-it_markdown-it_hullabaloo_README.md 45.76 \n",
"\n",
" flesch_kincaid_grade linsear_write_formula dale_chall_readability_score \\\n",
"0 11.4 14.625000 12.47 \n",
"1 13.5 7.714286 10.70 \n",
"2 8.4 7.583333 10.76 \n",
"3 7.2 5.888889 8.77 \n",
"4 11.1 12.000000 12.84 \n",
"\n",
" mcalpine_eflaw reading_time char_count word_count t0 ... \\\n",
"0 26.3 7.37 502 83 0.001976 ... \n",
"1 37.2 33.05 2250 395 0.000489 ... \n",
"2 18.3 6.82 464 78 0.002597 ... \n",
"3 18.1 56.76 3864 742 0.000266 ... \n",
"4 24.0 1.51 103 17 0.006993 ... \n",
"\n",
" effect group level \\\n",
"0 ran_vals project_id libevent_libevent \n",
"1 ran_vals project_id zaach_jison \n",
"2 ran_vals project_id lincolnloop_python-qrcode.git \n",
"3 ran_vals project_id mati75_evilwm.git \n",
"4 ran_vals project_id markdown-it_markdown-it \n",
"\n",
" term estimate std.error conf.low conf.high \\\n",
"0 before_after:week_index 0.186522 0.478091 -0.750519 1.123563 \n",
"1 before_after:week_index 1.001879 0.217063 0.576443 1.427315 \n",
"2 before_after:week_index -0.220654 0.679910 -1.553254 1.111946 \n",
"3 before_after:week_index -0.293846 0.697316 -1.660560 1.072868 \n",
"4 before_after:week_index 0.238615 0.602856 -0.942961 1.420190 \n",
"\n",
" ranef_grouping rank \n",
"0 1 3229.0 \n",
"1 2 4068.0 \n",
"2 1 1467.0 \n",
"3 1 1184.0 \n",
"4 1 3459.0 \n",
"\n",
"[5 rows x 33 columns]"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"second_merge.head()"
]
},
{
"cell_type": "code",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 27,
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th colspan=\"2\" halign=\"left\">flesch_reading_ease</th>\n",
" <th colspan=\"2\" halign=\"left\">reading_time</th>\n",
" <th colspan=\"2\" halign=\"left\">word_count</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>mean</th>\n",
" <th>median</th>\n",
" <th>mean</th>\n",
" <th>median</th>\n",
" <th>mean</th>\n",
" <th>median</th>\n",
" </tr>\n",
" <tr>\n",
" <th>ranef_grouping</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
" <td>55.089708</td>\n",
" <td>52.900</td>\n",
" <td>23.486886</td>\n",
" <td>9.28</td>\n",
" <td>256.973406</td>\n",
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>46.425909</td>\n",
" <td>51.365</td>\n",
" <td>22.760642</td>\n",
" <td>9.53</td>\n",
" <td>249.534759</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" flesch_reading_ease reading_time word_count \\\n",
" mean median mean median mean \n",
"ranef_grouping \n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"1 55.089708 52.900 23.486886 9.28 256.973406 \n",
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
"2 46.425909 51.365 22.760642 9.53 249.534759 \n",
"\n",
" \n",
" median \n",
"ranef_grouping \n",
"1 90.0 \n",
"2 100.0 "
]
},
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 27,
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"readability_aggregate = second_merge.groupby('ranef_grouping').agg({\n",
" 'flesch_reading_ease' : ['mean', 'median'],\n",
" 'reading_time' : ['mean', 'median'],\n",
" 'word_count' : ['mean', 'median'],\n",
"})\n",
"\n",
"readability_aggregate"
]
},
{
"cell_type": "code",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 29,
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th>t0</th>\n",
" <th>t1</th>\n",
" <th>t2</th>\n",
" <th>t3</th>\n",
" <th>t4</th>\n",
" <th>t5</th>\n",
" <th>t6</th>\n",
" <th>t7</th>\n",
" <th>t8</th>\n",
" <th>t9</th>\n",
" <th>t10</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" <th>mean</th>\n",
" </tr>\n",
" <tr>\n",
" <th>ranef_grouping</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>1</th>\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
" <td>0.095142</td>\n",
" <td>0.101366</td>\n",
" <td>0.084609</td>\n",
" <td>0.127050</td>\n",
" <td>0.024873</td>\n",
" <td>0.060155</td>\n",
" <td>0.088660</td>\n",
" <td>0.136191</td>\n",
" <td>0.060451</td>\n",
" <td>0.134408</td>\n",
" <td>0.087095</td>\n",
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
" <td>0.122912</td>\n",
" <td>0.102389</td>\n",
" <td>0.050782</td>\n",
" <td>0.168774</td>\n",
" <td>0.015752</td>\n",
" <td>0.072154</td>\n",
" <td>0.102839</td>\n",
" <td>0.115994</td>\n",
" <td>0.053674</td>\n",
" <td>0.119278</td>\n",
" <td>0.075450</td>\n",
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" t0 t1 t2 t3 t4 t5 \\\n",
" mean mean mean mean mean mean \n",
"ranef_grouping \n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"1 0.095142 0.101366 0.084609 0.127050 0.024873 0.060155 \n",
"2 0.122912 0.102389 0.050782 0.168774 0.015752 0.072154 \n",
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
"\n",
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
" t6 t7 t8 t9 t10 \n",
" mean mean mean mean mean \n",
"ranef_grouping \n",
"1 0.088660 0.136191 0.060451 0.134408 0.087095 \n",
"2 0.102839 0.115994 0.053674 0.119278 0.075450 "
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
]
},
updated topic model and combo analysis
2025-02-04 01:06:13 +00:00
"execution_count": 29,
parttion analysis of topic distributions and readability
2025-02-03 19:51:17 +00:00
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"topic_aggregate = second_merge.groupby('ranef_grouping').agg({\n",
" 't0' : ['mean'],\n",
" 't1' : ['mean'],\n",
" 't2' : ['mean'],\n",
" 't3' : ['mean'],\n",
" 't4' : ['mean'],\n",
" 't5' : ['mean'],\n",
" 't6' : ['mean'],\n",
" 't7' : ['mean'],\n",
" 't8' : ['mean'],\n",
" 't9' : ['mean'],\n",
" 't10': ['mean']\n",
"})\n",
"\n",
"topic_aggregate"
]
preparing for partitioned analysis
2025-02-03 17:57:20 +00:00
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Copy Permalink