govdoc-cr-analysis/text_analysis/.ipynb_checkpoints/partitioned_readability-checkpoint.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4c4796f-d109-472d-8f9c-95c6ec85f757",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "import textstat\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f1f2fce-2335-4ee3-81f2-55822e2f63f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "readme_wd = \"\"\n",
    "contributing_wd = \"\"\n",
    "\n",
    "csv_fieldnames = ['subdir', 'filename', 'flesch_reading_ease', 'flesch_kincaid_grade', 'linsear_write_formula', 'dale_chall_readability_score', 'mcalpine_eflaw', 'reading_time', 'char_count', 'word_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0d3b5b1-ae97-4a46-95e0-92232c46c2fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "gets the 3 readability scores for each individual textfile\n",
    "'''\n",
    "def get_readibility(file_address, file_dict):\n",
    "    file = open(file_address, \"r\")\n",
    "    document = file.read()\n",
    "    file_dict['flesch_reading_ease'] = textstat.flesch_reading_ease(document)\n",
    "    file_dict['flesch_kincaid_grade'] = textstat.flesch_kincaid_grade(document)\n",
    "    file_dict['linsear_write_formula'] = textstat.linsear_write_formula(document)\n",
    "    file_dict['dale_chall_readability_score'] = textstat.dale_chall_readability_score(document)\n",
    "    file_dict['mcalpine_eflaw'] = textstat.mcalpine_eflaw(document)\n",
    "    file_dict['reading_time']  = textstat.reading_time(document, ms_per_char=14.69)\n",
    "    file_dict['char_count'] = textstat.char_count(document, ignore_spaces=True)\n",
    "    file_dict['word_count'] = textstat.lexicon_count(document, removepunct=True)\n",
    "    return file_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b3c481e-c521-4e1d-926e-88f4b75ae7de",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "getting readability scoring for each type of document\n",
    "'''\n",
    "def generate_file(output_csv, wdirectory, document_type):\n",
    "    with open(output_csv, 'w') as csvfile: \n",
    "        writer = csv.DictWriter(csvfile, fieldnames = csv_fieldnames) \n",
    "        writer.writeheader() \n",
    "        subdirs = os.listdir(wdirectory)\n",
    "        print(document_type)\n",
    "        for dir in subdirs: \n",
    "            print(dir)\n",
    "            files = os.listdir(wdirectory + \"/\" + dir)\n",
    "            count = 0\n",
    "            for file in files:\n",
    "                file_dict = {\"subdir\": dir, \"filename\": file}\n",
    "                print(file)\n",
    "                full_address = wdirectory + \"/\" + dir + \"/\" + file\n",
    "                file_dict = get_readibility(full_address, file_dict)\n",
    "                writer.writerow(file_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e0a7b88-49b6-4053-84b8-f54f1c6536c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_file('dwo_readability_contributing.csv', contributing_wd, \"contributing\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}