diff --git a/src/lib/spark-warehouse/bot_isolation.ipynb b/src/lib/spark-warehouse/bot_isolation.ipynb index ea73875..3c0236d 100644 --- a/src/lib/spark-warehouse/bot_isolation.ipynb +++ b/src/lib/spark-warehouse/bot_isolation.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -25,16 +25,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "mediawiki_history_path = \"/data/users/mgaughan/mw-repo-lifecycles/wiki_activity_data/test\"" + "mediawiki_history_path = \"/data_ext/users/nws8519/mw-repo-lifecycles/wiki_activity_data/yearly_activity_files/\"" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -120,7 +120,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -129,7 +129,7 @@ "'/usr/lib/jvm/java-11-openjdk-amd64'" ] }, - "execution_count": 5, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -140,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -149,7 +149,7 @@ "text": [ "Setting default log level to \"WARN\".\n", "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n", - "25/01/10 10:42:38 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" + "25/01/16 12:17:29 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n" ] } ], @@ -162,7 +162,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -183,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -192,7 +192,7 @@ "" ] }, - "execution_count": 8, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -204,7 +204,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -289,32 +289,22 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - " \r" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+-------+--------------+\n", - "|wiki_db |month |revision_count|\n", - "+------------+-------+--------------+\n", - "|kwwiki |2019-12|1079 |\n", - "|kowikiquote |2019-12|146 |\n", - "|zuwiktionary|2019-12|135 |\n", - "+------------+-------+--------------+\n", - "\n" - ] + "data": { + "text/plain": [ + "'\\nmediawiki_history. where(\"event_entity = \\'revision\\' and event_type = \\'create\\'\"). selectExpr(\"wiki_db\", \"SUBSTR(event_timestamp, 0, 7) as month\"). where(\"month = \\'2019-12\\'\"). groupBy(\"wiki_db\", \"month\"). agg(count(lit(1)).alias(\"revision_count\")). sort(desc(\"revision_count\")). show(10, False)\\n'" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ + "'''\n", "mediawiki_history. \\\n", " where(\"event_entity = 'revision' and event_type = 'create'\"). \\\n", " selectExpr(\"wiki_db\", \"SUBSTR(event_timestamp, 0, 7) as month\"). \\\n", @@ -322,55 +312,63 @@ " groupBy(\"wiki_db\", \"month\"). \\\n", " agg(count(lit(1)).alias(\"revision_count\")). \\\n", " sort(desc(\"revision_count\")). \\\n", - " show(10, False)" + " show(10, False)\n", + "'''" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "[Stage 27:===> (2 + 33) / 35]\r" + "ERROR:root:KeyboardInterrupt while sending command. (64 + 56) / 8283]\n", + "Traceback (most recent call last):\n", + " File \"/opt/conda-analytics/lib/python3.10/site-packages/py4j/java_gateway.py\", line 1038, in send_command\n", + " response = connection.send_command(command)\n", + " File \"/opt/conda-analytics/lib/python3.10/site-packages/py4j/clientserver.py\", line 511, in send_command\n", + " answer = smart_decode(self.stream.readline()[:-1])\n", + " File \"/opt/conda-analytics/lib/python3.10/socket.py\", line 717, in readinto\n", + " return self._sock.recv_into(b)\n", + "KeyboardInterrupt\n" ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "+------------+----------+------------+\n", - "|wiki_db |day |action_count|\n", - "+------------+----------+------------+\n", - "|kwwiki |2024-11-30|6 |\n", - "|kowikiquote |2024-11-29|3 |\n", - "|kwwiki |2024-11-29|17 |\n", - "|zuwiktionary|2024-11-29|1 |\n", - "|kwwiki |2024-11-28|38 |\n", - "|kowikiquote |2024-11-28|2 |\n", - "|kwwiki |2024-11-27|45 |\n", - "|kowikiquote |2024-11-26|2 |\n", - "|kwwiki |2024-11-26|138 |\n", - "|zuwiktionary|2024-11-26|1 |\n", - "+------------+----------+------------+\n", - "only showing top 10 rows\n", - "\n" + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[15], line 6\u001b[0m\n\u001b[1;32m 2\u001b[0m activity_count_df \u001b[38;5;241m=\u001b[39m activity_count_df\u001b[38;5;241m.\u001b[39mselectExpr(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwiki_db\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSUBSTR(event_timestamp, 0, 10) as day\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mevent_entity\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mevent_type\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 3\u001b[0m activity_count_df \u001b[38;5;241m=\u001b[39m activity_count_df\u001b[38;5;241m.\u001b[39mgroupBy(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwiki_db\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mday\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mevent_entity\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mevent_type\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39magg(count(lit(\u001b[38;5;241m1\u001b[39m))\u001b[38;5;241m.\u001b[39malias(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mactivity_count\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[1;32m 4\u001b[0m \u001b[43mactivity_count_df\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m\\\u001b[49m\n\u001b[1;32m 5\u001b[0m \u001b[43m \u001b[49m\u001b[43msort\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdesc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mday\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m \u001b[49m\u001b[43m\\\u001b[49m\n\u001b[0;32m----> 6\u001b[0m \u001b[43m \u001b[49m\u001b[43mshow\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m10\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda-analytics/lib/python3.10/site-packages/pyspark/sql/dataframe.py:947\u001b[0m, in \u001b[0;36mDataFrame.show\u001b[0;34m(self, n, truncate, vertical)\u001b[0m\n\u001b[1;32m 887\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mshow\u001b[39m(\u001b[38;5;28mself\u001b[39m, n: \u001b[38;5;28mint\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m20\u001b[39m, truncate: Union[\u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28mint\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m, vertical: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 888\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Prints the first ``n`` rows to the console.\u001b[39;00m\n\u001b[1;32m 889\u001b[0m \n\u001b[1;32m 890\u001b[0m \u001b[38;5;124;03m .. versionadded:: 1.3.0\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 945\u001b[0m \u001b[38;5;124;03m name | Bob\u001b[39;00m\n\u001b[1;32m 946\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 947\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_show_string\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtruncate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvertical\u001b[49m\u001b[43m)\u001b[49m)\n", + "File \u001b[0;32m/opt/conda-analytics/lib/python3.10/site-packages/pyspark/sql/dataframe.py:978\u001b[0m, in \u001b[0;36mDataFrame._show_string\u001b[0;34m(self, n, truncate, vertical)\u001b[0m\n\u001b[1;32m 969\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m:\n\u001b[1;32m 970\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m PySparkTypeError(\n\u001b[1;32m 971\u001b[0m error_class\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNOT_BOOL\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 972\u001b[0m message_parameters\u001b[38;5;241m=\u001b[39m{\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 975\u001b[0m },\n\u001b[1;32m 976\u001b[0m )\n\u001b[0;32m--> 978\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jdf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshowString\u001b[49m\u001b[43m(\u001b[49m\u001b[43mn\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mint_truncate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvertical\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m/opt/conda-analytics/lib/python3.10/site-packages/py4j/java_gateway.py:1321\u001b[0m, in \u001b[0;36mJavaMember.__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m 1314\u001b[0m args_command, temp_args \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_args(\u001b[38;5;241m*\u001b[39margs)\n\u001b[1;32m 1316\u001b[0m command \u001b[38;5;241m=\u001b[39m proto\u001b[38;5;241m.\u001b[39mCALL_COMMAND_NAME \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1317\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_header \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1318\u001b[0m args_command \u001b[38;5;241m+\u001b[39m\\\n\u001b[1;32m 1319\u001b[0m proto\u001b[38;5;241m.\u001b[39mEND_COMMAND_PART\n\u001b[0;32m-> 1321\u001b[0m answer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgateway_client\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1322\u001b[0m return_value \u001b[38;5;241m=\u001b[39m get_return_value(\n\u001b[1;32m 1323\u001b[0m answer, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgateway_client, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtarget_id, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mname)\n\u001b[1;32m 1325\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m temp_arg \u001b[38;5;129;01min\u001b[39;00m temp_args:\n", + "File \u001b[0;32m/opt/conda-analytics/lib/python3.10/site-packages/py4j/java_gateway.py:1038\u001b[0m, in \u001b[0;36mGatewayClient.send_command\u001b[0;34m(self, command, retry, binary)\u001b[0m\n\u001b[1;32m 1036\u001b[0m connection \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_connection()\n\u001b[1;32m 1037\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1038\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend_command\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1039\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m binary:\n\u001b[1;32m 1040\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_connection_guard(connection)\n", + "File \u001b[0;32m/opt/conda-analytics/lib/python3.10/site-packages/py4j/clientserver.py:511\u001b[0m, in \u001b[0;36mClientServerConnection.send_command\u001b[0;34m(self, command)\u001b[0m\n\u001b[1;32m 509\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m 510\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 511\u001b[0m answer \u001b[38;5;241m=\u001b[39m smart_decode(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreadline\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m[:\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m])\n\u001b[1;32m 512\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAnswer received: \u001b[39m\u001b[38;5;132;01m{0}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;241m.\u001b[39mformat(answer))\n\u001b[1;32m 513\u001b[0m \u001b[38;5;66;03m# Happens when a the other end is dead. There might be an empty\u001b[39;00m\n\u001b[1;32m 514\u001b[0m \u001b[38;5;66;03m# answer before the socket raises an error.\u001b[39;00m\n", + "File \u001b[0;32m/opt/conda-analytics/lib/python3.10/socket.py:717\u001b[0m, in \u001b[0;36mSocketIO.readinto\u001b[0;34m(self, b)\u001b[0m\n\u001b[1;32m 715\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m 716\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 717\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_sock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv_into\u001b[49m\u001b[43m(\u001b[49m\u001b[43mb\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 718\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m timeout:\n\u001b[1;32m 719\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_timeout_occurred \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] }, { "name": "stderr", "output_type": "stream", "text": [ - " \r" + "25/01/16 12:21:31 WARN RowBasedKeyValueBatch: Calling spill() on RowBasedKeyValueBatch. Will not spill but return 0.\n", + "25/01/16 12:21:31 WARN RowBasedKeyValueBatch: Calling spill() on RowBasedKeyValueBatch. Will not spill but return 0.\n", + "25/01/16 12:21:37 WARN RowBasedKeyValueBatch: Calling spill() on RowBasedKeyValueBatch. Will not spill but return 0.\n", + "25/01/16 12:21:37 WARN RowBasedKeyValueBatch: Calling spill() on RowBasedKeyValueBatch. Will not spill but return 0.\n", + "[Stage 0:==> (397 + 56) / 8283]\r" ] } ], "source": [ "activity_count_df = mediawiki_history.where(\"event_user_is_bot_by_historical is not null and event_user_is_bot_by is not null\")\n", - "activity_count_df = activity_count_df.selectExpr(\"wiki_db\", \"SUBSTR(event_timestamp, 0, 10) as day\")\n", - "activity_count_df = activity_count_df.groupBy(\"wiki_db\", \"day\", \"event_\" \"event_type\").agg(count(lit(1)).alias(\"activity_count\"))\n", + "activity_count_df = activity_count_df.selectExpr(\"wiki_db\", \"SUBSTR(event_timestamp, 0, 10) as day\", \"event_entity\", \"event_type\")\n", + "activity_count_df = activity_count_df.groupBy(\"wiki_db\", \"day\", \"event_entity\", \"event_type\").agg(count(lit(1)).alias(\"activity_count\"))\n", "activity_count_df.\\\n", " sort(desc(\"day\")). \\\n", " show(10, False)" @@ -397,6 +395,7 @@ } ], "source": [ + "'''\n", "filtered_df = mediawiki_history. \\\n", " where(\"event_entity = 'user' and event_type='create'\"). \\\n", " drop(\"event_user_blocks_historical\") .\\\n", @@ -405,7 +404,8 @@ " groupBy(\"wiki_db\"). \\\n", " agg(count(lit(1)).alias(\"revision_count\")). \\\n", " sort(desc(\"revision_count\")). \\\n", - " show(10, False)" + " show(10, False)\n", + "'''" ] }, { @@ -430,7 +430,7 @@ } ], "source": [ - "filtered_df.write.format(\"csv\").save(\"test.csv\")" + "activity_count_df.write.format(\"csv\").save(\"011625_dab_yearly.csv\")" ] } ], diff --git a/src/lib/spark-warehouse/bot_isolation.py b/src/lib/spark-warehouse/bot_isolation.py new file mode 100644 index 0000000..648d82d --- /dev/null +++ b/src/lib/spark-warehouse/bot_isolation.py @@ -0,0 +1,200 @@ +import re +import os +from pyspark.sql.types import StructType, StructField, StringType, LongType, BooleanType, IntegerType, ArrayType +from pyspark.sql.functions import count, lit, desc +from pyspark.sql import SparkSession + +os.environ['JAVA_HOME'] = "/usr/lib/jvm/java-11-openjdk-amd64" +os.environ['JRE_HOME'] = "/usr/lib/jvm/java-11-openjdk-amd64/jre" + +# Unescaping and array-splitting UDFs +def unescape(str): + if (str is None): + return None + else: + return str.replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t") +# The comma splitter applies a negative lookahead for \ to prevent splitting escaped commas +def toArray(str): + if (str is None): + return [] + else: + return [s.strip().replace("\\,", ",") for s in re.split("(?\":\n", - " dtype = \"object\"\n", - "\n", - " if \"timestamp\" in fname:\n", - " dtype = \"object\"\n", - "\n", - " CSV_FIELDS_META[fname] = {\"class\": fclass, \"dtype\": dtype, \"comment\": comment}" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "id\tfield \tdtype\n", - "----\t------------------------------------------------------------\t------\n", - "1\twiki_db \tstring\n", - "2\tevent_entity \tstring\n", - "3\tevent_type \tstring\n", - "4\tevent_timestamp \tobject\n", - "5\tevent_comment \tstring\n", - "6\tevent_user_id \tInt64\n", - "7\tevent_user_text_historical \tstring\n", - "8\tevent_user_text \tstring\n", - "9\tevent_user_blocks_historical \tobject\n", - "10\tevent_user_blocks \tobject\n", - "11\tevent_user_groups_historical \tobject\n", - "12\tevent_user_groups \tobject\n", - "13\tevent_user_is_bot_by_historical \tobject\n", - "14\tevent_user_is_bot_by \tobject\n", - "15\tevent_user_is_created_by_self \tboolean\n", - "16\tevent_user_is_created_by_system \tboolean\n", - "17\tevent_user_is_created_by_peer \tboolean\n", - "18\tevent_user_is_anonymous \tboolean\n", - "19\tevent_user_is_temporary \tboolean\n", - "20\tevent_user_is_permanent \tboolean\n", - "21\tevent_user_registration_timestamp \tobject\n", - "22\tevent_user_creation_timestamp \tobject\n", - "23\tevent_user_first_edit_timestamp \tobject\n", - "24\tevent_user_revision_count \tInt64\n", - "25\tevent_user_seconds_since_previous_revision \tInt64\n", - "26\tpage_id \tInt64\n", - "27\tpage_title_historical \tstring\n", - "28\tpage_title \tstring\n", - "29\tpage_namespace_historical \tInt64\n", - "30\tpage_namespace_is_content_historical \tboolean\n", - "31\tpage_namespace \tInt64\n", - "32\tpage_namespace_is_content \tboolean\n", - "33\tpage_is_redirect \tboolean\n", - "34\tpage_is_deleted \tboolean\n", - "35\tpage_creation_timestamp \tobject\n", - "36\tpage_first_edit_timestamp \tobject\n", - "37\tpage_revision_count \tInt64\n", - "38\tpage_seconds_since_previous_revision \tInt64\n", - "39\tuser_id \tInt64\n", - "40\tuser_text_historical \tstring\n", - "41\tuser_text \tstring\n", - "42\tuser_blocks_historical \tobject\n", - "43\tuser_blocks \tobject\n", - "44\tuser_groups_historical \tobject\n", - "45\tuser_groups \tobject\n", - "46\tuser_is_bot_by_historical \tobject\n", - "47\tuser_is_bot_by \tobject\n", - "48\tuser_is_created_by_self \tboolean\n", - "49\tuser_is_created_by_system \tboolean\n", - "50\tuser_is_created_by_peer \tboolean\n", - "51\tuser_is_anonymous \tboolean\n", - "52\tuser_is_temporary \tboolean\n", - "53\tuser_is_permanent \tboolean\n", - "54\tuser_registration_timestamp \tobject\n", - "55\tuser_creation_timestamp \tobject\n", - "56\tuser_first_edit_timestamp \tobject\n", - "57\trevision_id \tInt64\n", - "58\trevision_parent_id \tInt64\n", - "59\trevision_minor_edit \tboolean\n", - "60\trevision_deleted_parts \tobject\n", - "61\trevision_deleted_parts_are_suppressed \tboolean\n", - "62\trevision_text_bytes \tInt64\n", - "63\trevision_text_bytes_diff \tInt64\n", - "64\trevision_text_sha1 \tstring\n", - "65\trevision_content_model \tstring\n", - "66\trevision_content_format \tstring\n", - "67\trevision_is_deleted_by_page_deletion \tboolean\n", - "68\trevision_deleted_by_page_deletion_timestamp \tobject\n", - "69\trevision_is_identity_reverted \tboolean\n", - "70\trevision_first_identity_reverting_revision_id \tInt64\n", - "71\trevision_seconds_to_identity_revert \tInt64\n", - "72\trevision_is_identity_revert \tboolean\n", - "73\trevision_is_from_before_page_creation \tboolean\n", - "74\trevision_tags \tobject\n", - "id\tfield \tdtype\n", - "----\t------------------------------------------------------------\t------\n", - "4\tevent_timestamp \tobject\n", - "21\tevent_user_registration_timestamp \tobject\n", - "22\tevent_user_creation_timestamp \tobject\n", - "23\tevent_user_first_edit_timestamp \tobject\n", - "35\tpage_creation_timestamp \tobject\n", - "36\tpage_first_edit_timestamp \tobject\n", - "54\tuser_registration_timestamp \tobject\n", - "55\tuser_creation_timestamp \tobject\n", - "56\tuser_first_edit_timestamp \tobject\n", - "68\trevision_deleted_by_page_deletion_timestamp \tobject\n" - ] - } - ], - "source": [ - "# +\n", - "maxl = 60\n", - "\n", - "print(f\"id\\t{'field': <{maxl}}\\tdtype\")\n", - "print(\"----\\t\" + \"-\" * maxl + \"\\t\" + \"------\")\n", - "for id, field in enumerate(CSV_FIELDS, start=1):\n", - " print(f\"{id}\\t{field: <{maxl}}\\t{CSV_FIELDS_META[field]['dtype']}\")\n", - "\n", - "# +\n", - "timestamp_fields = [\n", - " (id, field) for id, field in enumerate(CSV_FIELDS, start=1) if \"timestamp\" in field\n", - "]\n", - "\n", - "print(f\"id\\t{'field': <{maxl}}\\tdtype\")\n", - "print(\"----\\t\" + \"-\" * maxl + \"\\t\" + \"------\")\n", - "for id, field in timestamp_fields:\n", - " print(f\"{id}\\t{field: <{maxl}}\\t{CSV_FIELDS_META[field]['dtype']}\")\n", - "\n", - "# +" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (14,15,16,17,27,29,31,37,38,41,42,43,44,45,46,47,48,49,50,51) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,18,19,39,40,41,42,43,44,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,41,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,13,14,15,16,17,39,40,41,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,12,13,17,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,12,13,17,42,43,44,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,17,19,39,40,41,42,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,19,39,40,41,42,43,44,50,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,14,15,16,17,27,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (41,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,8,9,10,11,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,10,14,15,16,17,39,41,42,43,44,51,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,12,13,14,15,16,17,27,29,31,37,38,39,40,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,13,14,15,16,17,42,43,44,51,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,39,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,11,12,13,14,15,16,17,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,42,51,63) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,8,9,10,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,59,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,39,40,41,42,43,44,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,12,13,14,15,16,17,27,29,30,31,37,38,42,43,44,45,46,47,48,49,50,51,54,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,12,13,17,42,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,39,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,12,13,17,39,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,17,19,39,40,41,42,43,44,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,12,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,19,20,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,41,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,39,40,41,42,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,12,13,14,15,16,17,39,40,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,14,15,16,17,39,40,41,42,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,17,39,41,42,43,44) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,17,39,40,41,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,19,39,40,41,42,43,44,49,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,41,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,14,15,16,17,18,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,11,17,41,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,39,40,41,42,43,44,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,12,17,41,42,43,44,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,17,39,41,42,43,44,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,12,13,14,15,16,17,27,29,31,37,38,39,40,42,43,44,45,46,47,48,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,49,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,13,14,15,16,17,27,29,30,31,37,38,42,43,44,45,46,47,48,49,50,51,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,12,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,14,15,16,17,18,19,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,41,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,43,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,12,17,39,40,41,42,43,44,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,13,17,42,43,44,63) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,17,39,40,42,43,44,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,8,9,10,11,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,59,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,9,11,30,39,40,42,51,54,56,59,62,63,64,67,68) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,8,9,10,11,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,59,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,12,13,17,41,42,43,44) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,12,14,15,16,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,19,39,40,41,42,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,41,42,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,12,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,8,9,10,12,13,14,15,16,17,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,59,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,14,15,16,17,18,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,40,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,13,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,12,17,41,42,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,12,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,17,18,19,39,40,41,42,43,44,49,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,17,18,19,39,40,41,42,43,44,49,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,18,19,27,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,42,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,11,39,40,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,39,40,41,42,43,44,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (4,11,12,13,17,30,39,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,41,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,12,13,14,15,16,17,39,40,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,17,41) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,13,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,39,40,41,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,12,39,40,42,43,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,39,40,41,42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,19,20,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,18,19,39,40,41,42,43,44,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,19,20,27,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,18,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,41,43,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,14,15,16,17,39,40,41,42,43,44,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,12,17,39,40,41,42,43,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,12,14,15,16,17,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,42,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,10,11,14,15,16,17,27,29,30,31,37,38,39,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,19,39,40,41,42,43,44,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,19,39,40,41,42,43,44,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,27,29,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,18,19,39,40,41,42,43,44,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,14,15,16,17,27,29,30,31,37,38,39,40,42,43,44,45,46,47,48,49,50,51,54,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,11,12,13,14,15,16,17,18,19,20,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,19,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,12,17,42,43,44,63) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,41,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,12,17,39,40,41,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,39,40,41,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,54,55,56,62,63,64,67,68,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,12,14,15,16,17,39,40,42,43,44,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,12,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,12,13,17,39,40,41,42,43,44,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,39,40,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,11,12,13,14,15,16,17,18,19,20,27,29,31,37,38,41,42,43,44,45,46,47,48,49,50,51,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,19,20,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,14,15,16,17,27,29,30,31,37,38,39,40,42,43,44,45,46,47,48,49,50,51,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,13,14,15,16,17,27,29,30,31,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,19,39,40,41,42,49,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (42,55) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,39,40,41,42,43,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,17,39,40,41,42,43,44,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,39,40,41,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,14,15,16,17,19,39,40,41,42,43,44,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,11,12,14,15,16,17,27,37,38,39,40,42,43,44,45,46,47,48,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,11,39,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,12,14,15,16,17,39,40,41,42,43,44,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,42,43,44,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,14,15,16,17,18,19,27,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,41,42,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,11,12,13,14,15,16,17,18,19,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,55,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,39,40,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,12,13,17,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,13,17,42,43,44,51,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (14,15,16,17,42,43,44,63) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (17,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,42) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (10,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,12,17,39,40,42,43,44,63,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,19,39,40,42,50,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (11,17,42,43,44) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (12,13,17,42,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (13,17,42,43,44,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,10,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,17,39,40,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (9,17,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,12,13,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,39,40,41,42,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n", - "/tmp/ipykernel_1030433/720178351.py:4: DtypeWarning: Columns (8,9,10,12,17,39,40,41,42,43,44,55,69) have mixed types. Specify dtype option on import or set low_memory=False.\n", - " tmpdf = pd.read_csv(\n" - ] - }, - { - "ename": "", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n", - "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n", - "\u001b[1;31mClick here for more info. \n", - "\u001b[1;31mView Jupyter log for further details." - ] - } - ], - "source": [ - "df_list = []\n", - "\n", - "for file in csv_files:\n", - " tmpdf = pd.read_csv(\n", - " file,\n", - " delimiter=\"\\t\",\n", - " encoding=\"utf-8\",\n", - " quotechar='\"',\n", - " quoting=csv.QUOTE_NONE,\n", - " doublequote=False,\n", - " header=None,\n", - " names=CSV_FIELDS,\n", - " #dtype={field: CSV_FIELDS_META[field][\"dtype\"] for field in CSV_FIELDS},\n", - " #date_format={\n", - " # field: \"%Y-%m-%d %H:%M:%S.%f\"\n", - " # for field in CSV_FIELDS\n", - " # if \"timestamp\" in field\n", - " #},\n", - " )\n", - " df_list.append(tmpdf)\n", - "\n", - "df = pd.concat(df_list)\n", - "df.head()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.15" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}