collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

parquet path is code-complete

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2021-10-17 21:46:31 -07:00
parent 26f6d8f984
commit ae870fed0b
1 changed files with 58 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
wikiq
View File

@ -25,7 +25,8 @@ PERSISTENCE_RADIUS=7
from deltas import SequenceMatcher
from deltas import SegmentMatcher
from dataclasses import dataclass
import dataclasses as dc
from dataclasses import dataclass, make_dataclass
import pyarrow as pa
import pyarrow.parquet as pq
@ -39,7 +40,6 @@ def calculate_persistence(tokens_added):
return(sum([(len(x.revisions)-1) for x in tokens_added]),
len(tokens_added))
class WikiqIterator():
def __init__(self, fh, collapse_user=False):
self.fh = fh
@ -187,8 +187,8 @@ class RegexPair(object):
temp_dict[self.label] = None
# update rev_data with our new columns
for k, v in temp_dict:
rev_data.setattr(k,v)
for k, v in temp_dict.items():
setattr(rev_data, k, v)
return rev_data
@ -208,28 +208,26 @@ class RevDataBase():
minor: bool = None
editor: str = None
anon: bool = None
collapsed_revs:int = None
pa_schema_fields = [
pa.field("revid", pa.int64),
pa.field("revid", pa.int64()),
pa.field("date_time",pa.timestamp('ms')),
pa.field("articleid",pa.int64()),
pa.field("editorid",pa.int64()),
pa.field("title",pa.string()),
pa.field("namespace",pa.int32()),
pa.field("deleted",pa.binary()),
pa.field("deleted",pa.bool_()),
pa.field("test_chars",pa.int32()),
pa.field("revert",pa.binary()),
pa.field("revert",pa.bool_()),
pa.field("reverteds",pa.list_(pa.int64())),
pa.field("sha1",pa.string()),
pa.field("minor",pa.binary()),
pa.field("minor",pa.bool_()),
pa.field("editor",pa.string()),
pa.field("anon",pa.binary())
pa.field("anon",pa.bool_())
]
def to_pyarrow(self):
return pa.array(self.astuple(), map(self.pa_schema_fields, pa.field.type))
return dc.astuple(self)
def to_tsv_row(self):
@ -281,8 +279,7 @@ class RevDataBase():
class RevDataCollapse(RevDataBase):
collapsed_revs:int = None
pa_collapsed_revs_schema = pa.field('collapsed_revs',pa.int64())
pa_schema_fields = RevDataBase.pa_schema_fields + pa_collapsed_revs_schema
pa_schema = pa.schema(pa_schema_fields)
pa_schema_fields = RevDataBase.pa_schema_fields + [pa_collapsed_revs_schema]
@dataclass()
class RevDataPersistence(RevDataBase):
@ -290,17 +287,18 @@ class RevDataPersistence(RevDataBase):
tokens_added:int = None
tokens_removed:int = None
tokens_window:int = None
pa_persistence_schema_fields = [
pa.field(token_revs, pa.int64()),
pa.field(tokens_added, pa.int64()),
pa.field(tokens_removed, pa.int64()),
pa.tokens_window, pa.int64()]
pa.field("token_revs", pa.int64()),
pa.field("tokens_added", pa.int64()),
pa.field("tokens_removed", pa.int64()),
pa.field("tokens_window", pa.int64())]
pa_schema_fields = RevDataBase.pa_schema_fields + pa_persistence_schema_fields
@dataclass()
class RevDataCollapsePersistence(RevDataCollapse, RevDataPersistence):
pa_scehma_fields = RevDataCollapse.pa_schema_fields + RevDataPersistence.pa_persistence_schema_fields
pa_schema_fields = RevDataCollapse.pa_schema_fields + RevDataPersistence.pa_persistence_schema_fields
class WikiqParser():
def __init__(self, input_file, output_file, regex_match_revision, regex_match_comment, regex_revision_label, regex_comment_label, collapse_user=False, persist=None, urlencode=False, namespaces = None, revert_radius=15, output_parquet=True, parquet_buffer_size=2000):
@ -316,9 +314,6 @@ class WikiqParser():
self.urlencode = urlencode
self.revert_radius = revert_radius
self.output_buffer = []
self.output_buffer_size = output_buffer_size
if namespaces is not None:
self.namespace_filter = set(namespaces)
else:
@ -338,20 +333,24 @@ class WikiqParser():
else:
revdata_type = RevDataBase
regex_fields = [(field.name, list[str]), for field in self.regex_schemas]
self.revdata_type = dataclasses.make_dataclass('RevData_Parser',
fields=map(regex_fields,
lambda pa_field: (pa_field.name,
list[string],
field(default=None))),
bases=(revdata_type))
regex_fields = [(field.name, list[str], dc.field(default=None)) for field in self.regex_schemas]
self.revdata_type = make_dataclass('RevData_Parser',
fields=regex_fields,
bases=(revdata_type,))
self.revdata_type.pa_schema_fields = revdata_type.pa_schema_fields + regex_fields
self.revdata_type.pa_schema_fields = revdata_type.pa_schema_fields + self.regex_schemas
# print(list(map(lambda d: d.name, dc.fields(self.revdata_type))))
# print(self.revdata_type.pa_schema_fields)
if output_parquet is True:
self.output_parquet = True
self.pq_writer = None
self.output_file = output_file
self.parquet_buffer = []
self.parquet_buffer_size = parquet_buffer_size
else:
self.output_file = open(output_file,'w')
@ -450,11 +449,12 @@ class WikiqParser():
for rev in page:
rev_data = self.revdata_type(revid = rev.id,
date_time = rev.timestamp,
date_time = datetime.fromtimestamp(rev.timestamp.unix()),
articleid = page.id,
editorid = "" if rev.deleted.user == True or rev.user.id is None else rev.user.id,
title = page.title,
deleted = rev.deleted.text
deleted = rev.deleted.text,
namespace = namespace
)
rev_data = self.matchmake(rev, rev_data)
@ -477,10 +477,13 @@ class WikiqParser():
rev_data.text_chars = len(rev.text)
# generate revert data
rev_data.revert = rev_detector.process(text_sha1, rev.id)
revert = rev_detector.process(text_sha1, rev.id)
if revert:
rev_data.revert = True
rev_data.reverteds = revert.reverteds
else:
rev_data.revert = False
# if the fact that the edit was minor can be hidden, this might be an issue
rev_data.minor = rev.minor
@ -561,16 +564,32 @@ class WikiqParser():
def write_parquet_row(self, rev_data):
padata = rev_data.to_pyarrow()
self.output_buffer.append(padata)
self.parquet_buffer.append(padata)
if len(self.output_buffer) >= self.output_buffer_size:
if len(self.parquet_buffer) >= self.parquet_buffer_size:
self.flush_parquet_buffer()
def flush_parquet_buffer(self):
outtable = pa.table.concat_arrays(self.output_buffer)
schema = pa.schema(self.revdata_type.pa_schema_fields)
def row_to_col(rg, types):
cols = []
first = rg[0]
for col in first:
cols.append([col])
for row in rg[1:]:
for j in range(len(cols)):
cols[j].append(row[j])
arrays = []
for col, typ in zip(cols, types):
arrays.append(pa.array(col, typ))
return arrays
outtable = pa.Table.from_arrays(row_to_col(self.parquet_buffer, schema.types), schema=schema)
if self.pq_writer is None:
schema = pa.schema(self.revdata_type.pa_schema_field)
self.pq_writer = pq.ParquetWriter(self.output_file, schema, flavor='spark')
self.pq_writer.write_table(outtable)
@ -585,15 +604,9 @@ class WikiqParser():
printfunc(rev_data)
def write_tsv_row(self, rev_data):
self.output_buffer.append(rev_data.to_tsv_line())
line = rev_data.to_tsv_line()
print(line, file=self.output_file)
if len(self.output_buffer) >= self.output_buffer_size:
self.flush_tsv_buffer()
def flush_tsv_buffer():
if self.output_header:
def open_input_file(input_filename):
if re.match(r'.*\.7z$', input_filename):