Get columnar refactor partially working
Noargs works, now to do persistence. Signed-off-by: Will Beason <willbeason@gmail.com>
This commit is contained in:
Will Beason
124
tables.py
124
tables.py
@@ -1,8 +1,10 @@
|
||||
import sys
|
||||
from abc import abstractmethod, ABC
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha1
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import mwreverts
|
||||
import mwtypes
|
||||
import mwxml
|
||||
|
||||
@@ -12,12 +14,17 @@ T = TypeVar('T')
|
||||
|
||||
|
||||
class RevisionField(ABC, Generic[T]):
|
||||
def __init__(self):
|
||||
self.data: list[T] = []
|
||||
|
||||
"""
|
||||
Abstract type which represents a field in a table of page revisions.
|
||||
"""
|
||||
|
||||
def __init__(self, field: pa.Field):
|
||||
self.field = field
|
||||
@property
|
||||
@abstractmethod
|
||||
def field(self) -> pa.Field:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> T:
|
||||
@@ -29,51 +36,74 @@ class RevisionField(ABC, Generic[T]):
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class RevisionTableColumn(Generic[T]):
|
||||
def __init__(self, field: RevisionField[T]):
|
||||
self.field: RevisionField = field
|
||||
self.data: list[T] = []
|
||||
|
||||
def add(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> None:
|
||||
self.data.append(self.field.extract(page, revisions))
|
||||
self.data.append(self.extract(page, revisions))
|
||||
|
||||
def pop_column(self) -> list[T]:
|
||||
def pop(self) -> list[T]:
|
||||
data = self.data
|
||||
self.data = []
|
||||
return data
|
||||
|
||||
|
||||
class RevisionTable:
|
||||
columns: list[RevisionTableColumn]
|
||||
columns: list[RevisionField]
|
||||
|
||||
def add_revision_set(self, page: mwtypes.Page, revisions: list[mwxml.Revision]):
|
||||
def __init__(self, columns: list[RevisionField]):
|
||||
self.columns = columns
|
||||
|
||||
def add(self, page: mwtypes.Page, revisions: list[mwxml.Revision]):
|
||||
for column in self.columns:
|
||||
column.add(page, revisions)
|
||||
column.add(page=page, revisions=revisions)
|
||||
|
||||
def schema(self) -> pa.Schema:
|
||||
return pa.schema([c.field for c in self.columns])
|
||||
|
||||
def pop(self):
|
||||
schema = self.schema()
|
||||
data = []
|
||||
for column in self.columns:
|
||||
data.append(column.pop())
|
||||
|
||||
return pa.table(data, schema)
|
||||
|
||||
|
||||
class RevisionId(RevisionField[int]):
|
||||
field = pa.field("revid", pa.int64())
|
||||
|
||||
def extract(self, _: mwtypes.Page, revisions: list[mwxml.Revision]) -> int:
|
||||
revision = revisions[-1]
|
||||
return revision.id
|
||||
|
||||
|
||||
class RevisionTimestamp(RevisionField[datetime]):
|
||||
field = pa.field("date_time", pa.timestamp('s'))
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> datetime:
|
||||
revision = revisions[-1]
|
||||
return revision.timestamp
|
||||
|
||||
|
||||
class RevisionArticleId(RevisionField[int]):
|
||||
field = pa.field("articleid", pa.int64())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> int:
|
||||
return page.id
|
||||
|
||||
|
||||
class RevisionEditorId(RevisionField[int | None]):
|
||||
field = pa.field("editorid", pa.int64(), nullable=True)
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> int | None:
|
||||
revision = revisions[-1]
|
||||
if revision.deleted.user or revision.user.id is None:
|
||||
if revision.deleted.user:
|
||||
return None
|
||||
|
||||
return revision.user.id
|
||||
|
||||
|
||||
class RevisionAnon(RevisionField[bool | None]):
|
||||
class RevisionIsAnon(RevisionField[bool | None]):
|
||||
field = pa.field("anon", pa.bool_(), nullable=True)
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> bool | None:
|
||||
revision = revisions[-1]
|
||||
if revision.deleted.user:
|
||||
@@ -83,6 +113,8 @@ class RevisionAnon(RevisionField[bool | None]):
|
||||
|
||||
|
||||
class RevisionEditorText(RevisionField[str | None]):
|
||||
field = pa.field("editor", pa.string(), nullable=True)
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> str | None:
|
||||
revision = revisions[-1]
|
||||
if revision.deleted.user:
|
||||
@@ -92,43 +124,87 @@ class RevisionEditorText(RevisionField[str | None]):
|
||||
|
||||
|
||||
class RevisionPageTitle(RevisionField[str]):
|
||||
field = pa.field("title", pa.string())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> str:
|
||||
return page.title
|
||||
|
||||
|
||||
class RevisionDeleted(RevisionField[bool]):
|
||||
field = pa.field("deleted", pa.bool_())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> bool:
|
||||
revision = revisions[-1]
|
||||
return revision.deleted.text
|
||||
|
||||
|
||||
class RevisionNamespace(RevisionField[int]):
|
||||
field = pa.field("namespace", pa.int32())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> int:
|
||||
return page.namespace
|
||||
|
||||
|
||||
class RevisionSha1(RevisionField[str]):
|
||||
field = pa.field("sha1", pa.string())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> str:
|
||||
revision = revisions[-1]
|
||||
if revision.sha1:
|
||||
return revision.sha1
|
||||
|
||||
return sha1(revision.sha1).hexdigest()
|
||||
return revision.sha1
|
||||
|
||||
|
||||
class RevisionTextChars(RevisionField[int]):
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> int:
|
||||
class RevisionTextChars(RevisionField[int | None]):
|
||||
field = pa.field("text_chars", pa.int32(), nullable=True)
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> int | None:
|
||||
revision = revisions[-1]
|
||||
return len(revision.text)
|
||||
if not revision.deleted.text:
|
||||
return len(revision.text)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class RevisionMinor(RevisionField[bool]):
|
||||
class RevisionText(RevisionField[str]):
|
||||
field = pa.field("text", pa.string())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> str:
|
||||
revision = revisions[-1]
|
||||
return revision.text
|
||||
|
||||
|
||||
class RevisionIsMinor(RevisionField[bool]):
|
||||
field = pa.field("minor", pa.bool_())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> bool:
|
||||
revision = revisions[-1]
|
||||
return revision.minor
|
||||
|
||||
|
||||
class RevisionCollapse(RevisionField[int]):
|
||||
class RevisionReverts(RevisionField[str | None]):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.rev_detector: mwreverts.Detector | None = None
|
||||
|
||||
field = pa.field("reverteds", pa.string(), nullable=True)
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> str | None:
|
||||
if self.rev_detector is None:
|
||||
return None
|
||||
|
||||
revision = revisions[-1]
|
||||
if revision.deleted.text:
|
||||
return None
|
||||
|
||||
revert = self.rev_detector.process(revision.sha1, revision.id)
|
||||
if revert is None:
|
||||
return None
|
||||
|
||||
return ",".join([str(s) for s in revert.reverteds])
|
||||
|
||||
|
||||
class RevisionCollapsed(RevisionField[int]):
|
||||
field = pa.field("collapsed_revs", pa.int64())
|
||||
|
||||
def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> int:
|
||||
return len(revisions)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user