Start working on adding columns from mwparserfromhell.

src/wikiq/__init__.py

@@ -27,6 +27,7 @@ import asyncio
 import wikiq.tables as tables
 from wikiq.tables import RevisionTable
 from wikiq.wiki_diff_matcher import WikiDiffMatcher
+from wikiq.wikitext_parser import WikitextParser
 
 TO_ENCODE = ("title", "editor")
 PERSISTENCE_RADIUS = 7
@@ -242,6 +243,8 @@ class WikiqParser:
         batch_size: int = 1024,
         partition_namespaces: bool = False,
         resume_from_revid: int = None,
+        external_links: bool = False,
+        citations: bool = False,
     ):
         """
         Parameters:
@@ -258,6 +261,8 @@ class WikiqParser:
         self.text = text
         self.partition_namespaces = partition_namespaces
         self.resume_from_revid = resume_from_revid
+        self.external_links = external_links
+        self.citations = citations
         if namespaces is not None:
             self.namespace_filter = set(namespaces)
         else:
@@ -397,6 +402,16 @@ class WikiqParser:
         if self.collapse_user:
             table.columns.append(tables.RevisionCollapsed())
 
+        # Create shared parser if either wikitext feature is enabled
+        if self.external_links or self.citations:
+            wikitext_parser = WikitextParser()
+
+        if self.external_links:
+            table.columns.append(tables.RevisionExternalLinks(wikitext_parser))
+
+        if self.citations:
+            table.columns.append(tables.RevisionCitations(wikitext_parser))
+
         # extract list of namespaces
         self.namespaces = {
             ns.name: ns.id for ns in dump.mwiterator.site_info.namespaces
@@ -1135,6 +1150,22 @@ def main():
         help="Output the text of the revision.",
     )
 
+    parser.add_argument(
+        "--external-links",
+        dest="external_links",
+        action="store_true",
+        default=False,
+        help="Extract external links from each revision using mwparserfromhell.",
+    )
+
+    parser.add_argument(
+        "--citations",
+        dest="citations",
+        action="store_true",
+        default=False,
+        help="Extract citations (ref tags and cite templates) from each revision.",
+    )
+
     parser.add_argument(
         "-PNS",
         "--partition-namespaces",
@@ -1239,6 +1270,8 @@ def main():
                 partition_namespaces=args.partition_namespaces,
                 batch_size=args.batch_size,
                 resume_from_revid=resume_from_revid,
+                external_links=args.external_links,
+                citations=args.citations,
             )
 
             wikiq.process()
@@ -1266,6 +1299,8 @@ def main():
             text=args.text,
             batch_size=args.batch_size,
             resume_from_revid=None,
+            external_links=args.external_links,
+            citations=args.citations,
         )
 
         wikiq.process()