add test files

test/Wikiq_Unit_Test.py

@@ -19,6 +19,7 @@ from io import StringIO
 class Test_Persistence(unittest.TestCase):
 
     def setUp(self):
+
         if not os.path.exists("test_output"):
             os.mkdir("test_output")
 
@@ -40,7 +41,7 @@ class Test_Persistence(unittest.TestCase):
             os.remove(test_file)
         
         call = self.base_call.format(self.input_file, self.test_output_dir)
-        call = call + " --url-encode --persistence sequence --collapse-user"
+        call = call + " --url-encode --persistence sequence"
         print(os.path.abspath('.'))
         print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
@@ -51,17 +52,17 @@ class Test_Persistence(unittest.TestCase):
 
         # as a test let's make sure that we get equal data frames
         test = pd.read_table(test_file)
-        self.assertEqual(test['tokens_added'][0],4)
+        self.assertEqual(test['tokens_added'][0],7)
-        self.assertEqual(test['tokens_added'][1],5)
+        self.assertEqual(test['tokens_added'][1],10)
         self.assertEqual(test['tokens_added'][2],0)
-        self.assertEqual(test['tokens_added'][3],6)
+        self.assertEqual(test['tokens_added'][3],8)
-        self.assertEqual(test['tokens_added'][4],4)
+        self.assertEqual(test['tokens_added'][4],0)
         self.assertEqual(test['tokens_removed'][0],0)
         self.assertEqual(test['tokens_removed'][1],0)
-        self.assertEqual(test['tokens_removed'][2],5)
+        self.assertEqual(test['tokens_removed'][2],0)
         self.assertEqual(test['tokens_removed'][3],4)
-        self.assertEqual(test['tokens_removed'][4],6)
+        self.assertEqual(test['tokens_removed'][4],0)
-        self.assertEqual(test['token_revs'][0],4*3)
+        self.assertEqual(test['token_revs'][0],8*3)
         self.assertEqual(test['token_revs'][1],0)
         self.assertEqual(test['token_revs'][2],0)
         self.assertEqual(test['token_revs'][3],0)
@@ -69,7 +70,44 @@ class Test_Persistence(unittest.TestCase):
 
         baseline = pd.read_table(baseline_file)
         assert_frame_equal(test,baseline)
+
+    def test_legacy_persistence(self):
+        test_filename =  "legacy-" + self.wikiq_out_name
+        test_file = os.path.join(self.test_output_dir, test_filename)
+        if os.path.exists(test_file):
+            os.remove(test_file)
         
+        call = self.base_call.format(self.input_file, self.test_output_dir)
+        call = call + " --url-encode --persistence legacy"
+        print(os.path.abspath('.'))
+        print(call)
+        proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
+        proc.wait()
+
+        copyfile(self.call_output, test_file)
+        baseline_file = os.path.join(".", self.baseline_output_dir, test_filename)
+
+        # as a test let's make sure that we get equal data frames
+        test = pd.read_table(test_file)
+        self.assertEqual(test['tokens_added'][0],7)
+        self.assertEqual(test['tokens_added'][1],10)
+        self.assertEqual(test['tokens_added'][2],0)
+        self.assertEqual(test['tokens_added'][3],11)
+        self.assertEqual(test['tokens_added'][4],0)
+        self.assertEqual(test['tokens_removed'][0],0)
+        self.assertEqual(test['tokens_removed'][1],0)
+        self.assertEqual(test['tokens_removed'][2],0)
+        self.assertEqual(test['tokens_removed'][3],7)
+        self.assertEqual(test['tokens_removed'][4],0)
+        self.assertEqual(test['token_revs'][0],7*3)
+        self.assertEqual(test['token_revs'][1],0)
+        self.assertEqual(test['token_revs'][2],0)
+        self.assertEqual(test['token_revs'][3],0)
+        self.assertEqual(test['token_revs'][4],0)
+
+        baseline = pd.read_table(baseline_file)
+        assert_frame_equal(test,baseline)
+
 
 
 class Test_Persistence_Bug(unittest.TestCase):
@@ -135,6 +173,7 @@ class Test_Wikipedia(unittest.TestCase):
         
         call = self.base_call.format(self.input_file, self.test_output_dir)
         call = call + " --url-encode"
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
         proc.wait()
 
@@ -195,6 +234,7 @@ class Test_Basic(unittest.TestCase):
             os.remove(test_file)
         
         call = self.base_call.format(self.input_file, self.test_output_dir)
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
         proc.wait()
 
@@ -216,6 +256,7 @@ class Test_Basic(unittest.TestCase):
         call = self.base_call.format(self.input_file, self.test_output_dir)
         call = call + " --collapse-user"
 
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
         proc.wait()
 
@@ -256,6 +297,7 @@ class Test_Basic(unittest.TestCase):
         
         call = self.base_call.format(self.input_file, self.test_output_dir)
         call = call + " --persistence legacy"
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
         proc.wait()
 
@@ -275,6 +317,7 @@ class Test_Basic(unittest.TestCase):
         
         call = self.base_call.format(self.input_file, self.test_output_dir)
         call = call + " --persistence"
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
         proc.wait()
 
@@ -297,7 +340,9 @@ class Test_Basic(unittest.TestCase):
         
         call = self.base_call.format(self.input_file, self.test_output_dir)
         call = call + " --url-encode"
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
+
         proc.wait()
 
         copyfile(self.call_output, test_file)
@@ -326,6 +371,7 @@ class Test_Malformed(unittest.TestCase):
     def test_malformed_noargs(self):
 
         call = self.base_call.format(self.input_file, self.test_output_dir)
+        print(call)
         proc = subprocess.Popen(call,stdout=subprocess.PIPE,stderr=subprocess.PIPE, shell=True)
         proc.wait()
         outs, errs = proc.communicate()
@@ -347,6 +393,7 @@ class Test_Stdout(unittest.TestCase):
     def test_noargs(self):
 
         call = self.base_call.format(self.input_file)
+        print(call)
         proc = subprocess.run(call,stdout=subprocess.PIPE,shell=True)
         outs = proc.stdout.decode("utf8")
 

test/baseline_output/enwiki-test.tsv

@@ -0,0 +1,15 @@
+anon	articleid	collapsed_revs	date_time	deleted	editor	editor_id	minor	namespace	revert	reverteds	revid	sha1	text_chars	title	token_revs	tokens_added	tokens_removed	tokens_window
+TRUE	23397386	3	2010-12-13 02:52:57	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		402074089	bdoleau92eed59p01lwu3hx71jfplyv	54928	%22The%20Amazing%20Race%2016%22	118658	19778	0	6
+FALSE	23397386	1	2010-12-13 04:44:25	FALSE	%22Nmk829%22	12595469	FALSE	0	FALSE		402088763	7zt0ygstumo5fe0fj2xj32mcn1p4p1j	54926	%22The%20Amazing%20Race%2016%22	0	0	0	6
+FALSE	23397386	1	2010-12-14 18:31:01	FALSE	%22Ryulong%22	889851	FALSE	0	FALSE		402376185	2itumo8i1amz1anlmnvnqcaauk0c6ti	54912	%22The%20Amazing%20Race%2016%22	0	0	6	6
+TRUE	23397386	1	2010-12-14 23:12:50	FALSE	%22174.1.48.24%22		FALSE	0	FALSE		402421155	27izchr7l6uj5z0p9jc9w8mf5h45w8p	54973	%22The%20Amazing%20Race%2016%22	114	19	0	6
+FALSE	23397386	1	2010-12-14 23:18:49	FALSE	%22Ryulong%22	889851	FALSE	0	FALSE		402422015	60nmgy7reem6isznmtnjr37qwj3akkb	55013	%22The%20Amazing%20Race%2016%22	102	17	0	6
+TRUE	23397386	1	2010-12-15 01:05:24	FALSE	%22173.30.1.184%22		FALSE	0	FALSE		402436936	2rf7juf6sx1mi76rgnagt0v1h2vqkka	55033	%22The%20Amazing%20Race%2016%22	0	10	0	6
+FALSE	23397386	1	2010-12-15 04:03:25	FALSE	%22Kuyabribri%22	1904358	FALSE	0	TRUE	"402436936"	402459131	60nmgy7reem6isznmtnjr37qwj3akkb	55013	%22The%20Amazing%20Race%2016%22	0	0	0	6
+TRUE	23397386	1	2010-12-16 15:24:08	FALSE	%22220.255.1.79%22		FALSE	0	FALSE		402702013	p2u5jcoov0c3396cdih5h2v8azrmb5z	55010	%22The%20Amazing%20Race%2016%22	3320	1030	7	6
+TRUE	23397386	1	2010-12-18 10:45:33	FALSE	%2258.168.198.160%22		FALSE	0	FALSE		402999028	h9l6spwatmurwxcwcukky37iiozb8dq	55014	%22The%20Amazing%20Race%2016%22	0	2	0	5
+FALSE	23397386	1	2010-12-19 01:05:33	FALSE	%22Dravecky%22	4588674	TRUE	0	TRUE	"402999028"	403103973	p2u5jcoov0c3396cdih5h2v8azrmb5z	55010	%22The%20Amazing%20Race%2016%22	0	0	0	4
+FALSE	23397386	1	2010-12-22 12:21:27	FALSE	%22Xqbot%22	8066546	TRUE	0	FALSE		403690886	lar0epx20k4qvdj8otqna8v9c1ibtbt	55008	%22The%20Amazing%20Race%2016%22	3162	1054	26	3
+TRUE	23397386	1	2011-01-02 10:37:54	FALSE	%22118.137.242.186%22		FALSE	0	FALSE		405475313	ipyttx2y54xlilzeihwuaoatcmbcotz	54974	%22The%20Amazing%20Race%2016%22	0	0	0	2
+TRUE	23397386	2	2011-01-02 11:08:22	FALSE	%2271.133.162.137%22		FALSE	0	FALSE		405478231	lsxi138j1q7qmim71u33kx0qjb4ema6	54976	%22The%20Amazing%20Race%2016%22	0	0	0	1
+FALSE	23397386	2	2011-01-02 12:19:03	FALSE	%22Dravecky%22	4588674	FALSE	0	TRUE	"405478231,405475313"	405484921	lar0epx20k4qvdj8otqna8v9c1ibtbt	55008	%22The%20Amazing%20Race%2016%22	0	0	0	0

test/baseline_output/pwr-test.tsv

@@ -0,0 +1,6 @@
+anon	articleid	date_time	deleted	editor	editor_id	minor	namespace	revert	reverteds	revid	sha1	text_chars	title	token_revs	tokens_added	tokens_removed	tokens_window
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		None	e867e7cd0e53b58428b23ee346c6fa523756e7d1	21	%22Test%20Title%22	24	7	0	4
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		None	cb3aeff37c151ab439ee4a7a76cea85679272d5e	44	%22Test%20Title%22	0	10	0	3
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		None	cp0mfp4o90fbpyxooys1rw95zn3ddff	21	%22Test%20Title%22	0	0	0	2
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.53%22		FALSE	0	FALSE		None	9e63d8f3d87be0ed4cc3fab3f750a4429997bead	31	%22Test%20Title%22	0	8	4	1
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.60%22		FALSE	0	TRUE	"None,None,None"	None	e867e7cd0e53b58428b23ee346c6fa523756e7d1	21	%22Test%20Title%22	0	0	0	0

test/baseline_output/sequence-enwiki-test.tsv

@@ -0,0 +1,15 @@
+anon	articleid	collapsed_revs	date_time	deleted	editor	editor_id	minor	namespace	revert	reverteds	revid	sha1	text_chars	title	token_revs	tokens_added	tokens_removed	tokens_window
+TRUE	23397386	3	2010-12-13 02:52:57	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		402074089	bdoleau92eed59p01lwu3hx71jfplyv	54928	%22The%20Amazing%20Race%2016%22	118658	19778	0	6
+FALSE	23397386	1	2010-12-13 04:44:25	FALSE	%22Nmk829%22	12595469	FALSE	0	FALSE		402088763	7zt0ygstumo5fe0fj2xj32mcn1p4p1j	54926	%22The%20Amazing%20Race%2016%22	0	0	0	6
+FALSE	23397386	1	2010-12-14 18:31:01	FALSE	%22Ryulong%22	889851	FALSE	0	FALSE		402376185	2itumo8i1amz1anlmnvnqcaauk0c6ti	54912	%22The%20Amazing%20Race%2016%22	0	0	6	6
+TRUE	23397386	1	2010-12-14 23:12:50	FALSE	%22174.1.48.24%22		FALSE	0	FALSE		402421155	27izchr7l6uj5z0p9jc9w8mf5h45w8p	54973	%22The%20Amazing%20Race%2016%22	114	19	0	6
+FALSE	23397386	1	2010-12-14 23:18:49	FALSE	%22Ryulong%22	889851	FALSE	0	FALSE		402422015	60nmgy7reem6isznmtnjr37qwj3akkb	55013	%22The%20Amazing%20Race%2016%22	102	17	0	6
+TRUE	23397386	1	2010-12-15 01:05:24	FALSE	%22173.30.1.184%22		FALSE	0	FALSE		402436936	2rf7juf6sx1mi76rgnagt0v1h2vqkka	55033	%22The%20Amazing%20Race%2016%22	0	10	0	6
+FALSE	23397386	1	2010-12-15 04:03:25	FALSE	%22Kuyabribri%22	1904358	FALSE	0	TRUE	"402436936"	402459131	60nmgy7reem6isznmtnjr37qwj3akkb	55013	%22The%20Amazing%20Race%2016%22	0	0	0	6
+TRUE	23397386	1	2010-12-16 15:24:08	FALSE	%22220.255.1.79%22		FALSE	0	FALSE		402702013	p2u5jcoov0c3396cdih5h2v8azrmb5z	55010	%22The%20Amazing%20Race%2016%22	3320	1030	7	6
+TRUE	23397386	1	2010-12-18 10:45:33	FALSE	%2258.168.198.160%22		FALSE	0	FALSE		402999028	h9l6spwatmurwxcwcukky37iiozb8dq	55014	%22The%20Amazing%20Race%2016%22	0	2	0	5
+FALSE	23397386	1	2010-12-19 01:05:33	FALSE	%22Dravecky%22	4588674	TRUE	0	TRUE	"402999028"	403103973	p2u5jcoov0c3396cdih5h2v8azrmb5z	55010	%22The%20Amazing%20Race%2016%22	0	0	0	4
+FALSE	23397386	1	2010-12-22 12:21:27	FALSE	%22Xqbot%22	8066546	TRUE	0	FALSE		403690886	lar0epx20k4qvdj8otqna8v9c1ibtbt	55008	%22The%20Amazing%20Race%2016%22	3162	1054	26	3
+TRUE	23397386	1	2011-01-02 10:37:54	FALSE	%22118.137.242.186%22		FALSE	0	FALSE		405475313	ipyttx2y54xlilzeihwuaoatcmbcotz	54974	%22The%20Amazing%20Race%2016%22	0	0	0	2
+TRUE	23397386	2	2011-01-02 11:08:22	FALSE	%2271.133.162.137%22		FALSE	0	FALSE		405478231	lsxi138j1q7qmim71u33kx0qjb4ema6	54976	%22The%20Amazing%20Race%2016%22	0	0	0	1
+FALSE	23397386	2	2011-01-02 12:19:03	FALSE	%22Dravecky%22	4588674	FALSE	0	TRUE	"405478231,405475313"	405484921	lar0epx20k4qvdj8otqna8v9c1ibtbt	55008	%22The%20Amazing%20Race%2016%22	0	0	0	0

test/baseline_output/sequence-pwr-test.tsv

@@ -0,0 +1,6 @@
+anon	articleid	date_time	deleted	editor	editor_id	minor	namespace	revert	reverteds	revid	sha1	text_chars	title	token_revs	tokens_added	tokens_removed	tokens_window
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		None	e867e7cd0e53b58428b23ee346c6fa523756e7d1	21	%22Test%20Title%22	12	4	0	4
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		None	cb3aeff37c151ab439ee4a7a76cea85679272d5e	44	%22Test%20Title%22	0	5	0	3
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.52%22		FALSE	0	FALSE		None	cp0mfp4o90fbpyxooys1rw95zn3ddff	21	%22Test%20Title%22	0	0	0	2
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.53%22		FALSE	0	FALSE		None	9e63d8f3d87be0ed4cc3fab3f750a4429997bead	31	%22Test%20Title%22	0	6	4	1
+TRUE	1	2010-12-13 02:51:55	FALSE	%2276.102.106.60%22		FALSE	0	TRUE	"None,None,None"	None	e867e7cd0e53b58428b23ee346c6fa523756e7d1	21	%22Test%20Title%22	0	0	0	0

test/dumps/enwiki-test.xml~

@@ -0,0 +1,117 @@
+<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd" version="0.10" xml:lang="en">
+  <siteinfo>
+    <sitename>Wikipedia</sitename>
+    <dbname>enwiki</dbname>
+    <base>https://en.wikipedia.org/wiki/Main_Page</base>
+    <generator>MediaWiki 1.31.0-wmf.22</generator>
+    <case>first-letter</case>
+    <namespaces>
+      <namespace key="-2" case="first-letter">Media</namespace>
+      <namespace key="-1" case="first-letter">Special</namespace>
+      <namespace key="0" case="first-letter" />
+      <namespace key="1" case="first-letter">Talk</namespace>
+      <namespace key="2" case="first-letter">User</namespace>
+      <namespace key="3" case="first-letter">User talk</namespace>
+      <namespace key="4" case="first-letter">Wikipedia</namespace>
+      <namespace key="5" case="first-letter">Wikipedia talk</namespace>
+      <namespace key="6" case="first-letter">File</namespace>
+      <namespace key="7" case="first-letter">File talk</namespace>
+      <namespace key="8" case="first-letter">MediaWiki</namespace>
+      <namespace key="9" case="first-letter">MediaWiki talk</namespace>
+      <namespace key="10" case="first-letter">Template</namespace>
+      <namespace key="11" case="first-letter">Template talk</namespace>
+      <namespace key="12" case="first-letter">Help</namespace>
+      <namespace key="13" case="first-letter">Help talk</namespace>
+      <namespace key="14" case="first-letter">Category</namespace>
+      <namespace key="15" case="first-letter">Category talk</namespace>
+      <namespace key="100" case="first-letter">Portal</namespace>
+      <namespace key="101" case="first-letter">Portal talk</namespace>
+      <namespace key="108" case="first-letter">Book</namespace>
+      <namespace key="109" case="first-letter">Book talk</namespace>
+      <namespace key="118" case="first-letter">Draft</namespace>
+      <namespace key="119" case="first-letter">Draft talk</namespace>
+      <namespace key="446" case="first-letter">Education Program</namespace>
+      <namespace key="447" case="first-letter">Education Program talk</namespace>
+      <namespace key="710" case="first-letter">TimedText</namespace>
+      <namespace key="711" case="first-letter">TimedText talk</namespace>
+      <namespace key="828" case="first-letter">Module</namespace>
+      <namespace key="829" case="first-letter">Module talk</namespace>
+      <namespace key="2300" case="first-letter">Gadget</namespace>
+      <namespace key="2301" case="first-letter">Gadget talk</namespace>
+      <namespace key="2302" case="case-sensitive">Gadget definition</namespace>
+      <namespace key="2303" case="case-sensitive">Gadget definition talk</namespace>
+    </namespaces>
+    </siteinfo>
+
+<page>
+
+  <title>Test Title</title>
+  <ns>0</ns>
+    <id>1</id>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">
+	start with four words
+      </text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">
+	start with four words and then add five more
+      </text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">
+	start with four words
+      </text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.53</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">
+	this edit will also be reverted
+      </text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.60</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">
+	start with four words
+      </text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+  </page>
+</mediawiki>

test/dumps/pwr-test.xml

@@ -0,0 +1,102 @@
+<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd" version="0.10" xml:lang="en">
+  <siteinfo>
+    <sitename>Wikipedia</sitename>
+    <dbname>enwiki</dbname>
+    <base>https://en.wikipedia.org/wiki/Main_Page</base>
+    <generator>MediaWiki 1.31.0-wmf.22</generator>
+    <case>first-letter</case>
+    <namespaces>
+      <namespace key="-2" case="first-letter">Media</namespace>
+      <namespace key="-1" case="first-letter">Special</namespace>
+      <namespace key="0" case="first-letter" />
+      <namespace key="1" case="first-letter">Talk</namespace>
+      <namespace key="2" case="first-letter">User</namespace>
+      <namespace key="3" case="first-letter">User talk</namespace>
+      <namespace key="4" case="first-letter">Wikipedia</namespace>
+      <namespace key="5" case="first-letter">Wikipedia talk</namespace>
+      <namespace key="6" case="first-letter">File</namespace>
+      <namespace key="7" case="first-letter">File talk</namespace>
+      <namespace key="8" case="first-letter">MediaWiki</namespace>
+      <namespace key="9" case="first-letter">MediaWiki talk</namespace>
+      <namespace key="10" case="first-letter">Template</namespace>
+      <namespace key="11" case="first-letter">Template talk</namespace>
+      <namespace key="12" case="first-letter">Help</namespace>
+      <namespace key="13" case="first-letter">Help talk</namespace>
+      <namespace key="14" case="first-letter">Category</namespace>
+      <namespace key="15" case="first-letter">Category talk</namespace>
+      <namespace key="100" case="first-letter">Portal</namespace>
+      <namespace key="101" case="first-letter">Portal talk</namespace>
+      <namespace key="108" case="first-letter">Book</namespace>
+      <namespace key="109" case="first-letter">Book talk</namespace>
+      <namespace key="118" case="first-letter">Draft</namespace>
+      <namespace key="119" case="first-letter">Draft talk</namespace>
+      <namespace key="446" case="first-letter">Education Program</namespace>
+      <namespace key="447" case="first-letter">Education Program talk</namespace>
+      <namespace key="710" case="first-letter">TimedText</namespace>
+      <namespace key="711" case="first-letter">TimedText talk</namespace>
+      <namespace key="828" case="first-letter">Module</namespace>
+      <namespace key="829" case="first-letter">Module talk</namespace>
+      <namespace key="2300" case="first-letter">Gadget</namespace>
+      <namespace key="2301" case="first-letter">Gadget talk</namespace>
+      <namespace key="2302" case="case-sensitive">Gadget definition</namespace>
+      <namespace key="2303" case="case-sensitive">Gadget definition talk</namespace>
+    </namespaces>
+    </siteinfo>
+
+<page>
+  <title>Test Title</title>
+  <ns>0</ns>
+    <id>1</id>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words</text>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words and then add five more</text>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words</text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.53</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">this edit will also be reverted</text>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.60</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words</text>
+    </revision>
+  </page>
+</mediawiki>

test/dumps/pwr-test.xml~

@@ -0,0 +1,102 @@
+<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.10/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd" version="0.10" xml:lang="en">
+  <siteinfo>
+    <sitename>Wikipedia</sitename>
+    <dbname>enwiki</dbname>
+    <base>https://en.wikipedia.org/wiki/Main_Page</base>
+    <generator>MediaWiki 1.31.0-wmf.22</generator>
+    <case>first-letter</case>
+    <namespaces>
+      <namespace key="-2" case="first-letter">Media</namespace>
+      <namespace key="-1" case="first-letter">Special</namespace>
+      <namespace key="0" case="first-letter" />
+      <namespace key="1" case="first-letter">Talk</namespace>
+      <namespace key="2" case="first-letter">User</namespace>
+      <namespace key="3" case="first-letter">User talk</namespace>
+      <namespace key="4" case="first-letter">Wikipedia</namespace>
+      <namespace key="5" case="first-letter">Wikipedia talk</namespace>
+      <namespace key="6" case="first-letter">File</namespace>
+      <namespace key="7" case="first-letter">File talk</namespace>
+      <namespace key="8" case="first-letter">MediaWiki</namespace>
+      <namespace key="9" case="first-letter">MediaWiki talk</namespace>
+      <namespace key="10" case="first-letter">Template</namespace>
+      <namespace key="11" case="first-letter">Template talk</namespace>
+      <namespace key="12" case="first-letter">Help</namespace>
+      <namespace key="13" case="first-letter">Help talk</namespace>
+      <namespace key="14" case="first-letter">Category</namespace>
+      <namespace key="15" case="first-letter">Category talk</namespace>
+      <namespace key="100" case="first-letter">Portal</namespace>
+      <namespace key="101" case="first-letter">Portal talk</namespace>
+      <namespace key="108" case="first-letter">Book</namespace>
+      <namespace key="109" case="first-letter">Book talk</namespace>
+      <namespace key="118" case="first-letter">Draft</namespace>
+      <namespace key="119" case="first-letter">Draft talk</namespace>
+      <namespace key="446" case="first-letter">Education Program</namespace>
+      <namespace key="447" case="first-letter">Education Program talk</namespace>
+      <namespace key="710" case="first-letter">TimedText</namespace>
+      <namespace key="711" case="first-letter">TimedText talk</namespace>
+      <namespace key="828" case="first-letter">Module</namespace>
+      <namespace key="829" case="first-letter">Module talk</namespace>
+      <namespace key="2300" case="first-letter">Gadget</namespace>
+      <namespace key="2301" case="first-letter">Gadget talk</namespace>
+      <namespace key="2302" case="case-sensitive">Gadget definition</namespace>
+      <namespace key="2303" case="case-sensitive">Gadget definition talk</namespace>
+    </namespaces>
+    </siteinfo>
+
+<page>
+  <title>Test Title</title>
+  <ns>0</ns>
+    <id>1</id>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words</text>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words and then add five more</text>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.52</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words</text>
+      <sha1>cp0mfp4o90fbpyxooys1rw95zn3ddff</sha1>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.53</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">this edit will also be reverted</text>
+    </revision>
+    <revision>
+      <timestamp>2010-12-13T02:51:55Z</timestamp>
+      <contributor>
+        <ip>76.102.106.60</ip>
+      </contributor>
+      <comment>baseline</comment>
+      <model>wikitext</model>
+      <format>text/x-wiki</format>
+      <text xml:space="preserve">start with four words</text>
+    </revision>
+  </page>
+</mediawiki>

wikiq

@@ -17,7 +17,7 @@ from mwxml import Dump, Page
 from deltas.tokenizers import wikitext_split
 from mwdiffs.utilities import dump2diffs
 import mwpersistence
-from mwpersistence.state import Version, apply_opdocs, apply_operations, persist_revision_once
+from mwpersistence.state import DiffState
 
 from mwpersistence import Token
 from  mwpersistence.utilities import diffs2persistence 
@@ -29,46 +29,8 @@ from deltas import SegmentMatcher
 TO_ENCODE = ('title', 'editor')
 PERSISTENCE_RADIUS=7
 
-# this is a simple override of mwpersistence.DiffState that doesn't do anything special for reverts. 
+ws_lex = ['break','whitespace']
-class WikiqDiffState(mwpersistence.DiffState):
+punct_lex = ['period','qmark','epoint','comma','colon','scolon','paren_open','paren_close','brack_open','brack_close','dbrack_close','dbrack_open','tab_close','tab_open','dcurly_close','dcurly_open','equals','bar','etc','bold','italic','tag','comment_end','comment_start']
-    def _update(self, text=None, checksum=None, opdocs=None, revision=None):
-        if checksum is None:
-            if text is None:
-                raise TypeError("Either 'text' or 'checksum' must be " +
-                                "specified.")
-            else:
-                checksum = sha1(bytes(text, 'utf8')).hexdigest()
-
-        current_version = Version()
-
-        # the main difference we have is that we don't do anything special for reverts
-        if opdocs is not None:
-            transition = apply_opdocs(opdocs, self.last.tokens or [])
-            current_version.tokens, _, _ = transition
-        else:
-            # NOTICE: HEAVY COMPUTATION HERE!!!
-            #
-            # Diffs usually run in O(n^2) -- O(n^3) time and most
-            # tokenizers produce a lot of tokens.
-            if self.diff_processor is None:
-                raise RuntimeError("DiffState cannot process raw text " +
-                                   "without a diff_engine specified.")
-            operations, _, current_tokens = \
-                self.diff_processor.process(text, token_class=Token)
-
-            transition = apply_operations(operations,
-                                          self.last.tokens or [],
-                                          current_tokens)
-            current_version.tokens, _, _ = transition
-
-        # Record persistence
-        persist_revision_once(current_version.tokens, revision)
-
-        # Update last version
-        self.last = current_version
-
-        # Return the tranisitoned state
-        return transition
 
 class PersistMethod:
     none = 0
@@ -76,20 +38,14 @@ class PersistMethod:
     segment = 2
     legacy = 3
 
-def calculate_persistence(tokens_added, tokens_removed, exclude_ws = True, exclude_punct = False):
+def calculate_persistence(tokens_added, tokens_removed, exclude_ws = False, exclude_punct = False, legacy = False):
-    cond = True
-    ws_lex = ['break','whitespace']
-    punct_lex = ['period','qmark','epoint','comma','colon','scolon','paren_open','paren_close','brack_open','brack_close','dbrack_close','dbrack_open','tab_close','tab_open','dcurly_close','dcurly_open','equals','bar','etc','bold','italic','tag','comment_end','comment_start']
 
+    if not legacy:
+        cond =  lambda t: not (exclude_punct and (t.type in punct_lex)) \
+                and not(exclude_ws and (t.type in ws_lex))
 
-    if exclude_ws:
+        tokens_added = [t for t in tokens_added if cond(t)]
-        cond = lambda t: cond and not t.type in ws_lex
+        tokens_removed = [t for t in tokens_removed if cond(t)]
-
-    if exclude_punct:
-        cond = lambda t: cond and not t.type in punct_lex
-    
-    tokens_added = [t for t in tokens_added if cond(t)]
-    tokens_removed = [t for t in tokens_removed if cond(t)]
 
     return(sum([(len(x.revisions)-1) for x in tokens_added]),
            len(tokens_added),
@@ -100,17 +56,18 @@ class WikiqIterator(Dump):
 
     @classmethod
     def from_file(cls, fh, collapse_user = False):
-        cls = super(WikiqIterator, cls).from_file(fh)
         cls.fh = fh
         cls.collapse_user = collapse_user
-        cls.namespace_map = { ns.id : ns.name for ns in
+        cls = super(WikiqIterator, cls).from_file(fh)
-                               cls.site_info.namespaces }
         return cls
 
     @classmethod
-    def process_item(cls, item_element, namespace_map, collapse_user = False):
+    def process_item(cls, item_element, namespace_map):
+        if not hasattr(cls,'inv_namespace_map'):
+            cls.inv_namespace_map = {ns.id:name for name, ns in namespace_map.items()}
+
         if item_element.tag == "page":
-            return WikiqPage.from_element(item_element, namespace_map, collapse_user)
+            return WikiqPage.from_element(item_element, namespace_map, cls.inv_namespace_map, cls.collapse_user)
         elif item_element.tag == "logitem":
             return LogItem.from_element(item_element, namespace_map)
         else:
@@ -122,11 +79,9 @@ class WikiqPage(Page):
                  'restrictions','collapse_user')
         
     @classmethod
-    def from_element(cls, item_element, namespace_map, collapse_user = False):
+    def from_element(cls, item_element, namespace_map, inv_namespace_map, collapse_user = False):
         cls.prev_rev = None
 
-        inv_namespace_map = {ns.id:name for name,ns in namespace_map.items()}
-        
         cls = super(WikiqPage, cls).from_element(item_element, namespace_map)
 
         # following mwxml, we assume namespace 0 in cases where
@@ -186,7 +141,8 @@ class WikiqPage(Page):
 
         if self.collapse_user: 
             collapsed_revs = 1
-            rev.collapsed_revs = collapsed_revs
+            self.prev_rev.collapsed_revs = collapsed_revs
+            prev_rev = self.prev_rev
 
         for rev in self.revisions:
             rev = WikiqPage._correct_sha(rev)
@@ -235,12 +191,6 @@ class WikiqPage(Page):
             revision.page = self
             yield revision
 
-    # def __iter__(self):
-    #     return self.__revisions
-
-    # def __next__(self):
-    #     return next(self.__revisions)
-
 class WikiqParser():
     
     def __init__(self, input_file, output_file, collapse_user=False, persist=None, urlencode=False, namespaces = None):
@@ -315,11 +265,11 @@ class WikiqParser():
                 window = deque(maxlen=PERSISTENCE_RADIUS)
 
                 if self.persist == PersistMethod.sequence:
-                    state = WikiqDiffState(SequenceMatcher(tokenizer = wikitext_split),
+                    state = DiffState(SequenceMatcher(tokenizer = wikitext_split),
                                                     revert_radius=PERSISTENCE_RADIUS)
 
                 elif self.persist == PersistMethod.segment:
-                    state = WikiqDiffState(SegmentMatcher(tokenizer = wikitext_split),
+                    state = DiffState(SegmentMatcher(tokenizer = wikitext_split),
                                                     revert_radius=PERSISTENCE_RADIUS)
 
                 else:
@@ -410,7 +360,7 @@ class WikiqParser():
                         if len(window) == PERSISTENCE_RADIUS:
                             old_rev_id, old_rev_data, old_tokens_added, old_tokens_removed = window[0]
                             
-                            num_token_revs, num_tokens_added, num_tokens_removed  = calculate_persistence(old_tokens_added, old_tokens_removed)
+                            num_token_revs, num_tokens_added, num_tokens_removed  = calculate_persistence(old_tokens_added, old_tokens_removed, legacy = self.persist == PersistMethod.legacy)
 
                             old_rev_data["token_revs"] = num_token_revs
                             old_rev_data["tokens_added"] = num_tokens_added
@@ -433,7 +383,7 @@ class WikiqParser():
 
                     rev_id, rev_data, tokens_added, tokens_removed = item
 
-                    num_token_revs, num_tokens_added, num_tokens_removed = calculate_persistence(tokens_added, tokens_removed)
+                    num_token_revs, num_tokens_added, num_tokens_removed = calculate_persistence(tokens_added, tokens_removed, legacy = self.persist == PersistMethod.legacy)
 
                     rev_data["token_revs"] = num_token_revs
                     rev_data["tokens_added"] = num_tokens_added