collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

add support for persistence with segment matching

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2018-08-20 16:08:16 -07:00
parent bf396ad366
commit f468d1a5b6
3 changed files with 4714 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
test/Wikiq_Unit_Test.py
View File

@ -108,6 +108,26 @@ class Test_Basic(unittest.TestCase):
baseline = pd.read_table(baseline_file)
assert_frame_equal(test,baseline)
def test_pwr_segment(self):
test_filename = "persistence_segment_" + self.wikiq_out_name
test_file = os.path.join(self.test_output_dir, test_filename)
if os.path.exists(test_file):
os.remove(test_file)
call = self.base_call.format(self.input_file, self.test_output_dir)
call = call + " --persistence segment"
proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
proc.wait()
copyfile(self.call_output, test_file)
baseline_file = os.path.join(".", self.baseline_output_dir, test_filename)
test = pd.read_table(test_file)
baseline = pd.read_table(baseline_file)
assert_frame_equal(test,baseline)
def test_pwr_legacy(self):
test_filename = "persistence_legacy_" + self.wikiq_out_name
test_file = os.path.join(self.test_output_dir, test_filename)
@ -115,7 +135,7 @@ class Test_Basic(unittest.TestCase):
os.remove(test_file)
call = self.base_call.format(self.input_file, self.test_output_dir)
call = call + " --persistence-legacy"
call = call + " --persistence legacy"
proc = subprocess.Popen(call,stdout=subprocess.PIPE,shell=True)
proc.wait()
@ -132,7 +152,7 @@ class Test_Basic(unittest.TestCase):
test_filename = "persistence_" + self.wikiq_out_name
test_file = os.path.join(self.test_output_dir, test_filename)
if os.path.exists(test_file):
os.remove(test_file)
os.remove(test_file)
call = self.base_call.format(self.input_file, self.test_output_dir)
call = call + " --persistence"

4652
test/baseline_output/persistence_segment_sailormoon.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

56
wikiq
View File

@ -22,12 +22,18 @@ from urllib.parse import quote
TO_ENCODE = ('title', 'editor')
PERSISTENCE_RADIUS=7
from deltas import SequenceMatcher
from deltas import SegmentMatcher
class PersistMethod:
none = 0
sequence = 1
segment = 2
legacy = 3
def calculate_persistence(tokens_added):
return(sum([(len(x.revisions)-1) for x in tokens_added]),
len(tokens_added))
class WikiqIterator():
def __init__(self, fh, collapse_user=False):
self.fh = fh
@ -117,13 +123,16 @@ class WikiqPage():
class WikiqParser():
def __init__(self, input_file, output_file, collapse_user=False, persist=False, urlencode=False, persist_legacy=False):
def __init__(self, input_file, output_file, collapse_user=False, persist=None, urlencode=False):
"""
Parameters:
persist : what persistence method to use. Takes a PersistMethod value
"""
self.input_file = input_file
self.output_file = output_file
self.collapse_user = collapse_user
self.persist = persist
self.persist_legacy = persist_legacy
self.printed_header = False
self.namespaces = []
self.urlencode = urlencode
@ -164,13 +173,18 @@ class WikiqParser():
for page in dump:
rev_detector = mwreverts.Detector()
if self.persist or self.persist_legacy:
if self.persist != PersistMethod.none:
window = deque(maxlen=PERSISTENCE_RADIUS)
if not self.persist_legacy:
if self.persist == PersistMethod.sequence:
state = mwpersistence.DiffState(SequenceMatcher(tokenizer = wikitext_split),
revert_radius=PERSISTENCE_RADIUS)
elif self.persist == PersistMethod.segment:
state = mwpersistence.DiffState(SegmentMatcher(tokenizer = wikitext_split),
revert_radius=PERSISTENCE_RADIUS)
# self.persist == PersistMethod.legacy
else:
from mw.lib import persistence
state = persistence.State()
@ -243,14 +257,13 @@ class WikiqParser():
if self.collapse_user:
rev_data['collapsed_revs'] = rev.collapsed_revs
if self.persist or self.persist_legacy:
if self.persist != PersistMethod.none:
if rev.deleted.text:
for k in ["token_revs", "tokens_added", "tokens_removed", "tokens_window"]:
old_rev_data[k] = None
else:
if not self.persist_legacy:
if self.persist != PersistMethod.legacy:
_, tokens_added, tokens_removed = state.update(rev.text, rev.id)
else:
@ -275,7 +288,7 @@ class WikiqParser():
rev_count += 1
if self.persist or self.persist_legacy:
if self.persist != PersistMethod.none:
# print out metadata for the last RADIUS revisions
for i, item in enumerate(window):
# if the window was full, we've already printed item 0
@ -349,17 +362,25 @@ parser.add_argument('-s', '--stdout', dest="stdout", action="store_true",
parser.add_argument('--collapse-user', dest="collapse_user", action="store_true",
help="Operate only on the final revision made by user a user within all sequences of consecutive edits made by a user. This can be useful for addressing issues with text persistence measures.")
parser.add_argument('-p', '--persistence', dest="persist", action="store_true",
help="Compute and report measures of content persistent: (1) persistent token revisions, (2) tokens added, and (3) number of revision used in computing the first measure.")
parser.add_argument('-p', '--persistence', dest="persist", default=None, const='', type=str, choices = ['','segment','sequence','legacy'], nargs='?',
help="Compute and report measures of content persistent: (1) persistent token revisions, (2) tokens added, and (3) number of revision used in computing the first measure. This may by slow. Use -p=segment for advanced persistence calculation method that is robust to content moves. This might be very slow. Use -p=legacy for legacy behavior.")
parser.add_argument('-u', '--url-encode', dest="urlencode", action="store_true",
help="Output url encoded text strings. This works around some data issues like newlines in editor names. In the future it may be used to output other text data.")
parser.add_argument('--persistence-legacy', dest="persist_legacy", action="store_true",
help="Legacy behavior for persistence calculation. Output url encoded text strings. This works around some data issues like newlines in editor names. In the future it may be used to output other text data.")
args = parser.parse_args()
# set persistence method
if args.persist is None:
persist = PersistMethod.none
elif args.persist == "segment":
persist = PersistMethod.segment
elif args.persist == "legacy":
persist = PersistMethod.legacy
else:
persist = PersistMethod.sequence
if len(args.dumpfiles) > 0:
for filename in args.dumpfiles:
input_file = open_input_file(filename)
@ -378,10 +399,9 @@ if len(args.dumpfiles) > 0:
filename = os.path.join(output_dir, os.path.basename(filename))
output_file = open_output_file(filename)
wikiq = WikiqParser(input_file, output_file,
wikiq = WikiqParser(input_file, output_file,
collapse_user=args.collapse_user,
persist=args.persist,
persist_legacy=args.persist_legacy,
persist=persist,
urlencode=args.urlencode)
@ -393,7 +413,7 @@ if len(args.dumpfiles) > 0:
else:
wikiq = WikiqParser(sys.stdin, sys.stdout,
collapse_user=args.collapse_user,
persist=args.persist,
persist=persist,
persist_legacy=args.persist_legacy,
urlencode=args.urlencode)
wikiq.process()