84 lines
4.1 KiB
Python
Executable File
84 lines
4.1 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# original wikiq headers are: title articleid revid date_time anon
|
|
# editor editor_id minor text_size text_entropy text_md5 reversion
|
|
# additions_size deletions_size
|
|
import argparse
|
|
import sys
|
|
import os
|
|
sys.path.append("..")
|
|
from wikiq_util import calculate_persistence
|
|
from wikiq_util import WikiqIterator
|
|
from wikiq_util import WikiqPage
|
|
from wikiq_util import WikiqParser
|
|
from wikiq_util import open_input_file
|
|
from wikiq_util import open_output_file
|
|
|
|
|
|
parser = argparse.ArgumentParser(description='Parse MediaWiki XML database dumps into tab delimitted data.')
|
|
|
|
# arguments for the input direction
|
|
parser.add_argument('dumpfiles', metavar="DUMPFILE", nargs="*", type=str,
|
|
help="Filename of the compressed or uncompressed XML database dump. If absent, we'll look for content on stdin and output on stdout.")
|
|
|
|
parser.add_argument('-o', '--output-dir', metavar='DIR', dest='output_dir', type=str, nargs=1,
|
|
help="Directory for output files.")
|
|
|
|
parser.add_argument('-s', '--stdout', dest="stdout", action="store_true",
|
|
help="Write output to standard out (do not create dump file)")
|
|
|
|
parser.add_argument('--collapse-user', dest="collapse_user", action="store_true",
|
|
help="Operate only on the final revision made by user a user within all sequences of consecutive edits made by a user. This can be useful for addressing issues with text persistence measures.")
|
|
|
|
parser.add_argument('-p', '--persistence', dest="persist", action="store_true",
|
|
help="Compute and report measures of content persistent: (1) persistent token revisions, (2) tokens added, and (3) number of revision used in computing the first measure.")
|
|
|
|
parser.add_argument('-u', '--url-encode', dest="urlencode", action="store_true",
|
|
help="Output url encoded text strings. This works around some data issues like newlines in editor names. In the future it may be used to output other text data.")
|
|
|
|
parser.add_argument('--persistence-legacy', dest="persist_legacy", action="store_true",
|
|
help="Legacy behavior for persistence calculation. Output url encoded text strings. This works around some data issues like newlines in editor names. In the future it may be used to output other text data.")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if len(args.dumpfiles) > 0:
|
|
for filename in args.dumpfiles:
|
|
input_file = open_input_file(filename)
|
|
|
|
# open directory for output
|
|
if args.output_dir:
|
|
output_dir = args.output_dir[0]
|
|
else:
|
|
output_dir = "."
|
|
|
|
print("Processing file: %s" % filename, file=sys.stderr)
|
|
|
|
if args.stdout:
|
|
output_file = sys.stdout
|
|
else:
|
|
filename = os.path.join(output_dir, os.path.basename(filename))
|
|
output_file = open_output_file(filename)
|
|
|
|
wikiq = WikiqParser(input_file, output_file,
|
|
collapse_user=args.collapse_user,
|
|
persist=args.persist,
|
|
persist_legacy=args.persist_legacy,
|
|
urlencode=args.urlencode)
|
|
|
|
|
|
wikiq.process()
|
|
|
|
# close things
|
|
input_file.close()
|
|
output_file.close()
|
|
else:
|
|
wikiq = WikiqParser(sys.stdin, sys.stdout,
|
|
collapse_user=args.collapse_user,
|
|
persist=args.persist,
|
|
persist_legacy=args.persist_legacy,
|
|
urlencode=args.urlencode)
|
|
wikiq.process()
|
|
|
|
# stop_words = "a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your"
|
|
# stop_words = stop_words.split(",")
|