collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

support 7z archives with multiple files. add urlencode paraeter

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2017-12-07 15:10:56 -08:00
parent 5d7dceb9e4
commit 3f9da40747
1 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
wikiq
View File

@ -16,7 +16,8 @@ from hashlib import sha1
from mw.xml_dump import Iterator from mw.xml_dump import Iterator
from mw.lib import persistence from mw.lib import persistence
from mw.lib import reverts from mw.lib import reverts
from urllib.parse import quote
TO_ENCODE = ('title', 'editor')
PERSISTENCE_RADIUS=7 PERSISTENCE_RADIUS=7
def calculate_persistence(tokens_added): def calculate_persistence(tokens_added):
@ -98,13 +99,17 @@ class WikiqPage():
return next(self.__revisions) return next(self.__revisions)
class WikiqParser(): class WikiqParser():
def __init__(self, input_file, output_file, collapse_user=False, persist=False):
def __init__(self, input_file, output_file, collapse_user=False, persist=False, urlencode=False):
self.input_file = input_file self.input_file = input_file
self.output_file = output_file self.output_file = output_file
self.collapse_user = collapse_user self.collapse_user = collapse_user
self.persist = persist self.persist = persist
self.printed_header = False self.printed_header = False
self.namespaces = [] self.namespaces = []
self.urlencode = urlencode
def __get_namespace_from_title(self, title): def __get_namespace_from_title(self, title):
default_ns = None default_ns = None
@ -122,7 +127,6 @@ class WikiqParser():
return default_ns return default_ns
def process(self): def process(self):
print("Processing file: %s" % self.input_file.name, file=sys.stderr)
# create a regex that creates the output filename # create a regex that creates the output filename
# output_filename = re.sub(r'^.*/(enwiki\-\d+)\-.*p(\d+)p.*$', # output_filename = re.sub(r'^.*/(enwiki\-\d+)\-.*p(\d+)p.*$',
@ -256,6 +260,10 @@ class WikiqParser():
def print_rev_data(self, rev_data): def print_rev_data(self, rev_data):
# if it's the first time through, print the header # if it's the first time through, print the header
if self.urlencode:
for field in TO_ENCODE:
rev_data[field] = quote(str(rev_data[field]))
if not self.printed_header: if not self.printed_header:
print("\t".join([str(k) for k in sorted(rev_data.keys())]), file=self.output_file) print("\t".join([str(k) for k in sorted(rev_data.keys())]), file=self.output_file)
self.printed_header = True self.printed_header = True
@ -265,11 +273,11 @@ class WikiqParser():
def open_input_file(input_filename): def open_input_file(input_filename):
if re.match(r'.*\.7z', input_filename): if re.match(r'.*\.7z', input_filename):
cmd = ["7za", "x", "-so", input_filename] cmd = ["7za", "x", "-so", input_filename, '*.xml']
elif re.match(r'.*\.gz', input_filename): elif re.match(r'.*\.gz', input_filename):
cmd = ["zcat", input_filename] cmd = ["zcat", input_filename]
elif re.match(r'.*\.bz2', input_filename): elif re.match(r'.*\.bz2', input_filename):
cmd = ["bzcat", input_filename] cmd = ["zcat", input_filename]
try: try:
input_file = Popen(cmd, stdout=PIPE).stdout input_file = Popen(cmd, stdout=PIPE).stdout
@ -280,7 +288,8 @@ def open_input_file(input_filename):
def open_output_file(input_filename): def open_output_file(input_filename):
# create a regex that creates the output filename # create a regex that creates the output filename
output_filename = re.sub(r'\.xml(\.(7z|gz|bz2))?$', '', input_filename) output_filename = re.sub(r'\.(7z|gz|bz2)?$', '', input_filename)
output_filename = re.sub(r'\.xml', '', output_filename)
output_filename = output_filename + ".tsv" output_filename = output_filename + ".tsv"
output_file = open(output_filename, "w") output_file = open(output_filename, "w")
@ -304,6 +313,9 @@ parser.add_argument('--collapse-user', dest="collapse_user", action="store_true"
parser.add_argument('-p', '--persistence', dest="persist", action="store_true", parser.add_argument('-p', '--persistence', dest="persist", action="store_true",
help="Compute and report measures of content persistent: (1) persistent token revisions, (2) tokens added, and (3) number of revision used in computing the first measure.") help="Compute and report measures of content persistent: (1) persistent token revisions, (2) tokens added, and (3) number of revision used in computing the first measure.")
parser.add_argument('-u', '--url-encode', dest="urlencode", action="store_true",
help="Output url encoded text strings. This works around some data issues like newlines in editor names. In the future it may be used to output other text data.")
args = parser.parse_args() args = parser.parse_args()
if len(args.dumpfiles) > 0: if len(args.dumpfiles) > 0:
@ -324,7 +336,11 @@ if len(args.dumpfiles) > 0:
wikiq = WikiqParser(input_file, output_file, wikiq = WikiqParser(input_file, output_file,
collapse_user=args.collapse_user, collapse_user=args.collapse_user,
persist=args.persist) persist=args.persist,
urlencode=args.urlencode)
print("Processing file: %s" % filename, file=sys.stderr)
wikiq.process() wikiq.process()
# close things # close things
@ -333,7 +349,8 @@ if len(args.dumpfiles) > 0:
else: else:
wikiq = WikiqParser(sys.stdin, sys.stdout, wikiq = WikiqParser(sys.stdin, sys.stdout,
collapse_user=args.collapse_user, collapse_user=args.collapse_user,
persist=args.persist) persist=args.persist,
urlencode=args.urlencode)
wikiq.process() wikiq.process()
# stop_words = "a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your" # stop_words = "a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your"