Initial commit

p#	new file:   runwikiq.sh

mediawiki_dump_tools/Mediawiki-Utilities/doc/Makefile

@@ -0,0 +1,182 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    = -v
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = _build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/mediawiki-utilities.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/mediawiki-utilities.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/mediawiki-utilities"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/mediawiki-utilities"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
+
+htmlzip: html
+	cd _build/html/ && \
+	zip -r ../../html.zip * && \
+	cd ../../

mediawiki_dump_tools/Mediawiki-Utilities/doc/conf.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# mediawiki-utilities documentation build configuration file, created by
+# sphinx-quickstart on Thu Apr 10 17:31:47 2014.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+sys.path.insert(0, os.path.abspath('../'))
+import mw
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.viewcode',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'mediawiki-utilities'
+copyright = '2014, Aaron Halfaker'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = mw.__version__
+# The full version, including alpha/beta/rc tags.
+release = mw.__version__
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build']
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'mediawiki-utilitiesdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+  ('index', 'mediawiki-utilities.tex', 'mediawiki-utilities Documentation',
+   'Aaron Halfaker', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'mediawiki-utilities', 'mediawiki-utilities Documentation',
+     ['Aaron Halfaker'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'mediawiki-utilities', 'mediawiki-utilities Documentation',
+   'Aaron Halfaker', 'mediawiki-utilities', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False

mediawiki_dump_tools/Mediawiki-Utilities/doc/core/api.rst

@@ -0,0 +1,77 @@
+.. _mw.api:
+
+===================================
+mw.api -- MediaWiki API abstraction
+===================================
+
+This module contains a set of utilities for interacting with the MediaWiki API.
+
+Here's an example of a common usage pattern:
+	
+	>>> from mw import api
+	>>> 
+	>>> session = api.Session("https://en.wikipedia.org/w/api.php")
+	>>> 
+	>>> revisions = session.revisions.query(
+	...     properties={'ids', 'content'},
+	...     titles={"User:EpochFail"},
+	...     direction="newer",
+	...     limit=3
+	... )
+	>>> 
+	>>> for rev in revisions:
+	...     print(
+	...             "rev_id={0}, length={1} characters".format(
+	...                     rev['revid'],
+	...                     len(rev.get('*', ""))
+	...             )
+	...     )
+	... 
+	rev_id=190055192, length=124 characters
+	rev_id=276121340, length=132 characters
+	rev_id=276121389, length=124 characters
+
+Session
+=======
+
+.. autoclass:: mw.api.Session
+   :members:
+   :member-order: bysource
+
+
+Collections
+===========
+
+.. autoclass:: mw.api.DeletedRevisions
+   :members:
+
+.. autoclass:: mw.api.Pages
+   :members:
+
+.. autoclass:: mw.api.RecentChanges
+   :members:
+
+.. autoclass:: mw.api.Revisions
+   :members:
+
+.. autoclass:: mw.api.SiteInfo
+   :members:
+
+.. autoclass:: mw.api.UserContribs
+   :members:
+
+Errors
+======
+
+
+.. autoclass:: mw.api.errors.APIError
+   :members:
+   :inherited-members:
+
+.. autoclass:: mw.api.errors.AuthenticationError
+   :members:
+   :inherited-members:
+
+.. autoclass:: mw.api.errors.MalformedResponse
+   :members:
+   :inherited-members:

mediawiki_dump_tools/Mediawiki-Utilities/doc/core/database.rst

@@ -0,0 +1,53 @@
+.. _mw.database:
+
+=========================================
+mw.database -- MySQL database abstraction
+=========================================
+
+This module contains a set of utilities for interacting with MediaWiki databases.
+
+Here's an example of a common usage pattern:
+::
+	
+	from mw import database
+	
+	db = database.DB.from_params(
+		host="s1-analytics-slave.eqiad.wmnet", 
+		read_default_file="~/.my.cnf", 
+		user="research", 
+		db="enwiki"
+	)
+	revisions = db.revisions.query(user_id=9133062)
+	
+	for rev_row in revisions:
+		rev_row['rev_id']
+
+
+DB
+======
+
+.. autoclass:: mw.database.DB
+   :members:
+   :member-order: bysource
+   
+
+Collections
+===========
+
+.. autoclass:: mw.database.Archives
+   :members:
+
+.. autoclass:: mw.database.AllRevisions
+   :members:
+
+.. autoclass:: mw.database.Pages
+   :members:
+
+.. autoclass:: mw.database.RecentChanges
+   :members:
+
+.. autoclass:: mw.database.Revisions
+   :members:
+
+.. autoclass:: mw.database.Users
+   :members:

mediawiki_dump_tools/Mediawiki-Utilities/doc/core/xml_dump.rst

@@ -0,0 +1,52 @@
+.. _mw.xml_dump:
+
+==================================
+mw.xml_dump -- XML dump processing
+==================================
+
+.. automodule:: mw.xml_dump
+
+The map() function
+==================
+
+.. autofunction:: mw.xml_dump.map
+
+Iteration
+=========
+
+.. autoclass:: mw.xml_dump.Iterator
+   :members:
+   :member-order: bysource
+
+.. autoclass:: mw.xml_dump.Page
+   :members:
+   :member-order: bysource
+
+.. autoclass:: mw.xml_dump.Redirect
+   :members:
+   :member-order: bysource
+
+.. autoclass:: mw.xml_dump.Revision
+   :members:
+   :member-order: bysource
+
+.. autoclass:: mw.xml_dump.Comment
+   :members:
+   :member-order: bysource
+
+.. autoclass:: mw.xml_dump.Contributor
+   :members:
+   :member-order: bysource
+
+.. autoclass:: mw.xml_dump.Text
+   :members:
+   :member-order: bysource
+
+Errors
+======
+
+.. autoclass:: mw.xml_dump.errors.FileTypeError
+   :members:
+
+.. autoclass:: mw.xml_dump.errors.MalformedXML
+   :members:

mediawiki_dump_tools/Mediawiki-Utilities/doc/index.rst

@@ -0,0 +1,100 @@
+.. mediawiki-utilities documentation master file, created by
+   sphinx-quickstart on Thu Apr 10 17:31:47 2014.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+===================
+MediaWiki Utilities
+===================
+
+MediaWiki Utilities is an open source (MIT Licensed) library developed by Aaron Halfaker for extracting and processing data from MediaWiki installations, slave databases and xml dumps.
+
+**Instal with pip:** ``pip install mediawiki-utilities``
+
+**Note:** *Use of this library requires Python 3 or later.*
+
+Types
+=====
+:ref:`mw.Timestamp <mw.types>`
+	A simple datatype for handling MediaWiki's various time formats.
+
+Core modules
+============
+
+:ref:`mw.api <mw.api>`
+	A set of utilities for interacting with MediaWiki's web API.
+	
+	* :class:`~mw.api.Session` -- Constructs an API session with a MediaWiki installation.  Contains convenience methods for accessing ``prop=revisions``,  ``list=usercontribs``, ``meta=siteinfo``, ``list=deletedrevs`` and ``list=recentchanges``.
+
+:ref:`mw.database <mw.database>`
+	A set of utilities for interacting with MediaWiki's database.
+	
+	* :class:`~mw.database.DB` -- Constructs a mysql database connector with convenience methods for accessing ``revision``, ``archive``, ``page``, ``user``, and ``recentchanges``.
+
+:ref:`mw.xml_dump <mw.xml_dump>`
+	A set of utilities for processing MediaWiki's XML database dumps quickly and without dealing with streaming XML. 
+	
+	* :func:`~mw.xml_dump.map` -- Applies a function to a set of dump files (:class:`~mw.xml_dump.Iterator`) using :class:`multiprocessing` and aggregates the output.
+	* :class:`~mw.xml_dump.Iterator` -- Constructs an iterator over a standard XML dump.  Dumps contain site_info and pages.  Pages contain metadata and revisions.  Revisions contain metadata and text.  This is probably why you are here.
+
+Libraries
+=========
+
+:ref:`mw.lib.persistence <mw.lib.persistence>`
+	A set of utilities for tracking the persistence of content between revisions.
+	
+	* :class:`~mw.lib.persistence.State` -- Constructs an object that represents the current content persistence state of a page.  Reports useful details about the persistence of content when updated.
+
+:ref:`mw.lib.reverts <mw.lib.reverts>`
+	A set of utilities for performing revert detection
+	
+	* :func:`~mw.lib.reverts.detect` -- Detects reverts in a sequence of revision events.
+	* :class:`~mw.lib.reverts.Detector` -- Constructs an identity revert detector that can be updated manually over the history of a page. 
+
+:ref:`mw.lib.sessions <mw.lib.sessions>`
+	A set of utilities for grouping revisions and other events into sessions
+	
+	* :func:`~mw.lib.sessions.cluster` -- Clusters a sequence of user actions into sessions.
+	* :class:`~mw.lib.sessions.Cache` -- Constructs a cache of recent user actions that can be updated manually in order to detect sessions.
+
+:ref:`mw.lib.title <mw.lib.title>`
+	A set of utilities for normalizing and parsing page titles
+	
+	* :func:`~mw.lib.title.normalize` -- Normalizes a page title.  
+	* :class:`~mw.lib.title.Parser` -- Constructs a parser with a set of namespaces that can be used to parse and normalize page titles. 
+
+About the author
+================
+:name: 
+	Aaron Halfaker
+:email:
+	aaron.halfaker@gmail.com
+:website:
+	http://halfaker.info --
+	http://en.wikipedia.org/wiki/User:EpochFail
+
+
+Contributors
+============
+None yet.  See http://github.com/halfak/mediawiki-utilities.  Pull requests are encouraged.
+
+
+Indices and tables
+==================
+
+.. toctree::
+   :maxdepth: 2
+   
+   types
+   core/api
+   core/database
+   core/xml_dump
+   lib/persistence
+   lib/reverts
+   lib/sessions
+   lib/title
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

mediawiki_dump_tools/Mediawiki-Utilities/doc/lib/persistence.rst

@@ -0,0 +1,35 @@
+.. _mw.lib.persistence:
+
+=======================================================
+mw.lib.persistence -- tracking content between revisions
+=======================================================
+
+.. autoclass:: mw.lib.persistence.State
+	:members:
+
+Tokenization
+============
+
+.. autoclass:: mw.lib.persistence.Tokens
+	:members:
+
+.. autoclass:: mw.lib.persistence.Token
+	:members:
+
+.. automodule:: mw.lib.persistence.tokenization
+   :members:
+   :member-order: bysource
+
+Difference
+==========
+
+.. automodule:: mw.lib.persistence.difference
+   :members:
+   :member-order: bysource
+
+Constants
+=========
+
+.. automodule:: mw.lib.persistence.defaults
+   :members:
+   :member-order: bysource

mediawiki_dump_tools/Mediawiki-Utilities/doc/lib/reverts.rst

@@ -0,0 +1,30 @@
+.. _mw.lib.reverts:
+
+=============================================
+mw.lib.reverts -- detecting reverts
+=============================================
+
+.. automodule:: mw.lib.reverts
+
+.. autofunction:: mw.lib.reverts.detect
+
+.. autoclass:: mw.lib.reverts.Revert
+
+.. autoclass:: mw.lib.reverts.Detector
+   :members:
+
+Convenience functions
+=====================
+.. automodule:: mw.lib.reverts.api
+   :members:
+   :member-order: bysource
+
+.. automodule:: mw.lib.reverts.database
+   :members:
+   :member-order: bysource
+
+Constants
+=========
+
+.. automodule:: mw.lib.reverts.defaults
+   :members:

mediawiki_dump_tools/Mediawiki-Utilities/doc/lib/sessions.rst

@@ -0,0 +1,18 @@
+.. _mw.lib.sessions:
+
+===================================
+mw.lib.sessions -- event clustering
+===================================
+
+.. autofunction:: mw.lib.sessions.cluster
+
+.. autoclass:: mw.lib.sessions.Session
+
+.. autoclass:: mw.lib.sessions.Cache
+	:members:
+
+Constants
+=========
+
+.. automodule:: mw.lib.sessions.defaults
+   :members:

mediawiki_dump_tools/Mediawiki-Utilities/doc/lib/title.rst

@@ -0,0 +1,15 @@
+.. _mw.lib.title:
+
+============================================================
+mw.lib.title -- parsing and normalizing titles
+============================================================
+
+.. autofunction:: mw.lib.title.normalize
+
+
+Title parser
+================
+.. autoclass:: mw.lib.title.Parser
+   :members:
+   :member-order: bysource
+

mediawiki_dump_tools/Mediawiki-Utilities/doc/types.rst

@@ -0,0 +1,11 @@
+.. _mw.types:
+
+========================
+mw.types -- common types
+========================
+
+.. autoclass:: mw.Timestamp
+   :members:
+
+.. autoclass:: mw.Namespace
+   :members: