Initial commit
p# new file: runwikiq.sh
This commit is contained in:
761
userroles_scraper_scripts/list_of_wikis.csv
Normal file
761
userroles_scraper_scripts/list_of_wikis.csv
Normal file
@@ -0,0 +1,761 @@
|
||||
24,http://24.wikia.com/
|
||||
6teen,http://6teen.wikia.com/
|
||||
8bittheater,http://8bittheater.wikia.com/
|
||||
aachen,http://aachen.wikia.com/
|
||||
abridgedseries,http://abridgedseries.wikia.com/
|
||||
absurdopedia,http://absurdopedia.wikia.com/
|
||||
academia,http://academia.wikia.com/
|
||||
academicjobs,http://academicjobs.wikia.com/
|
||||
aceattorney,http://aceattorney.wikia.com/
|
||||
acecombat,http://acecombat.wikia.com/
|
||||
actionzone,http://actionzone.wikia.com/
|
||||
admintools,http://admintools.wikia.com/
|
||||
advancewars,http://advancewars.wikia.com/
|
||||
adventurequestworlds,http://adventurequestworlds.wikia.com/
|
||||
ageofempires,http://ageofempires.wikia.com/
|
||||
aion,http://aion.wikia.com/
|
||||
airframes,http://airframes.wikia.com/
|
||||
alicesoft,http://alicesoft.wikia.com/
|
||||
aliens,http://aliens.wikia.com/
|
||||
althistory,http://althistory.wikia.com/
|
||||
americandad,http://americandad.wikia.com/
|
||||
americangirl,http://americangirl.wikia.com/
|
||||
anegkyklopaideia,http://frikipaideia.wikia.com/
|
||||
animalcrossing,http://animalcrossing.wikia.com/
|
||||
anime,http://anime.wikia.com/
|
||||
animeanswers,http://anime.answers.wikia.com/
|
||||
annex,http://annex.wikia.com/
|
||||
answers,http://answers.wikia.com/
|
||||
aoc,http://aoc.wikia.com/
|
||||
aokb,http://anarchyonline.wikia.com/
|
||||
applesparkles,http://applesparkles.wikia.com/
|
||||
aq,http://adventurequest.wikia.com/
|
||||
archiesonic,http://archiesonic.wikia.com/
|
||||
artemisfowl,http://artemisfowl.wikia.com/
|
||||
aruarose,http://aruarose.wikia.com/
|
||||
asdastory,http://asdastory.wikia.com/
|
||||
aselia,http://aselia.wikia.com/
|
||||
asheron,http://asheron.wikia.com/
|
||||
assassinscreed,http://assassinscreed.wikia.com/
|
||||
astroempires,http://astroempires.wikia.com/
|
||||
astronomy,http://astronomy.wikia.com/
|
||||
atlantica,http://atlantica.wikia.com/
|
||||
audrey,http://audrey.wikia.com/
|
||||
avatar,http://avatar.wikia.com/
|
||||
avatardev,http://avatardev.wikia.com/
|
||||
avatarfanfiction,http://avatarfanon.wikia.com/
|
||||
avatartest,http://avatartest.wikia.com/
|
||||
avp,http://avp.wikia.com/
|
||||
babylon5,http://babylon5.wikia.com/
|
||||
bakugan,http://bakugan.wikia.com/
|
||||
banjokazooie,http://banjokazooie.wikia.com/
|
||||
baseball,http://baseball.wikia.com/
|
||||
basilicus,http://basilicus.wikia.com/
|
||||
batman,http://batman.wikia.com/
|
||||
battlefield,http://battlefield.wikia.com/
|
||||
battleforge,http://battleforge.wikia.com/
|
||||
battlefront,http://battlefront.wikia.com/
|
||||
beidipedia,http://beidipedia.wikia.com/
|
||||
ben10,http://ben10.wikia.com/
|
||||
bionicle,http://bionicle.wikia.com/
|
||||
bioshock,http://bioshock.wikia.com/
|
||||
bleachfanfiction,http://bleachfanfiction.wikia.com/
|
||||
blogging,http://blogging.wikia.com/
|
||||
bmet,http://bmet.wikia.com/
|
||||
board8,http://board8.wikia.com/
|
||||
borderlands,http://borderlands.wikia.com/
|
||||
brawlhacks,http://brawlhacks.wikia.com/
|
||||
bttf,http://bttf.wikia.com/
|
||||
buffy,http://buffy.wikia.com/
|
||||
bullygame,http://bullygame.wikia.com/
|
||||
bungie,http://bungie.wikia.com/
|
||||
burnoutparadise,http://burnout.wikia.com/
|
||||
buzzoutloud,http://buzzoutloud.wikia.com/
|
||||
bzpcomics,http://bzpcomics.wikia.com/
|
||||
campaigns,http://campaigns.wikia.com/
|
||||
candh,http://calvinandhobbes.wikia.com/
|
||||
cartoonnetwork,http://cartoonnetwork.wikia.com/
|
||||
castlecrashers,http://castlecrashers.wikia.com/
|
||||
castlevania,http://castlevania.wikia.com/
|
||||
caw,http://caw.wikia.com/
|
||||
ceramica,http://ceramica.wikia.com/
|
||||
championsonline,http://championsonline.wikia.com/
|
||||
chaotic,http://chaotic.wikia.com/
|
||||
charmed,http://charmed.wikia.com/
|
||||
chdk,http://chdk.wikia.com/
|
||||
chowder,http://chowder.wikia.com/
|
||||
christianity,http://christianity.wikia.com/
|
||||
christmasspecials,http://christmas-specials.wikia.com/
|
||||
chrono,http://chrono.wikia.com/
|
||||
cimmeria_aoc,http://cimmeria-aoc.wikia.com/
|
||||
cities,http://cities.wikia.com/
|
||||
cloverfield,http://cloverfield.wikia.com/
|
||||
clubchicken,http://unclubpenguin.wikia.com/
|
||||
clubpenguin,http://clubpenguin.wikia.com/
|
||||
cnc,http://cnc.wikia.com/
|
||||
codegeass,http://codegeass.wikia.com/
|
||||
codelyoko,http://codelyoko.wikia.com/
|
||||
combatarms,http://combatarms.wikia.com/
|
||||
comics,http://comics.wikia.com/
|
||||
communitytest,http://communitytest.wikia.com/
|
||||
compedia,http://compedia.wikia.com/
|
||||
conlang,http://conlang.wikia.com/
|
||||
conmyth,http://conmyth.wikia.com/
|
||||
conworld,http://conworld.wikia.com/
|
||||
cookbook_import,http://recipes.wikia.com/
|
||||
costuming,http://halocostuming.wikia.com/
|
||||
crashban,http://crashbandicoot.wikia.com/
|
||||
crazybobs,http://crazybobs.wikia.com/
|
||||
creatures,http://creatures.wikia.com/
|
||||
crimeprobe,http://publicsafety.wikia.com/
|
||||
criticalmass,http://criticalmass.wikia.com/
|
||||
custombionicle,http://custombionicle.wikia.com/
|
||||
cybernations,http://cybernations.wikia.com/
|
||||
d20npcs,http://d20npcs.wikia.com/
|
||||
danball,http://danball.wikia.com/
|
||||
darkfall,http://darkfall.wikia.com/
|
||||
darkhorizons,http://darkhorizons.wikia.com/
|
||||
darkmoonfaire,http://darkmoonfaire.wikia.com/
|
||||
darkrunescape,http://darkrunescape.wikia.com/
|
||||
darrenshan,http://cirquedufreak.wikia.com/
|
||||
darth,http://darth.wikia.com/
|
||||
dauncyclopedia,http://spademanns.wikia.com/
|
||||
dcanimated,http://dcanimated.wikia.com/
|
||||
de,http://de.wikia.com/
|
||||
deadfrontier,http://deadfrontier.wikia.com/
|
||||
deadoralive,http://deadoralive.wikia.com/
|
||||
deadrising,http://deadrising.wikia.com/
|
||||
deadspace,http://deadspace.wikia.com/
|
||||
deaokb,http://de.anarchyonline.wikia.com/
|
||||
deathnote,http://deathnote.wikia.com/
|
||||
debionicle,http://de.bionicle.wikia.com/
|
||||
debleachwiki,http://de.bleach.wikia.com/
|
||||
dedenaruto3,http://de.naruto.wikia.com/
|
||||
dedigimon,http://de.digimon.wikia.com/
|
||||
dedofus,http://de.dofuswiki.wikia.com/
|
||||
dedragonball,http://de.dragonball.wikia.com/
|
||||
deerziehungs,http://de.erziehungs.wikia.com/
|
||||
deffxi,http://de.wiki.ffxiclopedia.org/
|
||||
defiaseu,http://defiaseu.wikia.com/
|
||||
deflyff,http://de.flyff.wikia.com/
|
||||
degelsenkirchen,http://gelsenkirchen.wikia.com/
|
||||
degrassi,http://degrassi.wikia.com/
|
||||
degta,http://de.gta.wikia.com/
|
||||
dehalo,http://de.halo.wikia.com/
|
||||
dekuenstler,http://de.kuenstler.wikia.com/
|
||||
deliteratur,http://de.literatur.wikia.com/
|
||||
delostpedia,http://de.lostpedia.wikia.com/
|
||||
delphi,http://delphi.wikia.com/
|
||||
demario,http://de.mario.wikia.com/
|
||||
demarjorie,http://marjorie.wikia.com/
|
||||
dememoryalpha,http://memory-alpha.org/de/
|
||||
demini,http://de.mini.wikia.com/
|
||||
demusic,http://musik.wikia.com/
|
||||
deonepiece,http://de.onepiece.wikia.com/
|
||||
dephineasandferb,http://de.phineasandferb.wikia.com/
|
||||
derunescape,http://de.runescape.wikia.com/
|
||||
descrubs,http://de.scrubs.wikia.com/
|
||||
desencyclopedie,http://desencyclopedie.wikia.com/
|
||||
desperate,http://desperatehousewives.wikia.com/
|
||||
desportverein,http://de.sportverein.wikia.com/
|
||||
detestwiki,http://de.testwiki.wikia.com/
|
||||
deunternehmen,http://unternehmen.wikia.com/
|
||||
devilmaycry,http://devilmaycry.wikia.com/
|
||||
deyugioh2,http://de.yugioh.wikia.com/
|
||||
dgrayman,http://dgrayman.wikia.com/
|
||||
diablo,http://diablo.wikia.com/
|
||||
diealdor,http://diealdor.wikia.com/
|
||||
digibutter,http://digibutter.wikia.com/
|
||||
digimon,http://digimon.wikia.com/
|
||||
dinosaurs,http://dinosaurs.wikia.com/
|
||||
disney,http://disney.wikia.com/
|
||||
disneychannel,http://disneychannel.wikia.com/
|
||||
dissidia,http://dissidia.wikia.com/
|
||||
dizzywood,http://dizzywood.wikia.com/
|
||||
dnd,http://dnd.wikia.com/
|
||||
dndta,http://dndta.wikia.com/
|
||||
dofus,http://dofuswiki.wikia.com/
|
||||
dollhouse,http://dollhouse.wikia.com/
|
||||
doom,http://doom.wikia.com/
|
||||
dothack,http://dothack.wikia.com/
|
||||
dragonage,http://dragonage.wikia.com/
|
||||
dragonball,http://dragonball.wikia.com/
|
||||
drakensang,http://drakensang.wikia.com/
|
||||
dune,http://dune.wikia.com/
|
||||
dungeons,http://dungeons.wikia.com/
|
||||
dynastywarriors,http://koei.wikia.com/
|
||||
earthbound,http://earthbound.wikia.com/
|
||||
earthenring,http://earthenring.wikia.com/
|
||||
eberron,http://eberron.wikia.com/
|
||||
ed,http://ed.wikia.com/
|
||||
edfanon,http://edfanon.wikia.com/
|
||||
education,http://education.wikia.com/
|
||||
egamia,http://gaming.wikia.com/
|
||||
eincyclopedia,http://eincyclopedia.wikia.com/
|
||||
elderscrolls,http://elderscrolls.wikia.com/
|
||||
elementsthegame,http://elementsthegame.wikia.com/
|
||||
elona,http://elona.wikia.com/
|
||||
elwow,http://el.wow.wikia.com/
|
||||
enbleach,http://bleach.wikia.com/
|
||||
endcdatabase,http://dc.wikia.com/
|
||||
endomo,http://domo.wikia.com/
|
||||
enfairlyoddparents,http://fairlyoddparents.wikia.com/
|
||||
enhabbo,http://habbo.wikia.com/
|
||||
enkirby,http://kirby.wikia.com/
|
||||
enmarveldatabase,http://marvel.wikia.com/
|
||||
enmemoryalpha,http://memory-alpha.org/en/
|
||||
ennintendo,http://nintendo.wikia.com/
|
||||
enprinceofpersia,http://princeofpersia.wikia.com/
|
||||
enshamanking,http://shamanking.wikia.com/
|
||||
entabulawiki,http://tabularasa.wikia.com/
|
||||
entekken,http://tekken.wikia.com/
|
||||
entertainment1,http://entertainment.wikia.com/
|
||||
entranshumanism,http://transhumanism.wikia.com/
|
||||
entravel,http://travel.wikia.com/
|
||||
entravian,http://travian.wikia.com/
|
||||
eq2i,http://eq2.wikia.com/
|
||||
es,http://es.wikia.com/
|
||||
esanswers,http://respuestas.wikia.com/
|
||||
esben10,http://es.ben10.wikia.com/
|
||||
esbionicle,http://es.bionicle.wikia.com/
|
||||
esbleach,http://es.bleach.wikia.com/
|
||||
esdigimon,http://es.digimon.wikia.com/
|
||||
esdofus,http://es.dofuswiki.wikia.com/
|
||||
esdragonball,http://es.dragonball.wikia.com/
|
||||
esdrama,http://es.drama.wikia.com/
|
||||
esfakemon,http://es.fakemon.wikia.com/
|
||||
esfinalfantasy,http://es.finalfantasy.wikia.com/
|
||||
esgta,http://es.gta.wikia.com/
|
||||
eshalo,http://es.halo.wikia.com/
|
||||
esharrypotter,http://es.harrypotter.wikia.com/
|
||||
esiao,http://imperiumao.wikia.com/
|
||||
eslossimpson,http://es.simpsons.wikia.com/
|
||||
eslostpedia,http://es.lostpedia.wikia.com/
|
||||
esmortalkombat,http://es.mortalkombat.wikia.com/
|
||||
esnaruto,http://es.naruto.wikia.com/
|
||||
espokemon,http://es.pokemon.wikia.com/
|
||||
esrunescape,http://es.runescape.wikia.com/
|
||||
esstarwars,http://es.starwars.wikia.com/
|
||||
estibia,http://es.tibia.wikia.com/
|
||||
esticnologia,http://es.ticnologia.wikia.com/
|
||||
eswow,http://es.wow.wikia.com/
|
||||
esyugioh,http://es.yugioh.wikia.com/
|
||||
eszelda,http://es.zelda.wikia.com/
|
||||
evchk,http://evchk.wikia.com/
|
||||
evctw,http://evctw.wikia.com/
|
||||
eve,http://eve.wikia.com/
|
||||
events2,http://wowwikievent.wikia.com/
|
||||
ewrestling,http://ewrestling.wikia.com/
|
||||
fable,http://fable.wikia.com/
|
||||
facepunch,http://facepunch.wikia.com/
|
||||
fahrrad,http://fahrrad.wikia.com/
|
||||
fairytail,http://fairytail.wikia.com/
|
||||
fallout,http://fallout.wikia.com/
|
||||
falloutfanon,http://falloutfanon.wikia.com/
|
||||
falloutmods,http://falloutmods.wikia.com/
|
||||
familyguy,http://familyguy.wikia.com/
|
||||
fanfiction,http://fanfiction.wikia.com/
|
||||
fanon,http://fanon.wikia.com/
|
||||
fantendo,http://fantendo.wikia.com/
|
||||
fargofilmmaking,http://fargofilmmaking.wikia.com/
|
||||
farmville,http://farmville.wikia.com/
|
||||
farscape,http://farscape.wikia.com/
|
||||
fci,http://fci.wikia.com/
|
||||
fear,http://fear.wikia.com/
|
||||
ferrocarriles,http://ferrocarriles.wikia.com/
|
||||
ffxi,http://wiki.ffxiclopedia.org/
|
||||
fibionicle,http://fi.bionicle.wikia.com/
|
||||
film,http://film.wikia.com/
|
||||
filmguide,http://filmguide.wikia.com/
|
||||
finalfantasy,http://finalfantasy.wikia.com/
|
||||
fireemblem,http://fireemblem.wikia.com/
|
||||
firefly,http://firefly.wikia.com/
|
||||
firunescape,http://fi.runescape.wikia.com/
|
||||
fishwrangler,http://fishwrangler.wikia.com/
|
||||
fistarwars,http://fi.starwars.wikia.com/
|
||||
flapjack,http://flapjack.wikia.com/
|
||||
flashforward,http://flashforward.wikia.com/
|
||||
fma,http://fma.wikia.com/
|
||||
forgottenrealms,http://forgottenrealms.wikia.com/
|
||||
forscherliga,http://forscherliga.wikia.com/
|
||||
fraleatexte,http://fr.aleatexte.wikia.com/
|
||||
franswers,http://reponses.wikia.com/
|
||||
frbrunux,http://fr.brunux.wikia.com/
|
||||
frcontemporain,http://fr.contemporain.wikia.com/
|
||||
freeciv,http://freeciv.wikia.com/
|
||||
freespeech,http://freespeech.wikia.com/
|
||||
frffxi,http://fr.wiki.ffxiclopedia.org/
|
||||
frfilm,http://fr.film.wikia.com/
|
||||
frfr,http://fr.wikia.com/
|
||||
frguildwars,http://fr.guildwars.wikia.com/
|
||||
friends,http://friends.wikia.com/
|
||||
fringe,http://fringe.wikia.com/
|
||||
frlostpedia,http://fr.lostpedia.wikia.com/
|
||||
frmemoryalpha,http://memory-alpha.org/fr/
|
||||
frugooscape,http://frugooscape.wikia.com/
|
||||
frwow,http://fr.wowwiki.com/
|
||||
funorb,http://funorb.wikia.com/
|
||||
furry,http://furry.wikia.com/
|
||||
fusionfall,http://fusionfall.wikia.com/
|
||||
futurama,http://futurama.wikia.com/
|
||||
future,http://future.wikia.com/
|
||||
gaia,http://gaia.wikia.com/
|
||||
galava,http://galava.wikia.com/
|
||||
galciv,http://galciv.wikia.com/
|
||||
games,http://games.wikia.com/
|
||||
gearsofwar,http://gearsofwar.wikia.com/
|
||||
geekfeminism,http://geekfeminism.wikia.com/
|
||||
genealogy,http://familypedia.wikia.com/
|
||||
ghostbusters,http://ghostbusters.wikia.com/
|
||||
gijoe,http://gijoe.wikia.com/
|
||||
girlgenius,http://girlgenius.wikia.com/
|
||||
globalmedia,http://globalmedia.wikia.com/
|
||||
godfather,http://godfather.wikia.com/
|
||||
godofwar,http://godofwar.wikia.com/
|
||||
godzilla,http://godzilla.wikia.com/
|
||||
goldensun,http://goldensun.wikia.com/
|
||||
google,http://google.wikia.com/
|
||||
government,http://government.wikia.com/
|
||||
gowfanon,http://gowfanon.wikia.com/
|
||||
grandchase,http://grandchase.wikia.com/
|
||||
green,http://green.wikia.com/
|
||||
greenlantern,http://greenlantern.wikia.com/
|
||||
gtawiki,http://gta.wikia.com/
|
||||
guestbook,http://guestbook.wikia.com/
|
||||
guildopedia,http://guildopedia.wikia.com/
|
||||
guilds,http://neopets.wikia.com/
|
||||
guildwarsguilds,http://guildwarsguilds.wikia.com/
|
||||
guitarhero,http://guitarhero.wikia.com/
|
||||
gundam,http://gundam.wikia.com/
|
||||
guns,http://guns.wikia.com/
|
||||
gwguild,http://guildwars.wikia.com/
|
||||
halflife,http://half-life.wikia.com/
|
||||
halo,http://halo.wikia.com/
|
||||
halofanon,http://halofanon.wikia.com/
|
||||
halomachinima,http://halomachinima.wikia.com/
|
||||
hannahmontana,http://hannahmontana.wikia.com/
|
||||
happytreefriends,http://happytreefriends.wikia.com/
|
||||
harrypotter,http://harrypotter.wikia.com/
|
||||
harrypotterfanon,http://harrypotterfanon.wikia.com/
|
||||
help,http://help.wikia.com/
|
||||
heroes,http://heroes.wikia.com/
|
||||
hetalia,http://hetalia.wikia.com/
|
||||
hilfe,http://hilfe.wikia.com/
|
||||
hiliterature,http://hi.literature.wikia.com/
|
||||
hitman,http://hitman.wikia.com/
|
||||
hm,http://harvestmoon.wikia.com/
|
||||
homeandaway,http://homeandaway.wikia.com/
|
||||
homeworld,http://homeworld.wikia.com/
|
||||
horrormovies,http://horror-movies.wikia.com/
|
||||
hotwheels,http://hotwheels.wikia.com/
|
||||
house,http://house.wikia.com/
|
||||
howto,http://how-to.wikia.com/
|
||||
hrwooky,http://hrwooky.wikia.com/
|
||||
hsm,http://high-school-musical.wikia.com/
|
||||
huspam,http://hu.spam.wikia.com/
|
||||
hustarwars,http://hu.kaminopedia.wikia.com/
|
||||
icarly,http://icarly.wikia.com/
|
||||
icehockey,http://icehockey.wikia.com/
|
||||
ikariam,http://ikariam.wikia.com/
|
||||
ikkepedia,http://ikkjepedia.wikia.com/
|
||||
illogicopedia,http://wackypedia.wikia.com/
|
||||
imagine,http://imagine.wikia.com/
|
||||
inciclopedia,http://inciclopedia.wikia.com/
|
||||
india,http://india.wikia.com/
|
||||
indianajones,http://indianajones.wikia.com/
|
||||
infamous,http://infamous.wikia.com/
|
||||
inheritance,http://inheritance.wikia.com/
|
||||
internationalbusiness,http://internationalbusiness.wikia.com/
|
||||
inuyasha,http://inuyasha.wikia.com/
|
||||
ipod,http://apple.wikia.com/
|
||||
italia,http://italia.wikia.com/
|
||||
ithalo,http://it.halo.wikia.com/
|
||||
itlostpedia,http://it.lostpedia.wikia.com/
|
||||
itrunescape,http://it.runescape.wikia.com/
|
||||
itsoggettopedia,http://it.soggettopedia.wikia.com/
|
||||
ja,http://ja.wikia.com/
|
||||
jaeq2,http://ja.eq2.wikia.com/
|
||||
jakanddaxter,http://jakanddaxter.wikia.com/
|
||||
jamesbond,http://jamesbond.wikia.com/
|
||||
jamescameronsavatar,http://james-camerons-avatar.wikia.com/
|
||||
jet,http://jet.wikia.com/
|
||||
jfc,http://jfc.wikia.com/
|
||||
jfx,http://jfx.wikia.com/
|
||||
joinme,http://joinme.wikia.com/
|
||||
jurassicpark,http://jurassicpark.wikia.com/
|
||||
jvs,http://jvs.wikia.com/
|
||||
kanzaka,http://kanzaka.wikia.com/
|
||||
killzone,http://killzone.wikia.com/
|
||||
kimpossible,http://kimpossible.wikia.com/
|
||||
kingdomhearts,http://kingdomhearts.wikia.com/
|
||||
kitsch,http://kitsch.wikia.com/
|
||||
knd,http://knd.wikia.com/
|
||||
kongregate,http://kongregate.wikia.com/
|
||||
kosova,http://kosova.wikia.com/
|
||||
l5r,http://l5r.wikia.com/
|
||||
landbeforetime,http://landbeforetime.wikia.com/
|
||||
lastremnant,http://lastremnant.wikia.com/
|
||||
lawandorder,http://lawandorder.wikia.com/
|
||||
leagueoflegends,http://leagueoflegends.wikia.com/
|
||||
left,http://left.wikia.com/
|
||||
left4dead,http://left4dead.wikia.com/
|
||||
legacy,http://legacy.wikia.com/
|
||||
lego,http://lego.wikia.com/
|
||||
legobatman,http://legobatman.wikia.com/
|
||||
legoindianajones,http://legoindianajones.wikia.com/
|
||||
legostarwars,http://legostarwars.wikia.com/
|
||||
lgbt,http://lgbt.wikia.com/
|
||||
liberapedia,http://liberapedia.wikia.com/
|
||||
linux,http://linux.wikia.com/
|
||||
list,http://list.wikia.com/
|
||||
literature,http://literature.wikia.com/
|
||||
littlebigplanet,http://littlebigplanet.wikia.com/
|
||||
logocreation,http://logocreation.wikia.com/
|
||||
londonbirders,http://londonbirders.wikia.com/
|
||||
looneytunes,http://looneytunes.wikia.com/
|
||||
lost,http://lost.wikia.com/
|
||||
lostodyssey,http://lostodyssey.wikia.com/
|
||||
lostpedia,http://lostpedia.wikia.com/
|
||||
lotr,http://lotr.wikia.com/
|
||||
lotrowiki,http://lotro.wikia.com/
|
||||
lunarwars,http://lunarwars.wikia.com/
|
||||
madnesscombat,http://madnesscombat.wikia.com/
|
||||
mafiawars,http://mafiawars.wikia.com/
|
||||
maplestory,http://maplestory.wikia.com/
|
||||
mario,http://mario.wikia.com/
|
||||
mariokart,http://mariokart.wikia.com/
|
||||
marvelmovies,http://marvel-movies.wikia.com/
|
||||
masseffect,http://masseffect.wikia.com/
|
||||
math,http://math.wikia.com/
|
||||
matrixfilms,http://matrix.wikia.com/
|
||||
mcleodgaming,http://mcleodgaming.wikia.com/
|
||||
mechquest,http://mechquest.wikia.com/
|
||||
mechscape,http://stellardawn.wikia.com/
|
||||
megaman,http://megaman.wikia.com/
|
||||
megamitensei,http://megamitensei.wikia.com/
|
||||
memory_alfa,http://memory-alpha.org/it/
|
||||
memory_gamma,http://memory-gamma.wikia.com/
|
||||
metal,http://metal.wikia.com/
|
||||
metalgear,http://metalgear.wikia.com/
|
||||
metroid,http://metroid.wikia.com/
|
||||
micronaciones,http://micronaciones.wikia.com/
|
||||
micronations,http://micronations.wikia.com/
|
||||
micropedia,http://micropedia.wikia.com/
|
||||
mind_control,http://mind-control.wikia.com/
|
||||
misterwikki,http://misterwikki.wikia.com/
|
||||
mk,http://mortalkombat.wikia.com/
|
||||
monde,http://monde.wikia.com/
|
||||
monsterhunter,http://monsterhunter.wikia.com/
|
||||
moomin,http://moomin.wikia.com/
|
||||
moonguard,http://moonguard.wikia.com/
|
||||
mozilla,http://mozilla.wikia.com/
|
||||
mpd,http://mpd.wikia.com/
|
||||
mspafa,http://mspafa.wikia.com/
|
||||
mspaintadventures,http://mspaintadventures.wikia.com/
|
||||
mst3k,http://mst3k.wikia.com/
|
||||
mtg,http://mtg.wikia.com/
|
||||
mu,http://mu.wikia.com/
|
||||
muppet,http://muppet.wikia.com/
|
||||
music,http://music.wikia.com/
|
||||
mybio,http://mybio.wikia.com/
|
||||
myeloma,http://myeloma.wikia.com/
|
||||
mylegonetwork,http://mylegonetwork.wikia.com/
|
||||
mysims,http://mysims.wikia.com/
|
||||
n,http://n.wikia.com/
|
||||
namcotales,http://namcotales.wikia.com/
|
||||
narnia,http://narnia.wikia.com/
|
||||
naruto,http://naruto.wikia.com/
|
||||
narutofanon,http://narutofanon.wikia.com/
|
||||
nascar,http://thirdturn.wikia.com/
|
||||
nation,http://nation.wikia.com/
|
||||
nationstates,http://nationstates.wikia.com/
|
||||
ncis,http://ncis.wikia.com/
|
||||
necyklopedie,http://necyklopedie.wikia.com/
|
||||
nethack,http://nethack.wikia.com/
|
||||
newgrounds,http://newgrounds.wikia.com/
|
||||
nfl,http://nfl.wikia.com/
|
||||
nickelodeon,http://nickelodeon.wikia.com/
|
||||
nlmemoryalpha,http://memory-alpha.org/nl/
|
||||
nlrunescape,http://nl.runescape.wikia.com/
|
||||
nlstarwars,http://nl.starwars.wikia.com/
|
||||
nonciclopedia,http://nonciclopedia.wikia.com/
|
||||
nonsensopedia,http://nonsensopedia.wikia.pl/
|
||||
norunescape,http://no.runescape.wikia.com/
|
||||
novelas,http://fiction.wikia.com/
|
||||
nowow,http://no.wowwiki.com/
|
||||
nwn,http://nwn.wikia.com/
|
||||
nwn2,http://nwn2.wikia.com/
|
||||
nwp,http://nwp.wikia.com/
|
||||
oblivion,http://oblivion.wikia.com/
|
||||
oddworld,http://oddworld.wikia.com/
|
||||
offenbach,http://offenbach.wikia.com/
|
||||
ogame,http://ogame.wikia.com/
|
||||
olympians,http://percyjackson.wikia.com/
|
||||
onepiece,http://onepiece.wikia.com/
|
||||
onepiecefanon,http://onepiecefanon.wikia.com/
|
||||
opensource,http://opensource.wikia.com/
|
||||
othertitles,http://othertitles.wikia.com/
|
||||
otherverse,http://otherverse.wikia.com/
|
||||
ourbrant,http://ourbrant.wikia.com/
|
||||
oz,http://oz.wikia.com/
|
||||
pakistan,http://pakistan.wikia.com/
|
||||
papermario,http://papermario.wikia.com/
|
||||
paradisa,http://paradisa.wikia.com/
|
||||
paragon,http://cityofheroes.wikia.com/
|
||||
particracy,http://particracy.wikia.com/
|
||||
pathfinder,http://pathfinder.wikia.com/
|
||||
pffanon,http://pffanon.wikia.com/
|
||||
phineasandferb,http://phineasandferb.wikia.com/
|
||||
pikmin,http://pikmin.wikia.com/
|
||||
pinball,http://pinball.wikia.com/
|
||||
pirates,http://pirates.wikia.com/
|
||||
piratesonline,http://piratesonline.wikia.com/
|
||||
pixar,http://pixar.wikia.com/
|
||||
plbionicle,http://pl.bionicle.wikia.com/
|
||||
plcustombionicles,http://pl.custombionicles.wikia.com/
|
||||
plfallout,http://pl.fallout.wikia.com/
|
||||
plgothic,http://pl.gothic.wikia.com/
|
||||
plharrypotter,http://pl.harrypotter.wikia.com/
|
||||
pllostpedia,http://pl.lostpedia.wikia.com/
|
||||
plogame,http://pl.ogame.wikia.com/
|
||||
plstarwars,http://pl.starwars.wikia.com/
|
||||
plwikia,http://pl.wikia.com/
|
||||
plwow,http://pl.wow.wikia.com/
|
||||
plwykopedia,http://pl.wykopedia.wikia.com/
|
||||
pokemon,http://pokemon.wikia.com/
|
||||
pokemonfanon,http://pokemonfanon.wikia.com/
|
||||
pokemononline339,http://pokemononline.wikia.com/
|
||||
poker,http://poker.wikia.com/
|
||||
potbs,http://potbs.wikia.com/
|
||||
pov,http://opinion.wikia.com/
|
||||
powerrangers,http://powerrangers.wikia.com/
|
||||
ppc,http://ppc.wikia.com/
|
||||
primeval,http://primeval.wikia.com/
|
||||
prisonbreak,http://prisonbreak.wikia.com/
|
||||
prototype,http://prototype.wikia.com/
|
||||
prowrestling,http://prowrestling.wikia.com/
|
||||
psychology,http://psychology.wikia.com/
|
||||
ptdofus,http://pt.dofuspedia.wikia.com/
|
||||
ptlostpedia,http://pt.lostpedia.wikia.com/
|
||||
ptmicronations,http://pt.micronations.wikia.com/
|
||||
ptsimpsons,http://pt.simpsons.wikia.com/
|
||||
ptstarwars,http://pt.starwars.wikia.com/
|
||||
pttibia,http://pt.tibia.wikia.com/
|
||||
punchout,http://punchout.wikia.com/
|
||||
puppet,http://puppet.wikia.com/
|
||||
pushingdaisies,http://pushing-daisies.wikia.com/
|
||||
pvx,http://pvx.wikia.com/
|
||||
rainbowsix,http://rainbowsix.wikia.com/
|
||||
rappelz,http://rappelz.wikia.com/
|
||||
ratchet,http://ratchet.wikia.com/
|
||||
reboot,http://reboot.wikia.com/
|
||||
reborn,http://reborn.wikia.com/
|
||||
reddwarf,http://reddwarf.wikia.com/
|
||||
redwall,http://redwall.wikia.com/
|
||||
religionwiki,http://religion.wikia.com/
|
||||
renaissancekingdoms,http://renaissancekingdoms.wikia.com/
|
||||
requests,http://www.wikia.com/
|
||||
residentevil,http://residentevil.wikia.com/
|
||||
resistancefallofman,http://resistance.wikia.com/
|
||||
robotchicken3,http://robotchicken.wikia.com/
|
||||
robotwars,http://robotwars.wikia.com/
|
||||
rocketboom,http://rocketboom.wikia.com/
|
||||
rom,http://rom.wikia.com/
|
||||
routes,http://routes.wikia.com/
|
||||
rpg,http://rpg.wikia.com/
|
||||
rpgmaker,http://rpgmaker.wikia.com/
|
||||
ruharrypotter,http://ru.harrypotter.wikia.com/
|
||||
ruhistory,http://ru.history.wikia.com/
|
||||
rulostpedia,http://ru.lostpedia.wikia.com/
|
||||
runescape,http://runescape.wikia.com/
|
||||
runescapeclans,http://runescapeclans.wikia.com/
|
||||
runescapefanfiction,http://runescapefanfiction.wikia.com/
|
||||
rurpg,http://ru.rpg.wikia.com/
|
||||
ruscience,http://ru.science.wikia.com/
|
||||
rustarwars,http://ru.starwars.wikia.com/
|
||||
ruvlab,http://ru.vlab.wikia.com/
|
||||
ruwikia,http://ru.wikia.com/
|
||||
rvb,http://rvb.wikia.com/
|
||||
sacredseasons,http://sacredseasons.wikia.com/
|
||||
sailormoon,http://sailormoon.wikia.com/
|
||||
saintsrow,http://saintsrow.wikia.com/
|
||||
sarahtestwiki,http://sarahtestwiki.wikia.com/
|
||||
sca21,http://sca21.wikia.com/
|
||||
scarteleu,http://scarteleu.wikia.com/
|
||||
schoolcomputing,http://schoolcomputing.wikia.com/
|
||||
schools,http://schools.wikia.com/
|
||||
schulen,http://schulen.wikia.com/
|
||||
scoobydoo,http://scoobydoo.wikia.com/
|
||||
scratchpad,http://scratchpad.wikia.com/
|
||||
scrubs,http://scrubs.wikia.com/
|
||||
scum,http://scum.wikia.com/
|
||||
sealonline,http://sealonline.wikia.com/
|
||||
search,http://searchwiki.wikia.com/
|
||||
seattle,http://seattle.wikia.com/
|
||||
secondlife,http://secondlife.wikia.com/
|
||||
secretsaturdays,http://secretsaturdays.wikia.com/
|
||||
sega,http://sega.wikia.com/
|
||||
sentinels,http://sentinels.wikia.com/
|
||||
shaiya,http://shaiya.wikia.com/
|
||||
shugochara,http://shugochara.wikia.com/
|
||||
silent,http://silenthill.wikia.com/
|
||||
simpsons,http://simpsons.wikia.com/
|
||||
sims,http://sims.wikia.com/
|
||||
sknecyklopedia,http://necyklopedia.wikia.com/
|
||||
skyrates,http://skyrates.wikia.com/
|
||||
smallville,http://smallville.wikia.com/
|
||||
smashtasm,http://smashtasm.wikia.com/
|
||||
snicket,http://snicket.wikia.com/
|
||||
snk,http://snk.wikia.com/
|
||||
solarcooking,http://solarcooking.wikia.com/
|
||||
sonic,http://sonic.wikia.com/
|
||||
sonicfanon,http://sonicfanon.wikia.com/
|
||||
sonnywithachance,http://sonnywithachance.wikia.com/
|
||||
sot,http://sot.wikia.com/
|
||||
soulcalibur,http://soulcalibur.wikia.com/
|
||||
souleater,http://souleater.wikia.com/
|
||||
southpark,http://southpark.wikia.com/
|
||||
spacequest,http://spacequest.wikia.com/
|
||||
spiderman,http://spiderman.wikia.com/
|
||||
splintercell,http://splintercell.wikia.com/
|
||||
spongebob,http://spongebob.wikia.com/
|
||||
spongefan,http://spongefan.wikia.com/
|
||||
spore,http://spore.wikia.com/
|
||||
spyro,http://spyro.wikia.com/
|
||||
ssb,http://super-smash-bros.wikia.com/
|
||||
stad,http://stad.wikia.com/
|
||||
stalker,http://stalker.wikia.com/
|
||||
stanford,http://stanford.wikia.com/
|
||||
starcraft,http://starcraft.wikia.com/
|
||||
starfox,http://starfox.wikia.com/
|
||||
stargate,http://stargate.wikia.com/
|
||||
stargatewars,http://stargatewars.wikia.com/
|
||||
starocean,http://starocean.wikia.com/
|
||||
startingstrength,http://startingstrength.wikia.com/
|
||||
startrek,http://memory-beta.wikia.com/
|
||||
starwarsexodus,http://starwars-exodus.wikia.com/
|
||||
starwarsfr,http://fr.starwars.wikia.com/
|
||||
starwarsmush,http://starwarsmush.wikia.com/
|
||||
stexpanded,http://stexpanded.wikia.com/
|
||||
streetfighter,http://streetfighter.wikia.com/
|
||||
students,http://students.wikia.com/
|
||||
suitelife,http://suitelife.wikia.com/
|
||||
supcom,http://supcom.wikia.com/
|
||||
superman,http://superman.wikia.com/
|
||||
supernatural,http://supernatural.wikia.com/
|
||||
svtibia,http://sv.tibia.wikia.com/
|
||||
sw1mush,http://sw1mush.wikia.com/
|
||||
swfanon,http://swfanon.wikia.com/
|
||||
swfans,http://swfans.wikia.com/
|
||||
swg,http://swg.wikia.com/
|
||||
swgames,http://swgames.wikia.com/
|
||||
swmerchandise,http://swmerchandise.wikia.com/
|
||||
swrp,http://swrp.wikia.com/
|
||||
swrpg,http://swrpg.wikia.com/
|
||||
tamagotchi,http://tamagotchi.wikia.com/
|
||||
tardis,http://tardis.wikia.com/
|
||||
tdicamps,http://tdicamps.wikia.com/
|
||||
teentitans,http://teentitans.wikia.com/
|
||||
terminator,http://terminator.wikia.com/
|
||||
tesfanon,http://tesfanon.wikia.com/
|
||||
thatguywiththeglasses,http://thatguywiththeglasses.wikia.com/
|
||||
thegungancouncil,http://thegungancouncil.wikia.com/
|
||||
theoffice,http://theoffice.wikia.com/
|
||||
theppn_backup,http://ppn.wikia.com/
|
||||
theshatareu,http://theshatareu.wikia.com/
|
||||
theworldcc,http://theworldcc.wikia.com/
|
||||
thoriumbrotherhood,http://thoriumbrotherhood.wikia.com/
|
||||
tibiawiki,http://tibia.wikia.com/
|
||||
timesplitters,http://timesplitters.wikia.com/
|
||||
tmnt,http://tmnt.wikia.com/
|
||||
toastmasters,http://toastmasters.wikia.com/
|
||||
tokipona,http://tokipona.wikia.com/
|
||||
tolololpedia,http://tolololpedia.wikia.com/
|
||||
toohuman,http://toohuman.wikia.com/
|
||||
tor5,http://tor5.wikia.com/
|
||||
totalannihilation,http://totalannihilation.wikia.com/
|
||||
totaldramaisland,http://totaldramaisland.wikia.com/
|
||||
totaldramaislandfanfiction,http://totaldramaislandfanfiction.wikia.com/
|
||||
touhou,http://touhou.wikia.com/
|
||||
transfanon,http://transfanon.wikia.com/
|
||||
transformers,http://transformers.wikia.com/
|
||||
transformers2005,http://transformers2005.wikia.com/
|
||||
traveller,http://traveller.wikia.com/
|
||||
trekkipedia,http://memory-alpha.org/es/
|
||||
trsg,http://trsg.wikia.com/
|
||||
trukz,http://trukz.wikia.com/
|
||||
tryenisehir,http://tr.yenisehir.wikia.com/
|
||||
tsdc,http://tsdc.wikia.com/
|
||||
ttte,http://ttte.wikia.com/
|
||||
turtledove,http://turtledove.wikia.com/
|
||||
twewy,http://twewy.wikia.com/
|
||||
twilightsaga,http://twilightsaga.wikia.com/
|
||||
u5lazarus,http://ultima.wikia.com/
|
||||
uktransport,http://uktransport.wikia.com/
|
||||
unanswers,http://unanswers.wikia.com/
|
||||
uncharted,http://uncharted.wikia.com/
|
||||
uncyclopedia_de,http://de.uncyclopedia.org/
|
||||
unhalo,http://unhalo.wikia.com/
|
||||
unmario,http://unmario.wikia.com/
|
||||
unrunescape,http://unrunescape.wikia.com/
|
||||
utau,http://utau.wikia.com/
|
||||
valenciclopedia,http://valenciclopedia.wikia.com/
|
||||
vampireknight,http://vampireknight.wikia.com/
|
||||
varamozhi,http://varamozhi.wikia.com/
|
||||
vereins,http://vereins.wikia.com/
|
||||
videogamelies,http://videogamelies.wikia.com/
|
||||
videojuego,http://videojuego.wikia.com/
|
||||
vikipedo,http://vikipedo.wikia.com/
|
||||
villains,http://villains.wikia.com/
|
||||
vim,http://vim.wikia.com/
|
||||
vintagepatterns,http://vintagepatterns.wikia.com/
|
||||
vocaloid,http://vocaloid.wikia.com/
|
||||
vongopedia,http://vongopedia.wikia.com/
|
||||
vsrecommendedgames,http://vsrecommendedgames.wikia.com/
|
||||
wakfu,http://wakfu.wikia.com/
|
||||
war,http://war.wikia.com/
|
||||
warhammer40k,http://warhammer40k.wikia.com/
|
||||
warhammeronline,http://warhammeronline.wikia.com/
|
||||
warriors,http://warriors.wikia.com/
|
||||
warriorsfanfic,http://warriorsfanfic.wikia.com/
|
||||
warszawa,http://warszawa.wikia.com/
|
||||
watchmen,http://watchmen.wikia.com/
|
||||
webkinz,http://webkinz.wikia.com/
|
||||
weirdal,http://weirdal.wikia.com/
|
||||
westwing,http://westwing.wikia.com/
|
||||
whitewolf,http://whitewolf.wikia.com/
|
||||
wiccana_aoc,http://wiccana-aoc.wikia.com/
|
||||
wiedzmin,http://wiedzmin.wikia.com/
|
||||
wii,http://wii.wikia.com/
|
||||
wikiality,http://wikiality.wikia.com/
|
||||
wikicities,http://community.wikia.com/
|
||||
wikimac,http://mac.wikia.com/
|
||||
wirtschaftpedia,http://wirtschaftpedia.wikia.com/
|
||||
witcher,http://witcher.wikia.com/
|
||||
wizard101,http://wizard101.wikia.com/
|
||||
wizardsofwaverlyplace,http://wizardsofwaverlyplace.wikia.com/
|
||||
wonderlandonline,http://wonderlandonline.wikia.com/
|
||||
woot,http://woot.wikia.com/
|
||||
wot,http://wot.wikia.com/
|
||||
wowrp,http://wowrp.wikia.com/
|
||||
wowwiki,http://www.wowwiki.com/
|
||||
wswiki,http://websitewiki.wikia.com/
|
||||
wyrmrest,http://wyrmrest.wikia.com/
|
||||
xboxanswers,http://xbox.answers.wikia.com/
|
||||
xfiles,http://x-files.wikia.com/
|
||||
xiaolinshowdown,http://xiaolinshowdown.wikia.com/
|
||||
xmen,http://x-men.wikia.com/
|
||||
yahoomediaplayer,http://yahoomediaplayer.wikia.com/
|
||||
ycm,http://ycm.wikia.com/
|
||||
yellowikis,http://yellowikis.wikia.com/
|
||||
youtube,http://youtube.wikia.com/
|
||||
yoyo,http://yoyowiki.org/
|
||||
yugioh,http://yugioh.wikia.com/
|
||||
zelda,http://zelda.wikia.com/
|
||||
zeldafanon,http://zeldafanon.wikia.com/
|
||||
zhhkhongkong,http://zh-hk.hongkong.wikia.com/
|
||||
zhscratchpad,http://zh.scratchpad.wikia.com/
|
||||
zhuncyclopedia,http://zh.uncyclopedia.wikia.com/
|
||||
zombie,http://zombie.wikia.com/
|
||||
zomg,http://zomg.wikia.com/
|
||||
zootycoon,http://zootycoon.wikia.com/
|
||||
|
199
userroles_scraper_scripts/userroles_from_listusers.py
Executable file
199
userroles_scraper_scripts/userroles_from_listusers.py
Executable file
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) 2018 Nathan TeBlunthuis
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import time, re, os
|
||||
import sys
|
||||
from importlib import reload
|
||||
reload(sys)
|
||||
|
||||
import urllib
|
||||
import requests
|
||||
import json
|
||||
import gzip
|
||||
|
||||
from pprint import pprint
|
||||
from itertools import islice
|
||||
import csv
|
||||
|
||||
roles = ['bot','sysop', 'bureaucrat','staff','rollback', # 'util',
|
||||
'helper', 'vstf', 'checkuser-global', 'bot-global',
|
||||
'council','authenticated', 'checkuser', 'chatmoderator',
|
||||
'adminmentor','steward','oversight','founder','rollbacker','checkuser','researcher']
|
||||
output_path = "userlist-2017/"
|
||||
class ListUserAPI():
|
||||
def __init__(self, url_root,wikitype):
|
||||
self.wikitype = wikitype
|
||||
if self.wikitype=="wikia":
|
||||
self._api_url = url_root + 'index.php?action=ajax&rs=ListusersAjax::axShowUsers'
|
||||
else: # wikitype == "wikipedia"
|
||||
self._api_url = url_root + 'api.php'
|
||||
|
||||
def _fetch_http(self, url, params):
|
||||
if self.wikitype == "wikia":
|
||||
response = requests.get(url=url, params=params,headers={'Accept-encoding':'gzip'})
|
||||
return(response.text)
|
||||
else: #wikitype == "wikipedia"
|
||||
response = requests.get(url=url, params=params)
|
||||
return(response)
|
||||
|
||||
def call(self, params):
|
||||
response = self._fetch_http(self._api_url, params)
|
||||
if self.wikitype == "wikia":
|
||||
return json.loads(response)
|
||||
else:
|
||||
return response.json()
|
||||
|
||||
|
||||
def write_user_csvfile(output_file, user_list):
|
||||
csvfile = csv.writer(output_file, delimiter='\t',
|
||||
quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
|
||||
|
||||
# construct and output the header
|
||||
csvfile.writerow(['username', 'groups',
|
||||
'edits', 'last.logged', 'last.edited'])
|
||||
|
||||
for user in user_list:
|
||||
csvfile.writerow(user)
|
||||
|
||||
|
||||
def get_administrators_for_wiki(wikiname, url_root, wikitype="wikia"):
|
||||
increment_size = 500
|
||||
offset = 0
|
||||
|
||||
if wikitype == "wikia":
|
||||
|
||||
query = {'groups' :'bot,sysop,bureaucrat,',
|
||||
'edits' : 0,
|
||||
'limit' : increment_size,
|
||||
'offset' : offset,
|
||||
'numOrder' : 1,
|
||||
'order' : 'username:asc' }
|
||||
|
||||
else: # wikitype == "wikipedia"
|
||||
query = {'action': 'query',
|
||||
'list': 'allusers',
|
||||
'augroup' : "|".join(roles),
|
||||
'auprop' : 'groups',
|
||||
'aulimit' : 500,
|
||||
'format' : 'json'}
|
||||
|
||||
## FIND THE CORRECT URL (there may be redirects)
|
||||
|
||||
if wikitype=="wikia":
|
||||
url_root = requests.get(url_root).url
|
||||
re_str = "^http://(community|www).wikia.com/"
|
||||
if re.match(re_str, url_root):
|
||||
# api_url 'http://community.wikia.com/wiki/Community_Central:Not_a_valid_Wikia':
|
||||
print("ERROR: %s no longer exists" % wikiname)
|
||||
|
||||
return "deleted"
|
||||
try:
|
||||
wiki = ListUserAPI(url_root,wikitype=wikitype)
|
||||
rv = wiki.call(query)
|
||||
|
||||
except requests.ConnectionError as e:
|
||||
print("ERROR: cannot read the event log: %s" % wikiname)
|
||||
notauthorized.append(wikiname)
|
||||
return "notauthorized"
|
||||
|
||||
if wikitype == "wikia":
|
||||
raw_userlist = rv['aaData']
|
||||
|
||||
while (rv['iTotalRecords'] + offset) < rv['iTotalDisplayRecords']:
|
||||
# increment the offset and make a new query
|
||||
offset = offset + increment_size
|
||||
query['offset'] = offset
|
||||
rv = wiki.call(query)
|
||||
raw_userlist.extend(rv['aaData'])
|
||||
print("Another one: offset is %s" % offset)
|
||||
|
||||
# go through and edit the html output of the json
|
||||
processed_userlist = []
|
||||
for row in raw_userlist:
|
||||
row[0] = re.sub(r'^.*?<a href=.*?>(.*?)<.*$', r'\1', row[0])
|
||||
row[4] = re.sub(r'^.*oldid=(\d+)".*$', r'\1', row[4])
|
||||
row[4] = re.sub(r'^\-$', r'', row[4])
|
||||
processed_userlist.append(row)
|
||||
|
||||
output_file = open("{0}/{1}.tsv".format(output_path, wikiname),'w')
|
||||
write_user_csvfile(output_file, processed_userlist)
|
||||
output_file.close()
|
||||
|
||||
else:
|
||||
output_file = open("{0}/{1}.tsv".format(output_path, wikiname),'w')
|
||||
raw_userlist = rv['query']['allusers']
|
||||
outlines = ['\t'.join(["username","groups"])]
|
||||
outlines.extend(['\t'.join([q['name'],','.join(q['groups'])]) for q in raw_userlist])
|
||||
output_file.write('\n'.join(outlines))
|
||||
outlines = []
|
||||
|
||||
while 'continue' in rv:
|
||||
query['continue'] = str(rv['continue'])
|
||||
query['aufrom']= str(rv['continue']['aufrom'])
|
||||
rv = wiki.call(query)
|
||||
raw_userlist = rv['query']['allusers']
|
||||
outlines.extend(['\t'.join([q['name'],','.join(q['groups'])]) for q in raw_userlist])
|
||||
output_file.write('\n'.join(outlines))
|
||||
output_file.flush()
|
||||
outlines = []
|
||||
|
||||
|
||||
# open and then send data to the output data file
|
||||
|
||||
# read in the a list of files so we can skip them if we're already
|
||||
# downloaded them
|
||||
files = [os.path.join(output_path, i) for i in os.listdir(output_path)]
|
||||
|
||||
# iterate through the list of files
|
||||
|
||||
# for line in open("list_of_wikis.csv", "r").readlines():
|
||||
# next line useful for working with a reduced list:
|
||||
d = [(line.split(",")[0], line.split(",")[1]) for line in islice(open("../wikis.needing.userroles.csv"),1,None)]
|
||||
|
||||
deleted = []
|
||||
notauthorized = []
|
||||
for wiki, url in d:
|
||||
wiki = wiki.strip()
|
||||
url = url.strip()
|
||||
print(url)
|
||||
if os.path.join(output_path,wiki+".tsv") in files:
|
||||
print("SKIPPING: file \"%s\" already exists)" % wiki)
|
||||
continue
|
||||
|
||||
print("Processing wiki: %s" % wiki)
|
||||
if "wikipedia.org" in url:
|
||||
wikitype = "wikipedia"
|
||||
url = url + '/w/'
|
||||
if "wikia.com" in url:
|
||||
wikitype = "wikia"
|
||||
|
||||
result = get_administrators_for_wiki(wiki, url, wikitype=wikitype)
|
||||
if result == "deleted":
|
||||
deleted.append(wiki)
|
||||
elif result == "notauthorized":
|
||||
notauthorized.append(wiki)
|
||||
else:
|
||||
pass
|
||||
time.sleep(1)
|
||||
|
||||
df = open("allusers_WP_error_deleted.txt",'w')
|
||||
df.write('\n'.join(deleted))
|
||||
df.close()
|
||||
|
||||
na = open("allusers_WP_error_notauthorized.txt",'w')
|
||||
na.write('\n'.join(notauthorized))
|
||||
na.close()
|
||||
158
userroles_scraper_scripts/userroles_from_logevents.py
Executable file
158
userroles_scraper_scripts/userroles_from_logevents.py
Executable file
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Copyright (C) 2018 Nathan TeBlunthuis
|
||||
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
import time, re, os
|
||||
import sys
|
||||
import requests
|
||||
from mw import api
|
||||
from pprint import pprint
|
||||
from json.decoder import JSONDecodeError
|
||||
from itertools import islice
|
||||
|
||||
def write_logevents(logevents,out):
|
||||
for logevent in logevents:
|
||||
# if there is hidden information, we skip this one because there
|
||||
# is nothing to report
|
||||
if 'userhidden' in logevent or 'actionhidden' in logevent or 'commenthidden' in logevent:
|
||||
continue
|
||||
|
||||
le_output = [logevent['comment'],
|
||||
str(logevent['logid']),
|
||||
str(logevent['ns']),
|
||||
str(logevent['pageid']),
|
||||
logevent['timestamp'],
|
||||
logevent['title'],
|
||||
logevent['type'],
|
||||
str(logevent['user'])]
|
||||
|
||||
if "params" in logevent:
|
||||
params = logevent["params"]
|
||||
else:
|
||||
params = {}
|
||||
|
||||
if "rights" in logevent:
|
||||
le_output.extend(['false',
|
||||
logevent['rights']['new'],
|
||||
logevent['rights']['old']])
|
||||
|
||||
|
||||
elif "newgroups" in params and "oldgroups" in params:
|
||||
le_output.extend(['false',
|
||||
','.join(params['newgroups']),
|
||||
','.join(params['oldgroups'])])
|
||||
else:
|
||||
le_output.extend(['true', '', ''])
|
||||
|
||||
out.write("\t".join(le_output) + "\n")
|
||||
out.flush()
|
||||
# output data
|
||||
|
||||
def get_events_for_wiki(wikiname, url, wikitype="wikia"):
|
||||
if url[-1] != '/':
|
||||
url = url + '/'
|
||||
|
||||
#out = open("../wikipedias/adminlist_output/logevents/nobackup/%s.tsv" % wikiname, "w")
|
||||
out = open("logevents-2017/%s.tsv" % wikiname, "w")
|
||||
out.write("\t".join(['comment', 'logid', 'ns', 'pageid', 'timestamp','title', 'type', 'user', 'ancient', 'rights-new', 'rights-old\n']))
|
||||
|
||||
if wikitype == "wikia":
|
||||
api_url = url + 'api.php'
|
||||
else: #wikitype == wikipedia
|
||||
api_url = url + "w/api.php"
|
||||
|
||||
query = {'action': 'query',
|
||||
'list': 'logevents',
|
||||
'letype' : 'rights',
|
||||
'lelimit' : '500',
|
||||
'format':'json',
|
||||
'ledir':'newer'}
|
||||
|
||||
response = requests.get(api_url, params=query)
|
||||
hit_url = response.url
|
||||
|
||||
if wikitype == "wikia":
|
||||
re_str = "^http://(community|www)\.wikia\.com/"
|
||||
if re.match(re_str, hit_url):
|
||||
# api_url 'http://community.wikia.com/wiki/Community_Central:Not_a_valid_Wikia':
|
||||
print("ERROR: %s no longer exists" % wikiname)
|
||||
return
|
||||
else:
|
||||
re_str = "^(http|https)://.*\.wikia.com/api\.php"
|
||||
if re.match(re_str, hit_url):
|
||||
try:
|
||||
## this is the only way out
|
||||
rv = response.json()
|
||||
## check that we hit the right wiki
|
||||
except (JSONDecodeError):
|
||||
print(" New Error! ")
|
||||
else:
|
||||
re_str = "^((http|https)://.*\.wikia\.com)"
|
||||
new_url = re.findall(re_str, hit_url)[0][0]
|
||||
return get_events_for_wiki(wikiname, new_url, wikitype=wikitype)
|
||||
|
||||
try:
|
||||
logevents = rv['query']['logevents']
|
||||
write_logevents(logevents, out)
|
||||
except KeyError as e:
|
||||
print("ERROR: %s contains no logevent data" % wikiname)
|
||||
print(e)
|
||||
return
|
||||
|
||||
while 'query-continue' in rv or 'continue' in rv:
|
||||
if 'query-continue' in rv:
|
||||
query['lestart'] = rv['query-continue']['logevents']['lestart']
|
||||
else:
|
||||
query['continue'] = str(rv['continue'])
|
||||
query['lecontinue'] = str(rv['continue']['lecontinue'])
|
||||
|
||||
response = requests.get(api_url,params=query)
|
||||
rv = response.json()
|
||||
logevents=rv['query']['logevents']
|
||||
write_logevents(logevents, out)
|
||||
|
||||
out.close()
|
||||
|
||||
files = [re.sub('\.tsv$', '', i) for i in os.listdir("logevents-2017")]
|
||||
|
||||
# interate through the list of wikis
|
||||
#for line in ["anime,http://anime.wikia.com/"]:
|
||||
#for line in ["blogging,http://blogging.wikia.com/"]:
|
||||
header = True
|
||||
if header:
|
||||
i = 1
|
||||
else:
|
||||
i = 0
|
||||
|
||||
# for line in open("list_of_wikis.csv", "r").readlines():
|
||||
for line in islice(open("../wikis.needing.userroles.csv", "r"),i,None):
|
||||
|
||||
(wiki, url) = line.split(",")
|
||||
url = url.strip()
|
||||
print("Processing wiki: %s" % wiki)
|
||||
|
||||
if wiki in files:
|
||||
print("SKIPPING: file \"%s\" already exists)" % wiki)
|
||||
continue
|
||||
|
||||
if "wikia.com" in url:
|
||||
wikitype = "wikia"
|
||||
else:# "wikipedia.org in url":
|
||||
wikitype = "wikipedia"
|
||||
|
||||
|
||||
get_events_for_wiki(wiki, url, wikitype=wikitype)
|
||||
time.sleep(1)
|
||||
Reference in New Issue
Block a user