groceryheist/wikia_userroles_scraper
1
0
Fork 0
Code Activity

add copyright notice

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2018-06-02 14:56:49 -07:00
parent c7e2daffcc
commit 29337a8e2f
6 changed files with 52 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scrape_list.sh
View File

@@ -1,5 +1,13 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Run userroles_from_listusers.py
# Copyright (C) 2018 Nathan TeBlunthuis
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# you specify a csv file that has a list of wikis # you specify a csv file that has a list of wikis
wikilist=example/wikiList.csv wikilist=example/wikiList.csv

8
scrape_log.sh
View File

@@ -1,5 +1,13 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# Run userroles_from_logevents.py
# Copyright (C) 2018 Nathan TeBlunthuis
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# you specify a csv file that has a list of wikis # you specify a csv file that has a list of wikis
wikilist=example/wikiList.csv wikilist=example/wikiList.csv

8
scraper_utils.py
View File

@@ -1,3 +1,11 @@
# Functions common to both scrapers
# Copyright (C) 2018 Nathan TeBlunthuis
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# returns an iterator of wiki,url tuples # returns an iterator of wiki,url tuples
import pandas as pd import pandas as pd
from os import makedirs, path from os import makedirs, path

9
setup.py
View File

@@ -1,3 +1,12 @@
# configuration for the python package
# Copyright (C) 2018 Nathan TeBlunthuis
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#!/usr/bin/env python3 #!/usr/bin/env python3
from setuptools import setup from setuptools import setup

9
userroles_from_listusers.py
View File

@@ -1,4 +1,13 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# Scrape the Wikia userroles api
# Copyright (C) 2018 Nathan TeBlunthuis
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
import argparse import argparse
import csv import csv
import json import json

10
userroles_from_logevents.py
View File

@@ -1,4 +1,14 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# Obtain user roles data from the Wikia logevents api
# Copyright (C) 2018 Nathan TeBlunthuis
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
from os import path from os import path
import argparse import argparse
import time import time