add two small shellscripts for automation

- Added two bash scripts usable as cronjobs to automate the production
  of revisions and view data.

These commands automate the process of running code and copying material
This commit is contained in:
Benjamin Mako Hill 2020-04-01 15:15:11 -05:00
parent 04e00f363b
commit 141871eda6
2 changed files with 33 additions and 0 deletions

View File

@ -0,0 +1,17 @@
#!/bin/bash -x
TZ="UTC"
date_string=$(date +%Y%m%d)
./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/enwp-wikiproject_scraper-${date_string}.log)
revs_log="enwp-revisions-${date_string}.log"
./wikipedia/scripts/fetch_enwiki_revisions.py 2> >(tee wikipedia/logs/${rev_log})
cp wikipedia/logs/${rev_log} /var/www/covid19/wikipedia/logs/
revs_tsv="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.tsv"
cp wikipedia/data/${revs_tsv} /var/www/covid19/wikipedia
revs_json="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.json"
xz wikipedia/data/${revs_json}
cp wikipedia/data/${revs_json}.xz /var/www/covid19/wikipedia

16
cron-wikipedia_views.sh Normal file
View File

@ -0,0 +1,16 @@
#!/bin/bash -x
TZ="UTC"
date_string=$(date +%Y%m%d)
./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/enwp-wikiproject_scraper-${date_string}.log)
# get the list of files
view_log="enwp-daily_views-${date_string}.log"
./wikipedia/scripts/fetch_enwiki_daily_views.py 2> >(tee wikipedia/logs/${view_log})
cp wikipedia/logs/${view_log} /var/www/covid19/wikipedia/logs/${view_log}
cp wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.tsv /var/www/covid19/wikipedia/
# xz wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.json
cp wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.json /var/www/covid19/wikipedia/