add two small shellscripts for automation

- Added two bash scripts usable as cronjobs to automate the production
  of revisions and view data.

These commands automate the process of running code and copying material
This commit is contained in:
2020-04-01 15:15:11 -05:00
parent 04e00f363b
commit 141871eda6
2 changed files with 33 additions and 0 deletions

View File

@@ -0,0 +1,17 @@
#!/bin/bash -x
TZ="UTC"
date_string=$(date +%Y%m%d)
./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/enwp-wikiproject_scraper-${date_string}.log)
revs_log="enwp-revisions-${date_string}.log"
./wikipedia/scripts/fetch_enwiki_revisions.py 2> >(tee wikipedia/logs/${rev_log})
cp wikipedia/logs/${rev_log} /var/www/covid19/wikipedia/logs/
revs_tsv="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.tsv"
cp wikipedia/data/${revs_tsv} /var/www/covid19/wikipedia
revs_json="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.json"
xz wikipedia/data/${revs_json}
cp wikipedia/data/${revs_json}.xz /var/www/covid19/wikipedia