mgaughan/covid19
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update cron scripts with new data format

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2020-04-04 15:20:34 -07:00
parent 974dc48b12
commit c97028fabb
2 changed files with 35 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
cron-wikipedia_revisions.sh
View File

@ -6,12 +6,25 @@ date_string=$(date +%Y%m%d)
revs_log="enwp-revisions-${date_string}.log"
./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/${revs_log})
./wikipedia/scripts/fetch_enwiki_revisions.py 2> >(tee -a wikipedia/logs/${revs_log})
wd_log="wd-page-crawler-${date_string}.log"
python3 ./real-time-wiki-covid-tracker/PageCrawler.py -a "./wikipedia/resources/enwp_wikiproject_covid19_articles.txt" 2> >(tee wikipedia/logs/${wd_log})
./wikipedia/scripts/fetch_revisions.py 2> >(tee -a wikipedia/logs/${revs_log})
mv wikipedia/logs/${revs_log} /var/www/covid19/wikipedia/logs/
revs_tsv="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.tsv"
mv wikipedia/data/${revs_tsv} /var/www/covid19/wikipedia
python3 ./wikipedia/scripts/copy_revisions_data.py ${date_string}
revs_json="digobs_covid19-wikipedia-enwiki_revisions-${date_string}.json"
xz wikipedia/data/${revs_json}
mv wikipedia/data/${revs_json}.xz /var/www/covid19/wikipedia
cd wikipedia/data
xz */${date_string}/*revisions*.json
find */${date_string}/*revisions*.xz | while read line; do
mkdir -p /var/www/covid9/wikipedia/$line
mv $line /var/www/covid19/wikipedia/$line
done
find */${date_string}/*revisions*.tsv | while read line; do
mkdir -p /var/www/covid19/wikipedia/$line
mv $line /var/www/covid19/wikipedia/$line
done
cd ../..

21
cron-wikipedia_views.sh
View File

@ -3,14 +3,25 @@
TZ="UTC"
date_string=${OVERRIDE_DATE_STRING:-$(date +%Y%m%d)}
view_log="enwp-daily_views-${date_string}.log"
view_log="daily_views-${date_string}.log"
./wikipedia/scripts/wikiproject_scraper.py 2> >(tee wikipedia/logs/${view_log})
wd_log="wd-page-crawler-${date_string}.log"
python3 ./real-time-wiki-covid-tracker/PageCrawler.py -a "./wikipedia/resources/enwp_wikiproject_covid19_articles.txt" 2> >(tee wikipedia/logs/${wd_log})
# get the list of files
./wikipedia/scripts/fetch_enwiki_daily_views.py -d "${date_string}" 2> >(tee -a wikipedia/logs/${view_log})
./wikipedia/scripts/fetch_daily_views.py -d "${date_string}" 2> >(tee -a wikipedia/logs/${view_log})
mv wikipedia/logs/${view_log} /var/www/covid19/wikipedia/logs/${view_log}
mv wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.tsv /var/www/covid19/wikipedia/
# xz wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.json
mv wikipedia/data/digobs_covid19-wikipedia-enwiki_dailyviews-${date_string}.json /var/www/covid19/wikipedia/
cd wikipedia/data
find */${date_string}/*dailyviews*.tsv | while read line; do
mkdir -p /var/www/covid19/wikipedia/$line
mv $line /var/www/covid19/wikipedia/$line
done
find */${date_string}/*dailyviews*.json | while read line; do
mkdir -p /var/www/covid19/wikipedia/$line
mv $line /var/www/covid19/wikipedia/$line
done
cd ../..