mgaughan/covid19
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1 from CommunityDataScienceCollective/kaylea/master

Browse Source 
Some suggested changes.
This commit is contained in:
Kaylea Champion 2020-03-28 14:15:53 -07:00 committed by GitHub
commit dd7d968bb6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 15 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
data/dailyviews2020032600.json → wikipedia_views/data/dailyviews2020032600.json
View File

0
data/dailyviews2020032600.tsv → wikipedia_views/data/dailyviews2020032600.tsv
View File

0
resources/articles.txt → wikipedia_views/resources/articles.txt
View File

34
bin/fetch_daily_views.py → wikipedia_views/scripts/fetch_daily_views.py
View File

@ -20,7 +20,6 @@ import datetime
#import feather #import feather
def parse_args(): def parse_args():
parser = argparse.ArgumentParser(description='Call the views API repeatedly.') parser = argparse.ArgumentParser(description='Call the views API repeatedly.')
@ -49,35 +48,32 @@ def main():
articleList = [] articleList = []
#1 Load up the list of article names #1 Load up the list of article names
with open(articleFile, 'r') as infileHandle: j_Out = f"{outputPath}dailyviews{queryDate}.json"
theInfile = csv.reader(infileHandle) t_Out = f"{outputPath}dailyviews{queryDate}.tsv"
next(theInfile) #skip header
for currentLine in theInfile:
articleList.append(currentLine)
j_Out = outputPath + "dailyviews" + queryDate + ".json" with open(articleFile, 'r') as infile:
t_Out = outputPath + "dailyviews" + queryDate + ".tsv" next(infile) #skip header
articleList = list(infile)
j = [] j = []
i = 0 #iterator to deal with end of file #2 Repeatedly call the API with that list of names
#2 Repeatedly call the API with that list of names
for a in articleList: for a in articleList:
a = a[0] #destringify a = a.strip("\"\n") #destringify
i = i+1 url= f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/{a}/daily/{queryDate}/{queryDate}"
url= "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/"
url= url + a + "/daily/" + queryDate + "/" + queryDate #for now, single date at a time
response = requests.get(url) response = requests.get(url)
if response.ok: if response.ok:
jd = json.loads(response.content) jd = json.loads(response.content)
j.append(jd["items"][0]) j.append(jd["items"][0])
time.sleep(.1) time.sleep(.1)
else:
print(f"Not ok response: {response.status_code} from {url}")
#3 Save results as a JSON and TSV #3 Save results as a JSON and TSV
#all data in j now, make json file #all data in j now, make json file
with open(j_Out, 'w') as j_outfile: with open(j_Out, 'w') as j_outfile:
@ -89,8 +85,8 @@ def main():
dw.writerows(j) dw.writerows(j)
f_Out = outputPath + "dailyviews" + queryDate + ".feather" # f_Out = outputPath + "dailyviews" + queryDate + ".feather"
#read the json back in and make a feather file? # read the json back in and make a feather file?
if __name__ == "__main__": if __name__ == "__main__":