A few suggestions for the python script:
- using format strings (f-strings) is a nice way in python to build strings using variables. - you can read and process a file in one pass if you iterate over the open file itself instead of reading it into a variable and then looping - i had to change your strip code when i stopped using csv reader - my python linter and auto-formater hate non-indendent comments - i added a few lines to print cases where we don't get Ok responses.
This commit is contained in:
parent
c54d8ba28a
commit
49c3203d78
@ -20,7 +20,6 @@ import datetime
|
||||
#import feather
|
||||
|
||||
|
||||
|
||||
def parse_args():
|
||||
|
||||
parser = argparse.ArgumentParser(description='Call the views API repeatedly.')
|
||||
@ -51,31 +50,28 @@ def main():
|
||||
articleList = []
|
||||
#1 Load up the list of article names
|
||||
|
||||
with open(articleFile, 'r') as infileHandle:
|
||||
theInfile = csv.reader(infileHandle)
|
||||
next(theInfile) #skip header
|
||||
for currentLine in theInfile:
|
||||
articleList.append(currentLine)
|
||||
j_Out = f"{outputPath}dailyviews{queryDate}.json"
|
||||
t_Out = f"{outputPath}dailyviews{queryDate}.tsv"
|
||||
|
||||
j_Out = outputPath + "dailyviews" + queryDate + ".json"
|
||||
t_Out = outputPath + "dailyviews" + queryDate + ".tsv"
|
||||
with open(articleFile, 'r') as infile:
|
||||
next(infile) #skip header
|
||||
articleList = infile
|
||||
|
||||
j = []
|
||||
|
||||
i = 0 #iterator to deal with end of file
|
||||
|
||||
#2 Repeatedly call the API with that list of names
|
||||
|
||||
for a in articleList:
|
||||
a = a[0] #destringify
|
||||
i = i+1
|
||||
url= "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/"
|
||||
url= url + a + "/daily/" + queryDate + "/" + queryDate #for now, single date at a time
|
||||
a = a.strip("\"\n") #destringify
|
||||
url= f"https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/en.wikipedia/all-access/all-agents/{a}/daily/{queryDate}/{queryDate}"
|
||||
|
||||
response = requests.get(url)
|
||||
if response.ok:
|
||||
jd = json.loads(response.content)
|
||||
j.append(jd["items"][0])
|
||||
time.sleep(.1)
|
||||
else:
|
||||
print(f"Not ok response: {response.status_code} from {url}")
|
||||
|
||||
#3 Save results as a JSON and TSV
|
||||
|
||||
@ -89,7 +85,7 @@ def main():
|
||||
dw.writerows(j)
|
||||
|
||||
|
||||
f_Out = outputPath + "dailyviews" + queryDate + ".feather"
|
||||
# f_Out = outputPath + "dailyviews" + queryDate + ".feather"
|
||||
# read the json back in and make a feather file?
|
||||
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user