mgaughan/mw-convo-collections
1
0
Fork 0
Code

updating scripts

Browse Source
This commit is contained in:
Matthew Gaughan 2025-01-06 12:22:42 -06:00
parent 60dfe7e0a2
commit 83e668bfe5
2 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/helper_scripts/dump_collector.py
View File

@ -229,6 +229,8 @@ def batch_parallel_for_yearly():
def batch_parallel_for_single(): def batch_parallel_for_single():
urls = [] urls = []
for entry in ALL_PROJECTS: for entry in ALL_PROJECTS:
if ALL_PROJECTS.index(entry) < ALL_PROJECTS.index("skwikiquote"):
continue
if entry not in YEARLY_PROJECTS and entry not in MONTHLY_PROJECTS: if entry not in YEARLY_PROJECTS and entry not in MONTHLY_PROJECTS:
urls.append(f"{DUMP_LOC_PREFIX}{entry}/{DUMP}.{entry}.all-time.tsv.bz2") urls.append(f"{DUMP_LOC_PREFIX}{entry}/{DUMP}.{entry}.all-time.tsv.bz2")
for url in urls: for url in urls:

12
src/lib/phab_get.py
View File

@ -127,12 +127,12 @@ if __name__ == "__main__":
token = "api-wurg254ciq5uvfxlr4rszn5ynpy4" token = "api-wurg254ciq5uvfxlr4rszn5ynpy4"
api_base = 'https://phabricator.wikimedia.org/api/' api_base = 'https://phabricator.wikimedia.org/api/'
ve_ts1 = int(datetime.datetime.timestamp(datetime.datetime(2012, 1, 1, 0, 0, 0))) p_ts1 = int(datetime.datetime.timestamp(datetime.datetime(2024, 6, 10, 0, 0, 0)))
ve_ts2 = int(datetime.datetime.timestamp(datetime.datetime(2014, 12, 31, 0, 0, 0))) p_ts2 = int(datetime.datetime.timestamp(datetime.datetime(2024, 10, 10, 0, 0, 0)))
ve_data = query_task_tag("VisualEditor", ts1=ve_ts1, ts2=ve_ts2) p_data = query_task_tag("Parsoid", ts1=p_ts1, ts2=p_ts2)
for entry in ve_data: for entry in p_data:
task_id = entry['phid'] task_id = entry['phid']
print(task_id) print(task_id)
transactions = query_transactions_phid_task(task_id) transactions = query_transactions_phid_task(task_id)
@ -141,5 +141,5 @@ if __name__ == "__main__":
comments[item['id']] = item['comments'] comments[item['id']] = item['comments']
entry['task_comments'] = comments entry['task_comments'] = comments
DATA_PREFIX = "/data/users/mgaughan/mw-repo-lifecycles/phab_data/" DATA_PREFIX = "/data/users/mgaughan/mw-repo-lifecycles/phab_data/"
with open(DATA_PREFIX + "visualeditor/" + "2012_1_1_to_2014_12_31.json", "w") as outfile1: with open(DATA_PREFIX + "parsoid/" + "2024_6_10_to_2024_10_10.json", "w") as outfile1:
json.dump(ve_data, outfile1) json.dump(p_data, outfile1)