80 lines
2.3 KiB
Python
80 lines
2.3 KiB
Python
|
from wikidata_api_calls import run_sparql_query
|
||
|
|
from itertools import chain, islice
|
||
|
|
import csv
|
||
|
|
from json import JSONDecodeError
|
||
|
|
|
||
|
|
class LabelData:
|
||
|
|
__slots__ = ['entityid','label','langcode','is_alt']
|
||
|
|
|
||
|
|
def __init__(self, wd_res, entityid, is_alt):
|
||
|
|
obj = wd_res.get('label',None)
|
||
|
|
self.label = obj.get('value',None)
|
||
|
|
self.langcode = obj.get('xml:lang',None)
|
||
|
|
self.entityid = entityid
|
||
|
|
self.is_alt = is_alt
|
||
|
|
|
||
|
|
def to_list(self):
|
||
|
|
return [self.entityid,
|
||
|
|
self.label,
|
||
|
|
self.langcode,
|
||
|
|
self.is_alt]
|
||
|
|
|
||
|
|
|
||
|
|
def GetAllLabels(in_csv, outfile, topN):
|
||
|
|
|
||
|
|
def load_entity_ids(in_csv, topN=5):
|
||
|
|
with open(in_csv,'r',newline='') as infile:
|
||
|
|
reader = csv.DictReader(infile)
|
||
|
|
for row in reader:
|
||
|
|
if int(row['search_position']) < topN:
|
||
|
|
yield row["entityid"]
|
||
|
|
|
||
|
|
ids = set(load_entity_ids(in_csv, topN))
|
||
|
|
|
||
|
|
labeldata = chain(* map(GetEntityLabels, ids))
|
||
|
|
|
||
|
|
with open(outfile, 'w', newline='') as of:
|
||
|
|
writer = csv.writer(of)
|
||
|
|
writer.writerow(LabelData.__slots__)
|
||
|
|
writer.writerows(map(LabelData.to_list,labeldata))
|
||
|
|
|
||
|
|
|
||
|
|
def GetEntityLabels(entityid):
|
||
|
|
|
||
|
|
def run_query_and_parse(query, entityid, is_alt):
|
||
|
|
results = run_sparql_query(query % entityid)
|
||
|
|
try:
|
||
|
|
jobj = results.json()
|
||
|
|
res = jobj.get('results',None)
|
||
|
|
if res is not None:
|
||
|
|
res = res.get('bindings',None)
|
||
|
|
if res is None:
|
||
|
|
raise requests.APIError(f"got invalid response from wikidata for {query % entityid}")
|
||
|
|
for info in res:
|
||
|
|
yield LabelData(info, entityid, is_alt)
|
||
|
|
|
||
|
|
except JSONDecodeError as e:
|
||
|
|
print(e)
|
||
|
|
print(query % entityid)
|
||
|
|
|
||
|
|
|
||
|
|
label_base_query = """
|
||
|
|
SELECT DISTINCT ?label WHERE {
|
||
|
|
wd:%s rdfs:label ?label;
|
||
|
|
}"""
|
||
|
|
|
||
|
|
altLabel_base_query = """
|
||
|
|
SELECT DISTINCT ?label WHERE {
|
||
|
|
wd:%s skos:altLabel ?label;
|
||
|
|
}"""
|
||
|
|
|
||
|
|
label_results = run_query_and_parse(label_base_query, entityid, is_alt=False)
|
||
|
|
|
||
|
|
altLabel_results = run_query_and_parse(altLabel_base_query, entityid, is_alt=True)
|
||
|
|
|
||
|
|
return chain(label_results, altLabel_results)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
GetAllLabels("../data/output/wikidata_search_results.csv","../data/output/wikidata_entity_labels.csv", topN=20)
|