linguini
75 lines
1.6 KiB
Python
75 lines
1.6 KiB
Python
import sqlite3
|
|
import requests
|
|
import time
|
|
|
|
DB = "paintingzz.db"
|
|
|
|
BATCH_SIZE = 50
|
|
|
|
conn = sqlite3.connect(DB)
|
|
cur = conn.cursor()
|
|
|
|
# Get all QIDs that still need labels
|
|
cur.execute("""
|
|
SELECT qid
|
|
FROM wikidata_entities
|
|
WHERE label IS NULL
|
|
""")
|
|
|
|
qids = [row[0] for row in cur.fetchall()]
|
|
|
|
print(f"{len(qids)} entities to resolve.")
|
|
|
|
for start in range(0, len(qids), BATCH_SIZE):
|
|
batch = qids[start:start + BATCH_SIZE]
|
|
|
|
print(f"Processing {start + 1} - {start + len(batch)}")
|
|
|
|
response = requests.get(
|
|
"https://www.wikidata.org/w/api.php",
|
|
params={
|
|
"action": "wbgetentities",
|
|
"ids": "|".join(batch),
|
|
"props": "labels",
|
|
"languages": "en",
|
|
"format": "json"
|
|
},
|
|
headers={
|
|
# Wikidata appreciates a descriptive User-Agent
|
|
"User-Agent": "WikidataLabelResolver/1.0 (personal script)"
|
|
}
|
|
)
|
|
|
|
response.raise_for_status()
|
|
|
|
entities = response.json()["entities"]
|
|
|
|
for qid, entity in entities.items():
|
|
labels = entity.get("labels", {})
|
|
|
|
# Prefer English, otherwise use any available label
|
|
if "en" in labels:
|
|
label = labels["en"]["value"]
|
|
elif labels:
|
|
label = next(iter(labels.values()))["value"]
|
|
else:
|
|
label = None
|
|
|
|
cur.execute(
|
|
"""
|
|
UPDATE wikidata_entities
|
|
SET label = ?
|
|
WHERE qid = ?
|
|
""",
|
|
(label, qid)
|
|
)
|
|
|
|
conn.commit()
|
|
|
|
# Be polite to Wikidata
|
|
time.sleep(6)
|
|
|
|
conn.close()
|
|
|
|
print("Done!")
|