sync util scripts

utils/replace-urls.py

@@ -0,0 +1,74 @@
+import sqlite3
+import requests
+import time
+
+DB = "paintingzz.db"
+
+BATCH_SIZE = 50
+
+conn = sqlite3.connect(DB)
+cur = conn.cursor()
+
+# Get all QIDs that still need labels
+cur.execute("""
+    SELECT qid
+    FROM wikidata_entities
+    WHERE label IS NULL
+""")
+
+qids = [row[0] for row in cur.fetchall()]
+
+print(f"{len(qids)} entities to resolve.")
+
+for start in range(0, len(qids), BATCH_SIZE):
+    batch = qids[start:start + BATCH_SIZE]
+
+    print(f"Processing {start + 1} - {start + len(batch)}")
+
+    response = requests.get(
+        "https://www.wikidata.org/w/api.php",
+        params={
+            "action": "wbgetentities",
+            "ids": "|".join(batch),
+            "props": "labels",
+            "languages": "en",
+            "format": "json"
+        },
+        headers={
+            # Wikidata appreciates a descriptive User-Agent
+            "User-Agent": "WikidataLabelResolver/1.0 (personal script)"
+        }
+    )
+
+    response.raise_for_status()
+
+    entities = response.json()["entities"]
+
+    for qid, entity in entities.items():
+        labels = entity.get("labels", {})
+
+        # Prefer English, otherwise use any available label
+        if "en" in labels:
+            label = labels["en"]["value"]
+        elif labels:
+            label = next(iter(labels.values()))["value"]
+        else:
+            label = None
+
+        cur.execute(
+            """
+            UPDATE wikidata_entities
+            SET label = ?
+            WHERE qid = ?
+            """,
+            (label, qid)
+        )
+
+    conn.commit()
+
+    # Be polite to Wikidata
+    time.sleep(6)
+
+conn.close()
+
+print("Done!")