sync util scripts
This commit is contained in:
linguini
@@ -0,0 +1,238 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def parse_nt_line(line):
|
||||||
|
"""
|
||||||
|
Parse simple N-Triples:
|
||||||
|
<subject> <predicate> <object> .
|
||||||
|
"""
|
||||||
|
line = line.rstrip("\n")
|
||||||
|
|
||||||
|
if not line or line.startswith("#"):
|
||||||
|
return None
|
||||||
|
|
||||||
|
# split first two fields, keep object intact
|
||||||
|
m = re.match(r'(<.*?>)\s+(<.*?>)\s+(.*?)\s+\.$', line)
|
||||||
|
|
||||||
|
if not m:
|
||||||
|
return None
|
||||||
|
|
||||||
|
subject = m.group(1)[1:-1]
|
||||||
|
predicate = m.group(2)[1:-1]
|
||||||
|
obj = m.group(3)
|
||||||
|
|
||||||
|
# remove literal quotes
|
||||||
|
if obj.startswith('"'):
|
||||||
|
obj = obj.split('"')[1]
|
||||||
|
|
||||||
|
elif obj.startswith("<"):
|
||||||
|
obj = obj[1:-1]
|
||||||
|
|
||||||
|
return subject, predicate, obj
|
||||||
|
|
||||||
|
|
||||||
|
def sql_name(uri):
|
||||||
|
"""
|
||||||
|
Convert RDF predicate URI into safe SQLite column name.
|
||||||
|
"""
|
||||||
|
name = uri.split("/")[-1]
|
||||||
|
name = name.split("#")[-1]
|
||||||
|
|
||||||
|
name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
|
||||||
|
|
||||||
|
if not name:
|
||||||
|
name = "column"
|
||||||
|
|
||||||
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def discover_predicates(filename):
|
||||||
|
predicates = {}
|
||||||
|
|
||||||
|
print("Pass 1: discovering predicates...")
|
||||||
|
|
||||||
|
with open(filename, "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
triple = parse_nt_line(line)
|
||||||
|
|
||||||
|
if triple:
|
||||||
|
_, predicate, _ = triple
|
||||||
|
|
||||||
|
if predicate not in predicates:
|
||||||
|
predicates[predicate] = sql_name(predicate)
|
||||||
|
|
||||||
|
return predicates
|
||||||
|
|
||||||
|
|
||||||
|
def create_table(con, predicates):
|
||||||
|
cur = con.cursor()
|
||||||
|
|
||||||
|
columns = [
|
||||||
|
"subject TEXT PRIMARY KEY"
|
||||||
|
]
|
||||||
|
|
||||||
|
used = {"subject"}
|
||||||
|
|
||||||
|
for col in predicates.values():
|
||||||
|
|
||||||
|
original = col
|
||||||
|
counter = 1
|
||||||
|
|
||||||
|
while col in used:
|
||||||
|
col = f"{original}_{counter}"
|
||||||
|
counter += 1
|
||||||
|
|
||||||
|
used.add(col)
|
||||||
|
|
||||||
|
columns.append(f'"{col}" TEXT')
|
||||||
|
|
||||||
|
predicates[
|
||||||
|
next(k for k, v in predicates.items() if v == original)
|
||||||
|
] = col
|
||||||
|
|
||||||
|
sql = """
|
||||||
|
CREATE TABLE IF NOT EXISTS inspections (
|
||||||
|
%s
|
||||||
|
)
|
||||||
|
""" % ",".join(columns)
|
||||||
|
|
||||||
|
cur.execute(sql)
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def insert_row(cur, predicates, subject, values):
|
||||||
|
|
||||||
|
cols = ["subject"]
|
||||||
|
data = [subject]
|
||||||
|
|
||||||
|
for pred, value in values.items():
|
||||||
|
|
||||||
|
if pred in predicates:
|
||||||
|
cols.append(predicates[pred])
|
||||||
|
data.append(value)
|
||||||
|
|
||||||
|
placeholders = ",".join("?" for _ in data)
|
||||||
|
|
||||||
|
sql = (
|
||||||
|
"INSERT INTO inspections "
|
||||||
|
"(%s) VALUES (%s)"
|
||||||
|
% (
|
||||||
|
",".join('"%s"' % c for c in cols),
|
||||||
|
placeholders,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
cur.execute(sql, data)
|
||||||
|
|
||||||
|
|
||||||
|
def import_nt(filename, output):
|
||||||
|
|
||||||
|
predicates = discover_predicates(filename)
|
||||||
|
|
||||||
|
print(
|
||||||
|
"Found %d predicates"
|
||||||
|
% len(predicates)
|
||||||
|
)
|
||||||
|
|
||||||
|
con = sqlite3.connect(output)
|
||||||
|
|
||||||
|
create_table(con, predicates)
|
||||||
|
|
||||||
|
cur = con.cursor()
|
||||||
|
|
||||||
|
print("Pass 2: importing triples...")
|
||||||
|
|
||||||
|
current_subject = None
|
||||||
|
row = {}
|
||||||
|
|
||||||
|
count = 0
|
||||||
|
|
||||||
|
with open(filename, "r", encoding="utf-8") as f:
|
||||||
|
|
||||||
|
for line in f:
|
||||||
|
|
||||||
|
triple = parse_nt_line(line)
|
||||||
|
|
||||||
|
if not triple:
|
||||||
|
continue
|
||||||
|
|
||||||
|
subject, predicate, obj = triple
|
||||||
|
|
||||||
|
if current_subject is None:
|
||||||
|
current_subject = subject
|
||||||
|
|
||||||
|
if subject != current_subject:
|
||||||
|
|
||||||
|
insert_row(
|
||||||
|
cur,
|
||||||
|
predicates,
|
||||||
|
current_subject,
|
||||||
|
row
|
||||||
|
)
|
||||||
|
|
||||||
|
count += 1
|
||||||
|
|
||||||
|
if count % 10000 == 0:
|
||||||
|
con.commit()
|
||||||
|
print(
|
||||||
|
"Inserted",
|
||||||
|
count,
|
||||||
|
"rows"
|
||||||
|
)
|
||||||
|
|
||||||
|
current_subject = subject
|
||||||
|
row = {}
|
||||||
|
|
||||||
|
row[predicate] = obj
|
||||||
|
|
||||||
|
|
||||||
|
if current_subject is not None:
|
||||||
|
insert_row(
|
||||||
|
cur,
|
||||||
|
predicates,
|
||||||
|
current_subject,
|
||||||
|
row
|
||||||
|
)
|
||||||
|
|
||||||
|
con.commit()
|
||||||
|
|
||||||
|
print(
|
||||||
|
"Done. Inserted",
|
||||||
|
count + 1,
|
||||||
|
"rows"
|
||||||
|
)
|
||||||
|
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"-i",
|
||||||
|
"--input",
|
||||||
|
required=True
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
"-o",
|
||||||
|
"--output",
|
||||||
|
required=True
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
import_nt(
|
||||||
|
args.input,
|
||||||
|
args.output
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
echo -e " \033[33;5m __ _ _ ___ \033[0m"
|
||||||
|
echo -e " \033[33;5m \ \(_)_ __ ___( )__ / _ \__ _ _ __ __ _ __ _ ___ \033[0m"
|
||||||
|
echo -e " \033[33;5m \ \ | '_ \` _ \/ __| / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m"
|
||||||
|
echo -e " \033[33;5m /\_/ / | | | | | \__ \ / /_\\ (_| | | | (_| | (_| | __/ \033[0m"
|
||||||
|
echo -e " \033[33;5m \___/|_|_| |_| |_|___/ \____/\__,_|_| \__,_|\__, |\___| \033[0m"
|
||||||
|
echo -e " \033[33;5m |___/ \033[0m"
|
||||||
|
|
||||||
|
echo -e " \033[36;5m _ _________ ___ _ _ _ \033[0m"
|
||||||
|
echo -e " \033[36;5m | |/ |__ / __| |_ _|_ _ __| |_ __ _| | | \033[0m"
|
||||||
|
echo -e " \033[36;5m | ' < |_ \__ \ | || ' \(_-| _/ _\` | | | \033[0m"
|
||||||
|
echo -e " \033[36;5m |_|\_|___|___/ |___|_||_/__/\__\__,_|_|_| \033[0m"
|
||||||
|
echo -e " \033[36;5m \033[0m"
|
||||||
|
echo -e " \033[32;5m https://youtube.com/@jims-garage \033[0m"
|
||||||
|
echo -e " \033[32;5m \033[0m"
|
||||||
|
|
||||||
|
|
||||||
|
#############################################
|
||||||
|
# YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
|
||||||
|
#############################################
|
||||||
|
|
||||||
|
# Version of Kube-VIP to deploy
|
||||||
|
KVVERSION="v1.2.2"
|
||||||
|
|
||||||
|
# K3S Version
|
||||||
|
k3sVersion="v1.36.2+k3s1"
|
||||||
|
|
||||||
|
# Set the IP addresses of the master and work nodes
|
||||||
|
master1=192.168.1.133
|
||||||
|
worker1=192.168.1.134
|
||||||
|
worker2=192.168.1.222
|
||||||
|
worker3=192.168.1.223
|
||||||
|
|
||||||
|
# User of remote machines
|
||||||
|
user=kubemeister
|
||||||
|
|
||||||
|
# Interface used on remotes
|
||||||
|
interface=enp0s11
|
||||||
|
|
||||||
|
# Set the virtual IP address (VIP)
|
||||||
|
vip=192.168.3.50
|
||||||
|
|
||||||
|
# Array of master nodes
|
||||||
|
masters=($master1)
|
||||||
|
|
||||||
|
# Array of worker nodes
|
||||||
|
workers=($worker1 $worker2 $worker3)
|
||||||
|
|
||||||
|
# Array of all
|
||||||
|
all=($master1 $worker1 $worker2 $worker3)
|
||||||
|
|
||||||
|
# Array of all minus master
|
||||||
|
allnomaster1=($master1 $worker1 $worker2 $worker3)
|
||||||
|
|
||||||
|
#Loadbalancer IP range
|
||||||
|
lbrange=192.168.3.60-192.168.3.80
|
||||||
|
|
||||||
|
#ssh certificate name variable
|
||||||
|
certName=id_kubemeister
|
||||||
|
|
||||||
|
#ssh config file
|
||||||
|
config_file=~/.ssh/config
|
||||||
|
|
||||||
|
#############################################
|
||||||
|
# DO NOT EDIT BELOW #
|
||||||
|
#############################################
|
||||||
|
# For testing purposes - in case time is wrong due to VM snapshots
|
||||||
|
sudo timedatectl set-ntp off
|
||||||
|
sudo timedatectl set-ntp on
|
||||||
|
|
||||||
|
# Move SSH certs to ~/.ssh and change permissions
|
||||||
|
cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh
|
||||||
|
chmod 600 /home/$user/.ssh/$certName
|
||||||
|
chmod 644 /home/$user/.ssh/$certName.pub
|
||||||
|
|
||||||
|
# Install k3sup to local machine if not already present
|
||||||
|
if ! command -v k3sup version &> /dev/null
|
||||||
|
then
|
||||||
|
echo -e " \033[31;5mk3sup not found, installing\033[0m"
|
||||||
|
curl -sLS https://get.k3sup.dev | sh
|
||||||
|
sudo install k3sup-arm64 /usr/local/bin/
|
||||||
|
else
|
||||||
|
echo -e " \033[32;5mk3sup already installed\033[0m"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Install Kubectl if not already present
|
||||||
|
if ! command -v kubectl version &> /dev/null
|
||||||
|
then
|
||||||
|
echo -e " \033[31;5mKubectl not found, installing\033[0m"
|
||||||
|
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"
|
||||||
|
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||||
|
else
|
||||||
|
echo -e " \033[32;5mKubectl already installed\033[0m"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check for SSH config file, create if needed, add/change Strict Host Key Checking (don't use in production!)
|
||||||
|
|
||||||
|
if [ ! -f "$config_file" ]; then
|
||||||
|
# Create the file and add the line
|
||||||
|
echo "StrictHostKeyChecking no" > "$config_file"
|
||||||
|
# Set permissions to read and write only for the owner
|
||||||
|
chmod 600 "$config_file"
|
||||||
|
echo "File created and line added."
|
||||||
|
else
|
||||||
|
# Check if the line exists
|
||||||
|
if grep -q "^StrictHostKeyChecking" "$config_file"; then
|
||||||
|
# Check if the value is not "no"
|
||||||
|
if ! grep -q "^StrictHostKeyChecking no" "$config_file"; then
|
||||||
|
# Replace the existing line
|
||||||
|
sed -i 's/^StrictHostKeyChecking.*/StrictHostKeyChecking no/' "$config_file"
|
||||||
|
echo "Line updated."
|
||||||
|
else
|
||||||
|
echo "Line already set to 'no'."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
# Add the line to the end of the file
|
||||||
|
echo "StrictHostKeyChecking no" >> "$config_file"
|
||||||
|
echo "Line added."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
#add ssh keys for all nodes
|
||||||
|
for node in "${all[@]}"; do
|
||||||
|
ssh-copy-id $user@$node
|
||||||
|
done
|
||||||
|
|
||||||
|
# Install policycoreutils for each node
|
||||||
|
for newnode in "${all[@]}"; do
|
||||||
|
ssh $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
|
||||||
|
NEEDRESTART_MODE=a apt-get install policycoreutils -y
|
||||||
|
exit
|
||||||
|
EOF
|
||||||
|
echo -e " \033[32;5mPolicyCoreUtils installed!\033[0m"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Step 1: Bootstrap First k3s Node
|
||||||
|
mkdir ~/.kube
|
||||||
|
k3sup install \
|
||||||
|
--ip $master1 \
|
||||||
|
--user $user \
|
||||||
|
--tls-san $vip \
|
||||||
|
--cluster \
|
||||||
|
--k3s-version $k3sVersion \
|
||||||
|
--k3s-extra-args "--disable traefik --disable servicelb --flannel-iface=$interface --node-ip=$master1 --node-taint node-role.kubernetes.io/master=true:NoSchedule" \
|
||||||
|
--merge \
|
||||||
|
--sudo \
|
||||||
|
--local-path $HOME/.kube/config \
|
||||||
|
--ssh-key $HOME/.ssh/$certName \
|
||||||
|
--context k3s-ha
|
||||||
|
echo -e " \033[32;5mFirst Node bootstrapped successfully!\033[0m"
|
||||||
|
|
||||||
|
# Step 2: Install Kube-VIP for HA
|
||||||
|
kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
|
||||||
|
|
||||||
|
# Step 3: Download kube-vip
|
||||||
|
curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/kube-vip
|
||||||
|
cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g' > $HOME/kube-vip.yaml
|
||||||
|
|
||||||
|
# Step 4: Copy kube-vip.yaml to master1
|
||||||
|
scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$master1:~/kube-vip.yaml
|
||||||
|
|
||||||
|
|
||||||
|
# Step 5: Connect to Master1 and move kube-vip.yaml
|
||||||
|
ssh $user@$master1 -i ~/.ssh/$certName <<- EOF
|
||||||
|
sudo mkdir -p /var/lib/rancher/k3s/server/manifests
|
||||||
|
sudo mv kube-vip.yaml /var/lib/rancher/k3s/server/manifests/kube-vip.yaml
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# Step 6: Add new master nodes (servers) & workers
|
||||||
|
for newnode in "${masters[@]}"; do
|
||||||
|
k3sup join \
|
||||||
|
--ip $newnode \
|
||||||
|
--user $user \
|
||||||
|
--sudo \
|
||||||
|
--k3s-version $k3sVersion \
|
||||||
|
--server \
|
||||||
|
--server-ip $master1 \
|
||||||
|
--ssh-key $HOME/.ssh/$certName \
|
||||||
|
--k3s-extra-args "--disable traefik --disable servicelb --flannel-iface=$interface --node-ip=$newnode --node-taint node-role.kubernetes.io/master=true:NoSchedule" \
|
||||||
|
--server-user $user
|
||||||
|
echo -e " \033[32;5mMaster node joined successfully!\033[0m"
|
||||||
|
done
|
||||||
|
|
||||||
|
# add workers
|
||||||
|
for newagent in "${workers[@]}"; do
|
||||||
|
k3sup join \
|
||||||
|
--ip $newagent \
|
||||||
|
--user $user \
|
||||||
|
--sudo \
|
||||||
|
--k3s-version $k3sVersion \
|
||||||
|
--server-ip $master1 \
|
||||||
|
--ssh-key $HOME/.ssh/$certName \
|
||||||
|
--k3s-extra-args "--node-label \"longhorn=true\" --node-label \"worker=true\""
|
||||||
|
echo -e " \033[32;5mAgent node joined successfully!\033[0m"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Step 7: Install kube-vip as network LoadBalancer - Install the kube-vip Cloud Provider
|
||||||
|
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
|
||||||
|
|
||||||
|
# Step 8: Install Metallb
|
||||||
|
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
|
||||||
|
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
|
||||||
|
# Download ipAddressPool and configure using lbrange above
|
||||||
|
curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/ipAddressPool
|
||||||
|
cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml
|
||||||
|
kubectl apply -f $HOME/ipAddressPool.yaml
|
||||||
|
|
||||||
|
# Step 9: Test with Nginx
|
||||||
|
kubectl apply -f https://raw.githubusercontent.com/inlets/inlets-operator/master/contrib/nginx-sample-deployment.yaml -n default
|
||||||
|
kubectl expose deployment nginx-1 --port=80 --type=LoadBalancer -n default
|
||||||
|
|
||||||
|
echo -e " \033[32;5mWaiting for K3S to sync and LoadBalancer to come online\033[0m"
|
||||||
|
|
||||||
|
while [[ $(kubectl get pods -l app=nginx -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
# Step 10: Deploy IP Pools and l2Advertisement
|
||||||
|
kubectl wait --namespace metallb-system \
|
||||||
|
--for=condition=ready pod \
|
||||||
|
--selector=component=controller \
|
||||||
|
--timeout=120s
|
||||||
|
kubectl apply -f ipAddressPool.yaml
|
||||||
|
kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/l2Advertisement.yaml
|
||||||
|
|
||||||
|
kubectl get nodes
|
||||||
|
kubectl get svc
|
||||||
|
kubectl get pods --all-namespaces -o wide
|
||||||
|
|
||||||
|
echo -e " \033[32;5mHappy Kubing! Access Nginx at EXTERNAL-IP above\033[0m"
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
db = "paintingzz.db"
|
||||||
|
|
||||||
|
conn = sqlite3.connect(db)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
table = "inspections"
|
||||||
|
|
||||||
|
# Find completely NULL columns
|
||||||
|
cur.execute(f"PRAGMA table_info({table})")
|
||||||
|
columns = [row[1] for row in cur.fetchall()]
|
||||||
|
|
||||||
|
empty_columns = []
|
||||||
|
|
||||||
|
for col in columns:
|
||||||
|
print('cucurucu')
|
||||||
|
cur.execute(
|
||||||
|
f'SELECT COUNT(*) FROM "{table}" WHERE "{col}" IS NOT NULL'
|
||||||
|
)
|
||||||
|
count = cur.fetchone()[0]
|
||||||
|
|
||||||
|
if count <100 :
|
||||||
|
print('list b growing')
|
||||||
|
empty_columns.append(col)
|
||||||
|
print(empty_columns)
|
||||||
|
|
||||||
|
print("Removing:", empty_columns)
|
||||||
|
|
||||||
|
# SQLite >= 3.35 supports DROP COLUMN
|
||||||
|
for col in empty_columns:
|
||||||
|
cur.execute(
|
||||||
|
f'ALTER TABLE "{table}" DROP COLUMN "{col}"'
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import sqlite3
|
||||||
|
import requests
|
||||||
|
import time
|
||||||
|
|
||||||
|
DB = "paintingzz.db"
|
||||||
|
|
||||||
|
BATCH_SIZE = 50
|
||||||
|
|
||||||
|
conn = sqlite3.connect(DB)
|
||||||
|
cur = conn.cursor()
|
||||||
|
|
||||||
|
# Get all QIDs that still need labels
|
||||||
|
cur.execute("""
|
||||||
|
SELECT qid
|
||||||
|
FROM wikidata_entities
|
||||||
|
WHERE label IS NULL
|
||||||
|
""")
|
||||||
|
|
||||||
|
qids = [row[0] for row in cur.fetchall()]
|
||||||
|
|
||||||
|
print(f"{len(qids)} entities to resolve.")
|
||||||
|
|
||||||
|
for start in range(0, len(qids), BATCH_SIZE):
|
||||||
|
batch = qids[start:start + BATCH_SIZE]
|
||||||
|
|
||||||
|
print(f"Processing {start + 1} - {start + len(batch)}")
|
||||||
|
|
||||||
|
response = requests.get(
|
||||||
|
"https://www.wikidata.org/w/api.php",
|
||||||
|
params={
|
||||||
|
"action": "wbgetentities",
|
||||||
|
"ids": "|".join(batch),
|
||||||
|
"props": "labels",
|
||||||
|
"languages": "en",
|
||||||
|
"format": "json"
|
||||||
|
},
|
||||||
|
headers={
|
||||||
|
# Wikidata appreciates a descriptive User-Agent
|
||||||
|
"User-Agent": "WikidataLabelResolver/1.0 (personal script)"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
entities = response.json()["entities"]
|
||||||
|
|
||||||
|
for qid, entity in entities.items():
|
||||||
|
labels = entity.get("labels", {})
|
||||||
|
|
||||||
|
# Prefer English, otherwise use any available label
|
||||||
|
if "en" in labels:
|
||||||
|
label = labels["en"]["value"]
|
||||||
|
elif labels:
|
||||||
|
label = next(iter(labels.values()))["value"]
|
||||||
|
else:
|
||||||
|
label = None
|
||||||
|
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
UPDATE wikidata_entities
|
||||||
|
SET label = ?
|
||||||
|
WHERE qid = ?
|
||||||
|
""",
|
||||||
|
(label, qid)
|
||||||
|
)
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
|
||||||
|
# Be polite to Wikidata
|
||||||
|
time.sleep(6)
|
||||||
|
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
print("Done!")
|
||||||
Reference in New Issue
Block a user