From 21c7fde12d5020f0c13dcd78ea709398230a9f83 Mon Sep 17 00:00:00 2001 From: linguini Date: Fri, 31 Jul 2026 01:41:46 +0200 Subject: [PATCH] sync util scripts --- utils/convert-rdf-to-sql.py | 238 +++++++++++++++++++ utils/install_k3s_on_vm_cluster.sh | 230 ++++++++++++++++++ utils/remove_null_columns_under_threshold.py | 38 +++ utils/replace-urls.py | 74 ++++++ 4 files changed, 580 insertions(+) create mode 100644 utils/convert-rdf-to-sql.py create mode 100644 utils/install_k3s_on_vm_cluster.sh create mode 100644 utils/remove_null_columns_under_threshold.py create mode 100644 utils/replace-urls.py diff --git a/utils/convert-rdf-to-sql.py b/utils/convert-rdf-to-sql.py new file mode 100644 index 0000000..dd77772 --- /dev/null +++ b/utils/convert-rdf-to-sql.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sqlite3 +import re + + +def parse_nt_line(line): + """ + Parse simple N-Triples: + . + """ + line = line.rstrip("\n") + + if not line or line.startswith("#"): + return None + + # split first two fields, keep object intact + m = re.match(r'(<.*?>)\s+(<.*?>)\s+(.*?)\s+\.$', line) + + if not m: + return None + + subject = m.group(1)[1:-1] + predicate = m.group(2)[1:-1] + obj = m.group(3) + + # remove literal quotes + if obj.startswith('"'): + obj = obj.split('"')[1] + + elif obj.startswith("<"): + obj = obj[1:-1] + + return subject, predicate, obj + + +def sql_name(uri): + """ + Convert RDF predicate URI into safe SQLite column name. + """ + name = uri.split("/")[-1] + name = name.split("#")[-1] + + name = re.sub(r'[^a-zA-Z0-9_]', '_', name) + + if not name: + name = "column" + + return name + + +def discover_predicates(filename): + predicates = {} + + print("Pass 1: discovering predicates...") + + with open(filename, "r", encoding="utf-8") as f: + for line in f: + triple = parse_nt_line(line) + + if triple: + _, predicate, _ = triple + + if predicate not in predicates: + predicates[predicate] = sql_name(predicate) + + return predicates + + +def create_table(con, predicates): + cur = con.cursor() + + columns = [ + "subject TEXT PRIMARY KEY" + ] + + used = {"subject"} + + for col in predicates.values(): + + original = col + counter = 1 + + while col in used: + col = f"{original}_{counter}" + counter += 1 + + used.add(col) + + columns.append(f'"{col}" TEXT') + + predicates[ + next(k for k, v in predicates.items() if v == original) + ] = col + + sql = """ + CREATE TABLE IF NOT EXISTS inspections ( + %s + ) + """ % ",".join(columns) + + cur.execute(sql) + con.commit() + + +def insert_row(cur, predicates, subject, values): + + cols = ["subject"] + data = [subject] + + for pred, value in values.items(): + + if pred in predicates: + cols.append(predicates[pred]) + data.append(value) + + placeholders = ",".join("?" for _ in data) + + sql = ( + "INSERT INTO inspections " + "(%s) VALUES (%s)" + % ( + ",".join('"%s"' % c for c in cols), + placeholders, + ) + ) + + cur.execute(sql, data) + + +def import_nt(filename, output): + + predicates = discover_predicates(filename) + + print( + "Found %d predicates" + % len(predicates) + ) + + con = sqlite3.connect(output) + + create_table(con, predicates) + + cur = con.cursor() + + print("Pass 2: importing triples...") + + current_subject = None + row = {} + + count = 0 + + with open(filename, "r", encoding="utf-8") as f: + + for line in f: + + triple = parse_nt_line(line) + + if not triple: + continue + + subject, predicate, obj = triple + + if current_subject is None: + current_subject = subject + + if subject != current_subject: + + insert_row( + cur, + predicates, + current_subject, + row + ) + + count += 1 + + if count % 10000 == 0: + con.commit() + print( + "Inserted", + count, + "rows" + ) + + current_subject = subject + row = {} + + row[predicate] = obj + + + if current_subject is not None: + insert_row( + cur, + predicates, + current_subject, + row + ) + + con.commit() + + print( + "Done. Inserted", + count + 1, + "rows" + ) + + con.close() + + +def main(): + + parser = argparse.ArgumentParser() + + parser.add_argument( + "-i", + "--input", + required=True + ) + + parser.add_argument( + "-o", + "--output", + required=True + ) + + args = parser.parse_args() + + import_nt( + args.input, + args.output + ) + + +if __name__ == "__main__": + main() diff --git a/utils/install_k3s_on_vm_cluster.sh b/utils/install_k3s_on_vm_cluster.sh new file mode 100644 index 0000000..d1a4880 --- /dev/null +++ b/utils/install_k3s_on_vm_cluster.sh @@ -0,0 +1,230 @@ +#!/bin/bash +echo -e " \033[33;5m __ _ _ ___ \033[0m" +echo -e " \033[33;5m \ \(_)_ __ ___( )__ / _ \__ _ _ __ __ _ __ _ ___ \033[0m" +echo -e " \033[33;5m \ \ | '_ \` _ \/ __| / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m" +echo -e " \033[33;5m /\_/ / | | | | | \__ \ / /_\\ (_| | | | (_| | (_| | __/ \033[0m" +echo -e " \033[33;5m \___/|_|_| |_| |_|___/ \____/\__,_|_| \__,_|\__, |\___| \033[0m" +echo -e " \033[33;5m |___/ \033[0m" + +echo -e " \033[36;5m _ _________ ___ _ _ _ \033[0m" +echo -e " \033[36;5m | |/ |__ / __| |_ _|_ _ __| |_ __ _| | | \033[0m" +echo -e " \033[36;5m | ' < |_ \__ \ | || ' \(_-| _/ _\` | | | \033[0m" +echo -e " \033[36;5m |_|\_|___|___/ |___|_||_/__/\__\__,_|_|_| \033[0m" +echo -e " \033[36;5m \033[0m" +echo -e " \033[32;5m https://youtube.com/@jims-garage \033[0m" +echo -e " \033[32;5m \033[0m" + + +############################################# +# YOU SHOULD ONLY NEED TO EDIT THIS SECTION # +############################################# + +# Version of Kube-VIP to deploy +KVVERSION="v1.2.2" + +# K3S Version +k3sVersion="v1.36.2+k3s1" + +# Set the IP addresses of the master and work nodes +master1=192.168.1.133 +worker1=192.168.1.134 +worker2=192.168.1.222 +worker3=192.168.1.223 + +# User of remote machines +user=kubemeister + +# Interface used on remotes +interface=enp0s11 + +# Set the virtual IP address (VIP) +vip=192.168.3.50 + +# Array of master nodes +masters=($master1) + +# Array of worker nodes +workers=($worker1 $worker2 $worker3) + +# Array of all +all=($master1 $worker1 $worker2 $worker3) + +# Array of all minus master +allnomaster1=($master1 $worker1 $worker2 $worker3) + +#Loadbalancer IP range +lbrange=192.168.3.60-192.168.3.80 + +#ssh certificate name variable +certName=id_kubemeister + +#ssh config file +config_file=~/.ssh/config + +############################################# +# DO NOT EDIT BELOW # +############################################# +# For testing purposes - in case time is wrong due to VM snapshots +sudo timedatectl set-ntp off +sudo timedatectl set-ntp on + +# Move SSH certs to ~/.ssh and change permissions +cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh +chmod 600 /home/$user/.ssh/$certName +chmod 644 /home/$user/.ssh/$certName.pub + +# Install k3sup to local machine if not already present +if ! command -v k3sup version &> /dev/null +then + echo -e " \033[31;5mk3sup not found, installing\033[0m" + curl -sLS https://get.k3sup.dev | sh + sudo install k3sup-arm64 /usr/local/bin/ +else + echo -e " \033[32;5mk3sup already installed\033[0m" +fi + +# Install Kubectl if not already present +if ! command -v kubectl version &> /dev/null +then + echo -e " \033[31;5mKubectl not found, installing\033[0m" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +else + echo -e " \033[32;5mKubectl already installed\033[0m" +fi + +# Check for SSH config file, create if needed, add/change Strict Host Key Checking (don't use in production!) + +if [ ! -f "$config_file" ]; then + # Create the file and add the line + echo "StrictHostKeyChecking no" > "$config_file" + # Set permissions to read and write only for the owner + chmod 600 "$config_file" + echo "File created and line added." +else + # Check if the line exists + if grep -q "^StrictHostKeyChecking" "$config_file"; then + # Check if the value is not "no" + if ! grep -q "^StrictHostKeyChecking no" "$config_file"; then + # Replace the existing line + sed -i 's/^StrictHostKeyChecking.*/StrictHostKeyChecking no/' "$config_file" + echo "Line updated." + else + echo "Line already set to 'no'." + fi + else + # Add the line to the end of the file + echo "StrictHostKeyChecking no" >> "$config_file" + echo "Line added." + fi +fi + +#add ssh keys for all nodes +for node in "${all[@]}"; do + ssh-copy-id $user@$node +done + +# Install policycoreutils for each node +for newnode in "${all[@]}"; do + ssh $user@$newnode -i ~/.ssh/$certName sudo su < $HOME/kube-vip.yaml + +# Step 4: Copy kube-vip.yaml to master1 +scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$master1:~/kube-vip.yaml + + +# Step 5: Connect to Master1 and move kube-vip.yaml +ssh $user@$master1 -i ~/.ssh/$certName <<- EOF + sudo mkdir -p /var/lib/rancher/k3s/server/manifests + sudo mv kube-vip.yaml /var/lib/rancher/k3s/server/manifests/kube-vip.yaml +EOF + +# Step 6: Add new master nodes (servers) & workers +for newnode in "${masters[@]}"; do + k3sup join \ + --ip $newnode \ + --user $user \ + --sudo \ + --k3s-version $k3sVersion \ + --server \ + --server-ip $master1 \ + --ssh-key $HOME/.ssh/$certName \ + --k3s-extra-args "--disable traefik --disable servicelb --flannel-iface=$interface --node-ip=$newnode --node-taint node-role.kubernetes.io/master=true:NoSchedule" \ + --server-user $user + echo -e " \033[32;5mMaster node joined successfully!\033[0m" +done + +# add workers +for newagent in "${workers[@]}"; do + k3sup join \ + --ip $newagent \ + --user $user \ + --sudo \ + --k3s-version $k3sVersion \ + --server-ip $master1 \ + --ssh-key $HOME/.ssh/$certName \ + --k3s-extra-args "--node-label \"longhorn=true\" --node-label \"worker=true\"" + echo -e " \033[32;5mAgent node joined successfully!\033[0m" +done + +# Step 7: Install kube-vip as network LoadBalancer - Install the kube-vip Cloud Provider +kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + +# Step 8: Install Metallb +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml +# Download ipAddressPool and configure using lbrange above +curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/ipAddressPool +cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml +kubectl apply -f $HOME/ipAddressPool.yaml + +# Step 9: Test with Nginx +kubectl apply -f https://raw.githubusercontent.com/inlets/inlets-operator/master/contrib/nginx-sample-deployment.yaml -n default +kubectl expose deployment nginx-1 --port=80 --type=LoadBalancer -n default + +echo -e " \033[32;5mWaiting for K3S to sync and LoadBalancer to come online\033[0m" + +while [[ $(kubectl get pods -l app=nginx -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do + sleep 1 +done + +# Step 10: Deploy IP Pools and l2Advertisement +kubectl wait --namespace metallb-system \ + --for=condition=ready pod \ + --selector=component=controller \ + --timeout=120s +kubectl apply -f ipAddressPool.yaml +kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/l2Advertisement.yaml + +kubectl get nodes +kubectl get svc +kubectl get pods --all-namespaces -o wide + +echo -e " \033[32;5mHappy Kubing! Access Nginx at EXTERNAL-IP above\033[0m" diff --git a/utils/remove_null_columns_under_threshold.py b/utils/remove_null_columns_under_threshold.py new file mode 100644 index 0000000..577852b --- /dev/null +++ b/utils/remove_null_columns_under_threshold.py @@ -0,0 +1,38 @@ + +import sqlite3 + +db = "paintingzz.db" + +conn = sqlite3.connect(db) +cur = conn.cursor() + +table = "inspections" + +# Find completely NULL columns +cur.execute(f"PRAGMA table_info({table})") +columns = [row[1] for row in cur.fetchall()] + +empty_columns = [] + +for col in columns: + print('cucurucu') + cur.execute( + f'SELECT COUNT(*) FROM "{table}" WHERE "{col}" IS NOT NULL' + ) + count = cur.fetchone()[0] + + if count <100 : + print('list b growing') + empty_columns.append(col) + print(empty_columns) + +print("Removing:", empty_columns) + +# SQLite >= 3.35 supports DROP COLUMN +for col in empty_columns: + cur.execute( + f'ALTER TABLE "{table}" DROP COLUMN "{col}"' + ) + +conn.commit() +conn.close() diff --git a/utils/replace-urls.py b/utils/replace-urls.py new file mode 100644 index 0000000..e5512d8 --- /dev/null +++ b/utils/replace-urls.py @@ -0,0 +1,74 @@ +import sqlite3 +import requests +import time + +DB = "paintingzz.db" + +BATCH_SIZE = 50 + +conn = sqlite3.connect(DB) +cur = conn.cursor() + +# Get all QIDs that still need labels +cur.execute(""" + SELECT qid + FROM wikidata_entities + WHERE label IS NULL +""") + +qids = [row[0] for row in cur.fetchall()] + +print(f"{len(qids)} entities to resolve.") + +for start in range(0, len(qids), BATCH_SIZE): + batch = qids[start:start + BATCH_SIZE] + + print(f"Processing {start + 1} - {start + len(batch)}") + + response = requests.get( + "https://www.wikidata.org/w/api.php", + params={ + "action": "wbgetentities", + "ids": "|".join(batch), + "props": "labels", + "languages": "en", + "format": "json" + }, + headers={ + # Wikidata appreciates a descriptive User-Agent + "User-Agent": "WikidataLabelResolver/1.0 (personal script)" + } + ) + + response.raise_for_status() + + entities = response.json()["entities"] + + for qid, entity in entities.items(): + labels = entity.get("labels", {}) + + # Prefer English, otherwise use any available label + if "en" in labels: + label = labels["en"]["value"] + elif labels: + label = next(iter(labels.values()))["value"] + else: + label = None + + cur.execute( + """ + UPDATE wikidata_entities + SET label = ? + WHERE qid = ? + """, + (label, qid) + ) + + conn.commit() + + # Be polite to Wikidata + time.sleep(6) + +conn.close() + +print("Done!")