sync util scripts

utils/convert-rdf-to-sql.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import sqlite3
+import re
+
+
+def parse_nt_line(line):
+    """
+    Parse simple N-Triples:
+    <subject> <predicate> <object> .
+    """
+    line = line.rstrip("\n")
+
+    if not line or line.startswith("#"):
+        return None
+
+    # split first two fields, keep object intact
+    m = re.match(r'(<.*?>)\s+(<.*?>)\s+(.*?)\s+\.$', line)
+
+    if not m:
+        return None
+
+    subject = m.group(1)[1:-1]
+    predicate = m.group(2)[1:-1]
+    obj = m.group(3)
+
+    # remove literal quotes
+    if obj.startswith('"'):
+        obj = obj.split('"')[1]
+
+    elif obj.startswith("<"):
+        obj = obj[1:-1]
+
+    return subject, predicate, obj
+
+
+def sql_name(uri):
+    """
+    Convert RDF predicate URI into safe SQLite column name.
+    """
+    name = uri.split("/")[-1]
+    name = name.split("#")[-1]
+
+    name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
+
+    if not name:
+        name = "column"
+
+    return name
+
+
+def discover_predicates(filename):
+    predicates = {}
+
+    print("Pass 1: discovering predicates...")
+
+    with open(filename, "r", encoding="utf-8") as f:
+        for line in f:
+            triple = parse_nt_line(line)
+
+            if triple:
+                _, predicate, _ = triple
+
+                if predicate not in predicates:
+                    predicates[predicate] = sql_name(predicate)
+
+    return predicates
+
+
+def create_table(con, predicates):
+    cur = con.cursor()
+
+    columns = [
+        "subject TEXT PRIMARY KEY"
+    ]
+
+    used = {"subject"}
+
+    for col in predicates.values():
+
+        original = col
+        counter = 1
+
+        while col in used:
+            col = f"{original}_{counter}"
+            counter += 1
+
+        used.add(col)
+
+        columns.append(f'"{col}" TEXT')
+
+        predicates[
+            next(k for k, v in predicates.items() if v == original)
+        ] = col
+
+    sql = """
+    CREATE TABLE IF NOT EXISTS inspections (
+        %s
+    )
+    """ % ",".join(columns)
+
+    cur.execute(sql)
+    con.commit()
+
+
+def insert_row(cur, predicates, subject, values):
+
+    cols = ["subject"]
+    data = [subject]
+
+    for pred, value in values.items():
+
+        if pred in predicates:
+            cols.append(predicates[pred])
+            data.append(value)
+
+    placeholders = ",".join("?" for _ in data)
+
+    sql = (
+        "INSERT INTO inspections "
+        "(%s) VALUES (%s)"
+        % (
+            ",".join('"%s"' % c for c in cols),
+            placeholders,
+        )
+    )
+
+    cur.execute(sql, data)
+
+
+def import_nt(filename, output):
+
+    predicates = discover_predicates(filename)
+
+    print(
+        "Found %d predicates"
+        % len(predicates)
+    )
+
+    con = sqlite3.connect(output)
+
+    create_table(con, predicates)
+
+    cur = con.cursor()
+
+    print("Pass 2: importing triples...")
+
+    current_subject = None
+    row = {}
+
+    count = 0
+
+    with open(filename, "r", encoding="utf-8") as f:
+
+        for line in f:
+
+            triple = parse_nt_line(line)
+
+            if not triple:
+                continue
+
+            subject, predicate, obj = triple
+
+            if current_subject is None:
+                current_subject = subject
+
+            if subject != current_subject:
+
+                insert_row(
+                    cur,
+                    predicates,
+                    current_subject,
+                    row
+                )
+
+                count += 1
+
+                if count % 10000 == 0:
+                    con.commit()
+                    print(
+                        "Inserted",
+                        count,
+                        "rows"
+                    )
+
+                current_subject = subject
+                row = {}
+
+            row[predicate] = obj
+
+
+    if current_subject is not None:
+        insert_row(
+            cur,
+            predicates,
+            current_subject,
+            row
+        )
+
+    con.commit()
+
+    print(
+        "Done. Inserted",
+        count + 1,
+        "rows"
+    )
+
+    con.close()
+
+
+def main():
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-i",
+        "--input",
+        required=True
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        required=True
+    )
+
+    args = parser.parse_args()
+
+    import_nt(
+        args.input,
+        args.output
+    )
+
+
+if __name__ == "__main__":
+    main()

utils/install_k3s_on_vm_cluster.sh

@@ -0,0 +1,230 @@
+#!/bin/bash
+echo -e " \033[33;5m    __  _          _        ___                            \033[0m"
+echo -e " \033[33;5m    \ \(_)_ __ ___( )__    / _ \__ _ _ __ __ _  __ _  ___  \033[0m"
+echo -e " \033[33;5m     \ \ | '_ \` _ \/ __|  / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m"
+echo -e " \033[33;5m  /\_/ / | | | | | \__ \ / /_\\  (_| | | | (_| | (_| |  __/ \033[0m"
+echo -e " \033[33;5m  \___/|_|_| |_| |_|___/ \____/\__,_|_|  \__,_|\__, |\___| \033[0m"
+echo -e " \033[33;5m                                               |___/       \033[0m"
+
+echo -e " \033[36;5m         _  _________   ___         _        _ _           \033[0m"
+echo -e " \033[36;5m        | |/ |__ / __| |_ _|_ _  __| |_ __ _| | |          \033[0m"
+echo -e " \033[36;5m        | ' < |_ \__ \  | || ' \(_-|  _/ _\` | | |          \033[0m"
+echo -e " \033[36;5m        |_|\_|___|___/ |___|_||_/__/\__\__,_|_|_|          \033[0m"
+echo -e " \033[36;5m                                                           \033[0m"
+echo -e " \033[32;5m             https://youtube.com/@jims-garage              \033[0m"
+echo -e " \033[32;5m                                                           \033[0m"
+
+
+#############################################
+# YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
+#############################################
+
+# Version of Kube-VIP to deploy
+KVVERSION="v1.2.2"
+
+# K3S Version
+k3sVersion="v1.36.2+k3s1"
+
+# Set the IP addresses of the master and work nodes
+master1=192.168.1.133
+worker1=192.168.1.134
+worker2=192.168.1.222
+worker3=192.168.1.223
+
+# User of remote machines
+user=kubemeister
+
+# Interface used on remotes
+interface=enp0s11
+
+# Set the virtual IP address (VIP)
+vip=192.168.3.50
+
+# Array of master nodes
+masters=($master1)
+
+# Array of worker nodes
+workers=($worker1 $worker2 $worker3)
+
+# Array of all
+all=($master1 $worker1 $worker2 $worker3)
+
+# Array of all minus master
+allnomaster1=($master1 $worker1 $worker2 $worker3)
+
+#Loadbalancer IP range
+lbrange=192.168.3.60-192.168.3.80
+
+#ssh certificate name variable
+certName=id_kubemeister
+
+#ssh config file
+config_file=~/.ssh/config
+
+#############################################
+#            DO NOT EDIT BELOW              #
+#############################################
+# For testing purposes - in case time is wrong due to VM snapshots
+sudo timedatectl set-ntp off
+sudo timedatectl set-ntp on
+
+# Move SSH certs to ~/.ssh and change permissions
+cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh
+chmod 600 /home/$user/.ssh/$certName
+chmod 644 /home/$user/.ssh/$certName.pub
+
+# Install k3sup to local machine if not already present
+if ! command -v k3sup version &> /dev/null
+then
+    echo -e " \033[31;5mk3sup not found, installing\033[0m"
+    curl -sLS https://get.k3sup.dev | sh
+    sudo install k3sup-arm64 /usr/local/bin/
+else
+    echo -e " \033[32;5mk3sup already installed\033[0m"
+fi
+
+# Install Kubectl if not already present
+if ! command -v kubectl version &> /dev/null
+then
+    echo -e " \033[31;5mKubectl not found, installing\033[0m"
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl"
+    sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+else
+    echo -e " \033[32;5mKubectl already installed\033[0m"
+fi
+
+# Check for SSH config file, create if needed, add/change Strict Host Key Checking (don't use in production!)
+
+if [ ! -f "$config_file" ]; then
+  # Create the file and add the line
+  echo "StrictHostKeyChecking no" > "$config_file"
+  # Set permissions to read and write only for the owner
+  chmod 600 "$config_file"
+  echo "File created and line added."
+else
+  # Check if the line exists
+  if grep -q "^StrictHostKeyChecking" "$config_file"; then
+    # Check if the value is not "no"
+    if ! grep -q "^StrictHostKeyChecking no" "$config_file"; then
+      # Replace the existing line
+      sed -i 's/^StrictHostKeyChecking.*/StrictHostKeyChecking no/' "$config_file"
+      echo "Line updated."
+    else
+      echo "Line already set to 'no'."
+    fi
+  else
+    # Add the line to the end of the file
+    echo "StrictHostKeyChecking no" >> "$config_file"
+    echo "Line added."
+  fi
+fi
+
+#add ssh keys for all nodes
+for node in "${all[@]}"; do
+  ssh-copy-id $user@$node
+done
+
+# Install policycoreutils for each node
+for newnode in "${all[@]}"; do
+  ssh $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
+  NEEDRESTART_MODE=a apt-get install policycoreutils -y
+  exit
+EOF
+  echo -e " \033[32;5mPolicyCoreUtils installed!\033[0m"
+done
+
+# Step 1: Bootstrap First k3s Node
+mkdir ~/.kube
+k3sup install \
+  --ip $master1 \
+  --user $user \
+  --tls-san $vip \
+  --cluster \
+  --k3s-version $k3sVersion \
+  --k3s-extra-args "--disable traefik --disable servicelb --flannel-iface=$interface --node-ip=$master1 --node-taint node-role.kubernetes.io/master=true:NoSchedule" \
+  --merge \
+  --sudo \
+  --local-path $HOME/.kube/config \
+  --ssh-key $HOME/.ssh/$certName \
+  --context k3s-ha
+echo -e " \033[32;5mFirst Node bootstrapped successfully!\033[0m"
+
+# Step 2: Install Kube-VIP for HA
+kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+
+# Step 3: Download kube-vip
+curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/kube-vip
+cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g' > $HOME/kube-vip.yaml
+
+# Step 4: Copy kube-vip.yaml to master1
+scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$master1:~/kube-vip.yaml
+
+
+# Step 5: Connect to Master1 and move kube-vip.yaml
+ssh $user@$master1 -i ~/.ssh/$certName <<- EOF
+  sudo mkdir -p /var/lib/rancher/k3s/server/manifests
+  sudo mv kube-vip.yaml /var/lib/rancher/k3s/server/manifests/kube-vip.yaml
+EOF
+
+# Step 6: Add new master nodes (servers) & workers
+for newnode in "${masters[@]}"; do
+  k3sup join \
+    --ip $newnode \
+    --user $user \
+    --sudo \
+    --k3s-version $k3sVersion \
+    --server \
+    --server-ip $master1 \
+    --ssh-key $HOME/.ssh/$certName \
+    --k3s-extra-args "--disable traefik --disable servicelb --flannel-iface=$interface --node-ip=$newnode --node-taint node-role.kubernetes.io/master=true:NoSchedule" \
+    --server-user $user
+  echo -e " \033[32;5mMaster node joined successfully!\033[0m"
+done
+
+# add workers
+for newagent in "${workers[@]}"; do
+  k3sup join \
+    --ip $newagent \
+    --user $user \
+    --sudo \
+    --k3s-version $k3sVersion \
+    --server-ip $master1 \
+    --ssh-key $HOME/.ssh/$certName \
+    --k3s-extra-args "--node-label \"longhorn=true\" --node-label \"worker=true\""
+  echo -e " \033[32;5mAgent node joined successfully!\033[0m"
+done
+
+# Step 7: Install kube-vip as network LoadBalancer - Install the kube-vip Cloud Provider
+kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+
+# Step 8: Install Metallb
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
+# Download ipAddressPool and configure using lbrange above
+curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/ipAddressPool
+cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml
+kubectl apply -f $HOME/ipAddressPool.yaml
+
+# Step 9: Test with Nginx
+kubectl apply -f https://raw.githubusercontent.com/inlets/inlets-operator/master/contrib/nginx-sample-deployment.yaml -n default
+kubectl expose deployment nginx-1 --port=80 --type=LoadBalancer -n default
+
+echo -e " \033[32;5mWaiting for K3S to sync and LoadBalancer to come online\033[0m"
+
+while [[ $(kubectl get pods -l app=nginx -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do
+   sleep 1
+done
+
+# Step 10: Deploy IP Pools and l2Advertisement
+kubectl wait --namespace metallb-system \
+                --for=condition=ready pod \
+                --selector=component=controller \
+                --timeout=120s
+kubectl apply -f ipAddressPool.yaml
+kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/K3S-Deploy/l2Advertisement.yaml
+
+kubectl get nodes
+kubectl get svc
+kubectl get pods --all-namespaces -o wide
+
+echo -e " \033[32;5mHappy Kubing! Access Nginx at EXTERNAL-IP above\033[0m"

utils/remove_null_columns_under_threshold.py

@@ -0,0 +1,38 @@
+
+import sqlite3
+
+db = "paintingzz.db"
+
+conn = sqlite3.connect(db)
+cur = conn.cursor()
+
+table = "inspections"
+
+# Find completely NULL columns
+cur.execute(f"PRAGMA table_info({table})")
+columns = [row[1] for row in cur.fetchall()]
+
+empty_columns = []
+
+for col in columns:
+    print('cucurucu')
+    cur.execute(
+        f'SELECT COUNT(*) FROM "{table}" WHERE "{col}" IS NOT NULL'
+    )
+    count = cur.fetchone()[0]
+
+    if count <100 :
+        print('list b growing')
+        empty_columns.append(col)
+        print(empty_columns)
+
+print("Removing:", empty_columns)
+
+# SQLite >= 3.35 supports DROP COLUMN
+for col in empty_columns:
+    cur.execute(
+        f'ALTER TABLE "{table}" DROP COLUMN "{col}"'
+    )
+
+conn.commit()
+conn.close()

utils/replace-urls.py

@@ -0,0 +1,74 @@
+import sqlite3
+import requests
+import time
+
+DB = "paintingzz.db"
+
+BATCH_SIZE = 50
+
+conn = sqlite3.connect(DB)
+cur = conn.cursor()
+
+# Get all QIDs that still need labels
+cur.execute("""
+    SELECT qid
+    FROM wikidata_entities
+    WHERE label IS NULL
+""")
+
+qids = [row[0] for row in cur.fetchall()]
+
+print(f"{len(qids)} entities to resolve.")
+
+for start in range(0, len(qids), BATCH_SIZE):
+    batch = qids[start:start + BATCH_SIZE]
+
+    print(f"Processing {start + 1} - {start + len(batch)}")
+
+    response = requests.get(
+        "https://www.wikidata.org/w/api.php",
+        params={
+            "action": "wbgetentities",
+            "ids": "|".join(batch),
+            "props": "labels",
+            "languages": "en",
+            "format": "json"
+        },
+        headers={
+            # Wikidata appreciates a descriptive User-Agent
+            "User-Agent": "WikidataLabelResolver/1.0 (personal script)"
+        }
+    )
+
+    response.raise_for_status()
+
+    entities = response.json()["entities"]
+
+    for qid, entity in entities.items():
+        labels = entity.get("labels", {})
+
+        # Prefer English, otherwise use any available label
+        if "en" in labels:
+            label = labels["en"]["value"]
+        elif labels:
+            label = next(iter(labels.values()))["value"]
+        else:
+            label = None
+
+        cur.execute(
+            """
+            UPDATE wikidata_entities
+            SET label = ?
+            WHERE qid = ?
+            """,
+            (label, qid)
+        )
+
+    conn.commit()
+
+    # Be polite to Wikidata
+    time.sleep(6)
+
+conn.close()
+
+print("Done!")