main.bash
Rama main · réplica de lectura del repositorio original.
#!/bin/bash
set -o errexit # exit on any failure
set -o nounset # exit on undeclared variables
set -o pipefail # return value of all commands in a pipe
## set -o xtrace # command tracing
shopt -s failglob # If no matches are found, an error message is printed and the command is not executed
# PRESEED CONFIG: /machineConfigs/servers/preseed.cfg
# ANSIBLE CONFIG: /ansible/k3s.yaml
BLUE='\033[1;34m'
NC='\033[0m'
# https://www.how2shout.com/linux/how-to-install-and-configure-kvm-on-debian-11-bullseye-linux/
export LIBVIRT_DEFAULT_URI=qemu:///system
export SWAP_PATH=${SWAP_PATH:-"/home/swapfile.img"}
# HomeLab Functions
function retry {
local retries=$1
shift
local count=0
until "$@"; do
exit=$?
wait=$((15 + 5 ** count))
count=$((count + 1))
if [[ ${count} -lt "${retries}" ]]; then
echo "Retry ${count}/${retries} exited ${exit}, retrying in ${wait} seconds..."
sleep "${wait}"
else
echo "Retry ${count}/${retries} exited ${exit}, no more retries left."
return "${exit}"
fi
done
return 0
}
# wait_for_install_complete: Poll cluster operators for stability until timeout.
# This is a workaround for OKD 5.0 where EvaluationConditionsDetected stays
# Unknown/NoData on many operators in sandbox VMs, causing openshift-install
# to falsely report "cluster operators not stable" even when healthy.
# Only checks Progressing=False AND Degraded=False (ignores EvaluationConditionsDetected).
# Usage: wait_for_install_complete [dir]
# dir: optional assets directory (defaults to ".")
wait_for_install_complete() {
local dir="${1:-.}"
# Fall back to openshift-install if KUBECONFIG/kubectl not available
if [[ -z "${KUBECONFIG}" ]] || ! command -v kubectl &>/dev/null; then
echo "WARNING: KUBECONFIG not set or kubectl unavailable, falling back to openshift-install"
"${OKD}/openshift-install" wait-for install-complete --dir="${dir}" 2>&1 | \
grep -v "^INFO:" | grep -v "^time=" || true
return 0
fi
export KUBECONFIG
echo -e "${BLUE}Polling cluster operators for stability (timeout: 120m)...${NC}"
local timeout=$((120 * 60))
local interval=60
local elapsed=0
while (( elapsed < timeout )); do
local result
result=$(kubectl get co --no-headers 2>/dev/null | awk '
BEGIN { stable=0; unstable=0 }
{
progressing=$4
degraded=$5
if (progressing == "False" && degraded == "False") stable++
else unstable++
}
END {
if (stable > 0 && unstable == 0) print "STABLE"
else if (stable > 0) printf "WAITING:%d\n", unstable
else print "NO-OPERATORS"
}
')
if [[ "${result}" == "STABLE" ]]; then
echo "All cluster operators are stable."
return 0
fi
if [[ "${result}" == WAITING:* ]]; then
local count="${result#WAITING:}"
local minutes=$((elapsed / 60))
echo " [${minutes}m/$((timeout / 60))m] ${count} operator(s) still not stable..."
else
echo " [$((elapsed / 60))m/$((timeout / 60))m] Waiting for operators..."
fi
sleep "${interval}"
elapsed=$((elapsed + interval))
done
# Final check before failing
result=$(kubectl get co --no-headers 2>/dev/null | awk '
BEGIN { c=0 }
{ if ($4 != "False" || $5 != "False") c++ }
END { print c }
')
if [[ "${result}" -eq 0 ]] 2>/dev/null; then
echo "Cluster operators stabilized (final check passed)."
return 0
fi
echo "ERROR: Timeout waiting for cluster operators. ${result:-unknown} operator(s) still not stable."
return 1
}
ansible() {
ansible-playbook -i ansible/inventory --ask-become-pass ansible/servers.yaml --ask-pass \
-e 'ansible_python_interpreter=/usr/bin/python3'
}
stateful_workload_stop() {
# kubectl patch cronjobs -n netbox netbox-housekeeping -p '{"spec" : {"suspend" : true }}'
kubectl patch cronjobs -n immich immich-cron -p '{"spec" : {"suspend" : true }}'
kubectl patch cronjobs -n nextcloud nextcloud-preview -p '{"spec" : {"suspend" : true }}'
kubectl patch cronjobs -n nextcloud nextcloud-rsync -p '{"spec" : {"suspend" : true }}'
kubectl patch cronjobs -n nextcloud nextcloud-cron -p '{"spec" : {"suspend" : true }}'
kubectl scale --replicas=0 -n openshift-monitoring deployment/cluster-monitoring-operator
kubectl scale --replicas=0 -n openshift-monitoring deployment/prometheus-operator
kubectl scale --replicas=0 -n openshift-monitoring statefulset/alertmanager-main
kubectl scale --replicas=0 -n openshift-monitoring statefulset/prometheus-k8s
kubectl scale --replicas=0 -n openshift-user-workload-monitoring deployment/prometheus-operator
kubectl scale --replicas=0 -n openshift-user-workload-monitoring statefulset/alertmanager-user-workload
kubectl scale --replicas=0 -n openshift-user-workload-monitoring statefulset/prometheus-user-workload
kubectl scale --replicas=0 -n openshift-user-workload-monitoring statefulset/thanos-ruler-user-workload
kubectl scale --replicas=0 -n bitwarden statefulset/bitwarden
kubectl scale --replicas=0 -n gitea deployment/gitea
kubectl scale --replicas=0 -n grafana deployment/grafana
kubectl scale --replicas=0 -n heimdall statefulset/heimdall
kubectl scale --replicas=0 -n homeassistant statefulset/homeassistant
# kubectl scale --replicas=0 -n loki statefulset/loki
kubectl scale --replicas=0 -n nextcloud deployment/nextcloud
kubectl scale --replicas=0 -n immich statefulset/immich
kubectl scale --replicas=0 -n prometheus statefulset/prometheus
kubectl scale --replicas=0 -n prometheus statefulset/thanos-store-gateway
kubectl scale --replicas=0 -n prometheus statefulset/prometheus-truenas
kubectl scale --replicas=0 -n prometheus statefulset/thanos-truenas-store-gateway
kubectl scale --replicas=0 -n uptime-kuma statefulset/uptime-kuma
kubectl scale --replicas=0 -n vault statefulset/vault
kubectl scale --replicas=0 -n pihole statefulset/pihole
kubectl scale --replicas=0 -n technitium-dns statefulset/technitium-dns
kubectl scale --replicas=0 -n external-dns deployment/external-dns-microshift
kubectl scale --replicas=0 -n external-dns deployment/external-dns-microshift-vlan3
kubectl scale --replicas=0 -n external-dns deployment/external-dns-okd
kubectl scale --replicas=0 -n external-dns deployment/external-dns-okd-vlan3
kubectl scale --replicas=0 -n ntp deployment/ntp-rootless
kubectl scale --replicas=0 -n cloudflare-ddns deployment/cloudflare-ddns
kubectl scale --replicas=0 -n quay deployment/quay-operator-tng
kubectl scale --replicas=0 -n quay deployment/quay-quay-app
kubectl scale --replicas=0 -n quay deployment/quay-clair-app
kubectl scale --replicas=0 -n quay deployment/quay-quay-mirror
# kubectl scale --replicas=0 -n cockroach-operator-system deployments/cockroach-operator-manager
# kubectl scale --replicas=0 -n zitadel statefulset/crdb
kubectl scale --replicas=0 -n zitadel deployment/zitadel
# kubectl scale --replicas=0 -n mongodb-operator deployments/mongodb-kubernetes-operator
# kubectl scale --replicas=0 -n unifi-network-application statefulset/unifi-network-application
# kubectl scale --replicas=0 -n unifi-network-application statefulset/mongo-unifi-network-application
kubectl annotate cluster -n awx awx --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n quay clair --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n finance-tracker finance --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n gitea gitea --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n grafana grafana --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n homeassistant homeassistant --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n immich immich --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n netbox netbox --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n nextcloud-fpm nextcloud --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n quay quay --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n analytics-for-spotify spotify --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n analytics-for-spotify-ext spotify --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n stackrox stackrox --overwrite cnpg.io/hibernation=on
kubectl annotate cluster -n zitadel zitadel --overwrite cnpg.io/hibernation=on
kubectl scale --replicas=0 -n argocd deployment/argocd-operator-controller-manager
kubectl scale --replicas=0 -n argocd statefulset/argocd-application-controller
kubectl scale --replicas=0 -n argocd deployment/argocd-notifications-controller
kubectl scale --replicas=0 -n argocd deployment/argocd-repo-server
kubectl scale --replicas=0 -n argocd deployment/argocd-dex-server
kubectl scale --replicas=0 -n argocd deployment/argocd-server
kubectl scale --replicas=0 -n argocd-apps statefulset/argocd-apps-application-controller
kubectl scale --replicas=0 -n argocd-apps deployment/argocd-apps-dex-server
kubectl scale --replicas=0 -n argocd-apps deployment/argocd-apps-repo-server
kubectl scale --replicas=0 -n argocd-apps deployment/argocd-apps-server
kubectl scale --replicas=0 -n stackrox deployment/central
kubectl scale --replicas=0 -n stackrox deployment/scanner-db
kubectl scale --replicas=0 -n dragonfly-operator-system deployment/dragonfly-operator-controller-manager
kubectl scale --replicas=0 -n nextcloud statefulset/nextcloud-dragonfly
kubectl scale --replicas=0 -n gitea statefulset/gitea-dragonfly
kubectl scale --replicas=0 -n quay statefulset/quay-dragonfly
# kubectl scale --replicas=0 -n netbox statefulset/netbox-dragonfly
kubectl scale --replicas=0 -n loki-operator deployment/loki-operator-controller-manager
kubectl scale --replicas=0 -n openshift-netobserv-loki statefulset/netobserv-compactor
kubectl scale --replicas=0 -n openshift-netobserv-loki statefulset/netobserv-index-gateway
kubectl scale --replicas=0 -n openshift-netobserv-loki statefulset/netobserv-ingester
kubectl scale --replicas=0 -n openshift-netobserv-loki deployment/netobserv-distributor
kubectl scale --replicas=0 -n openshift-netobserv-loki deployment/netobserv-gateway
kubectl scale --replicas=0 -n openshift-netobserv-loki deployment/netobserv-querier
kubectl scale --replicas=0 -n openshift-netobserv-loki deployment/netobserv-query-frontend
kubectl scale --replicas=0 -n openshift-logging statefulset/logging-loki-compactor
kubectl scale --replicas=0 -n openshift-logging statefulset/logging-loki-index-gateway
kubectl scale --replicas=0 -n openshift-logging statefulset/logging-loki-ingester
kubectl scale --replicas=0 -n openshift-logging deployment/logging-loki-distributor
kubectl scale --replicas=0 -n openshift-logging deployment/logging-loki-gateway
kubectl scale --replicas=0 -n openshift-logging deployment/logging-loki-querier
kubectl scale --replicas=0 -n openshift-logging deployment/logging-loki-query-frontend
# kubectl scale --replicas=0 -n awx deployment/awx-operator-controller-manager
# kubectl scale --replicas=0 -n awx deployment/awx-task
# kubectl scale --replicas=0 -n awx deployment/awx-web
# kubectl scale --replicas=0 -n netbox deployment/netbox
# kubectl scale --replicas=0 -n netbox deployment/netbox-worker
kubectl delete jobs -A --all
kubectl delete pipelineruns -A --all
}
stateful_workload_start_pre() {
# kubectl scale --replicas=1 -n cockroach-operator-system deployments/cockroach-operator-manager
# kubectl scale --replicas=3 -n zitadel statefulset/crdb
# kubectl scale --replicas=1 -n mongodb-operator deployments/mongodb-kubernetes-operator
# kubectl scale --replicas=3 -n unifi-network-application statefulset/mongo-unifi-network-application
# kubectl scale --replicas=1 -n loki statefulset/loki
kubectl scale --replicas=1 -n prometheus statefulset/prometheus
kubectl scale --replicas=1 -n prometheus statefulset/thanos-store-gateway
kubectl scale --replicas=1 -n quay deployment/quay-operator-tng
kubectl annotate cluster -n awx awx --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n quay clair --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n finance-tracker finance --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n gitea gitea --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n grafana grafana --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n homeassistant homeassistant --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n immich immich --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n netbox netbox --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n nextcloud-fpm nextcloud --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n quay quay --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n analytics-for-spotify spotify --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n analytics-for-spotify-ext spotify --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n stackrox stackrox --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n zitadel zitadel --overwrite cnpg.io/hibernation=off
kubectl scale --replicas=1 -n pihole statefulset/pihole
kubectl scale --replicas=1 -n technitium-dns statefulset/technitium-dns
kubectl scale --replicas=2 -n ntp deployment/ntp-rootless
}
stateful_workload_start() {
kubectl scale --replicas=1 -n pihole statefulset/pihole
kubectl scale --replicas=1 -n technitium-dns statefulset/technitium-dns
kubectl scale --replicas=2 -n external-dns deployment/external-dns-microshift
kubectl scale --replicas=2 -n external-dns deployment/external-dns-microshift-vlan3
kubectl scale --replicas=2 -n external-dns deployment/external-dns-okd
kubectl scale --replicas=2 -n external-dns deployment/external-dns-okd-vlan3
kubectl scale --replicas=2 -n cloudflare-ddns deployment/cloudflare-ddns
# kubectl patch cronjobs -n netbox netbox-housekeeping -p '{"spec" : {"suspend" : false }}'
kubectl patch cronjobs -n immich immich-cron -p '{"spec" : {"suspend" : false }}'
kubectl patch cronjobs -n nextcloud nextcloud-preview -p '{"spec" : {"suspend" : false }}'
kubectl patch cronjobs -n nextcloud nextcloud-rsync -p '{"spec" : {"suspend" : false }}'
kubectl patch cronjobs -n nextcloud nextcloud-cron -p '{"spec" : {"suspend" : false }}'
kubectl scale --replicas=1 -n openshift-monitoring deployment/cluster-monitoring-operator
kubectl scale --replicas=1 -n dragonfly-operator-system deployment/dragonfly-operator-controller-manager
kubectl scale --replicas=2 -n nextcloud statefulset/nextcloud-dragonfly
kubectl scale --replicas=2 -n gitea statefulset/gitea-dragonfly
kubectl scale --replicas=2 -n quay statefulset/quay-dragonfly
# kubectl scale --replicas=2 -n netbox statefulset/netbox-dragonfly
# kubectl scale --replicas=1 -n cockroach-operator-system deployments/cockroach-operator-manager
# kubectl scale --replicas=3 -n zitadel statefulset/crdb
kubectl scale --replicas=2 -n zitadel deployment/zitadel
# kubectl scale --replicas=1 -n mongodb-operator deployments/mongodb-kubernetes-operator
# kubectl scale --replicas=3 -n unifi-network-application statefulset/mongo-unifi-network-application
# kubectl scale --replicas=1 -n unifi-network-application statefulset/unifi-network-application
kubectl scale --replicas=1 -n bitwarden statefulset/bitwarden
kubectl scale --replicas=2 -n gitea deployment/gitea
kubectl scale --replicas=2 -n grafana deployment/grafana
kubectl scale --replicas=1 -n heimdall statefulset/heimdall
kubectl scale --replicas=1 -n homeassistant statefulset/homeassistant
kubectl scale --replicas=1 -n loki statefulset/loki
kubectl scale --replicas=2 -n nextcloud deployment/nextcloud
kubectl scale --replicas=1 -n immich statefulset/immich
kubectl scale --replicas=1 -n prometheus statefulset/prometheus
kubectl scale --replicas=1 -n prometheus statefulset/thanos-store-gateway
kubectl scale --replicas=1 -n prometheus statefulset/prometheus-truenas
kubectl scale --replicas=1 -n prometheus statefulset/thanos-truenas-store-gateway
kubectl scale --replicas=1 -n uptime-kuma statefulset/uptime-kuma
kubectl scale --replicas=3 -n vault statefulset/vault
kubectl scale --replicas=1 -n quay deployment/quay-operator-tng
kubectl scale --replicas=2 -n quay deployment/quay-quay-app
kubectl scale --replicas=2 -n quay deployment/quay-clair-app
kubectl scale --replicas=2 -n quay deployment/quay-quay-mirror
kubectl annotate cluster -n awx awx --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n quay clair --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n finance-tracker finance --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n gitea gitea --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n grafana grafana --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n homeassistant homeassistant --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n immich immich --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n netbox netbox --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n nextcloud-fpm nextcloud --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n quay quay --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n analytics-for-spotify spotify --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n analytics-for-spotify-ext spotify --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n stackrox stackrox --overwrite cnpg.io/hibernation=off
kubectl annotate cluster -n zitadel zitadel --overwrite cnpg.io/hibernation=off
kubectl scale --replicas=1 -n argocd deployment/argocd-operator-controller-manager
kubectl scale --replicas=1 -n argocd statefulset/argocd-application-controller
kubectl scale --replicas=1 -n argocd statefulset/argocd-application-controller
kubectl scale --replicas=1 -n argocd deployment/argocd-repo-server
kubectl scale --replicas=1 -n argocd deployment/argocd-dex-server
kubectl scale --replicas=1 -n argocd deployment/argocd-server
kubectl scale --replicas=1 -n argocd-apps statefulset/argocd-apps-application-controller
kubectl scale --replicas=1 -n argocd-apps deployment/argocd-apps-dex-server
kubectl scale --replicas=1 -n argocd-apps deployment/argocd-apps-repo-server
kubectl scale --replicas=1 -n argocd-apps deployment/argocd-apps-server
kubectl scale --replicas=1 -n stackrox deployment/central
kubectl scale --replicas=1 -n stackrox deployment/scanner-db
kubectl scale --replicas=1 -n loki-operator deployment/loki-operator-controller-manager
# kubectl scale --replicas=1 -n awx deployment/awx-operator-controller-manager
# kubectl scale --replicas=1 -n awx deployment/netbox
# kubectl scale --replicas=1 -n awx deployment/netbox-worker
# echo -e "\nkubectl exec -it vault-0 -n vault -- vault operator unseal --tls-skip-verify"
}
kvm-infra() {
ssh 10.0.0.110
virt-install \
--noautoconsole \
--graphics vnc \
--name=k3s-server-2 \
--os-variant=debian10 \
--vcpus sockets=1,cores=1,threads=2 \
--ram=1792 \
--disk "/mnt/storage/vm/k3s-server-2".img,,format=raw,size=25 \
--network bridge=br0,mac="10:00:00:00:01:02" \
--location=http://ftp.us.debian.org/debian/dists/stable/main/installer-amd64/ \
--extra-args="\
auto=true priority=critical vga=normal hostname=k3s-server-2 \
url=http://10.0.0.5:7071/preseed.cfg"
virt-install \
--noautoconsole \
--graphics vnc \
--name=k3s-worker-1 \
--os-variant=debian10 \
--vcpus sockets=1,cores=2,threads=2 \
--ram=10240 \
--disk "/mnt/storage/vm/k3s-worker-1".img,,format=raw,size=125 \
--network bridge=br0,mac="10:00:00:00:01:11" \
--location=http://ftp.us.debian.org/debian/dists/stable/main/installer-amd64/ \
--extra-args="\
auto=true priority=critical vga=normal hostname=k3s-worker-1 \
url=http://10.0.0.5:7071/preseed.cfg"
}
kvm() {
ssh 10.101.1.6
export LIBVIRT_DEFAULT_URI=qemu:///system
virt-install \
--noautoconsole \
--graphics vnc \
--name=k3s-server-3 \
--os-variant=debian10 \
--vcpus sockets=1,cores=1,threads=2 \
--ram=1792 \
--disk "/mnt/storage/vm/k3s-server-3".img,,format=raw,size=25 \
--network bridge=br0,mac="10:00:00:00:01:03" \
--location=http://ftp.us.debian.org/debian/dists/stable/main/installer-amd64/ \
--extra-args="\
auto=true priority=critical vga=normal hostname=k3s-server-3 \
url=http://10.0.0.5:7071/preseed.cfg"
virt-install \
--noautoconsole \
--graphics vnc \
--name=k3s-worker-2 \
--os-variant=debian10 \
--vcpus sockets=1,cores=2,threads=2 \
--ram=4352 \
--disk "/mnt/storage/vm/k3s-worker-2".img,,format=raw,size=125 \
--network bridge=br0,mac="10:00:00:00:01:12" \
--location=http://ftp.us.debian.org/debian/dists/stable/main/installer-amd64/ \
--extra-args="\
auto=true priority=critical vga=normal hostname=k3s-worker-2 \
url=http://10.0.0.5:7071/preseed.cfg"
}
label_nodes() {
# Taint
kubectl taint node k3s-server-1 node-role.kubernetes.io/control-plane:NoSchedule --overwrite
kubectl taint node k3s-server-2 node-role.kubernetes.io/control-plane:NoSchedule --overwrite
kubectl taint node k3s-server-3 node-role.kubernetes.io/control-plane:NoSchedule --overwrite
kubectl taint node k3s-server-2 node-role.kubernetes.io/control-plane:NoSchedule --overwrite
kubectl taint node k3s-server-3 node-role.kubernetes.io/control-plane:NoSchedule --overwrite
# Role Label
kubectl label node k3s-worker-1 node-role.kubernetes.io/infra=true --overwrite
kubectl label node k3s-worker-1 node-role.kubernetes.io/worker=true --overwrite
kubectl label node k3s-worker-2 node-role.kubernetes.io/worker=true --overwrite
# Zone Labels
kubectl label node k3s-server-1 topology.kubernetes.io/zone=bare-metal --overwrite
kubectl label node k3s-server-2 topology.kubernetes.io/zone=infra-kvm --overwrite
kubectl label node k3s-server-3 topology.kubernetes.io/zone=kvm --overwrite
kubectl label node k3s-worker-1 topology.kubernetes.io/zone=infra-kvm --overwrite
kubectl label node k3s-worker-2 topology.kubernetes.io/zone=kvm --overwrite
# Longhorn Label
kubectl label node k3s-server-1 node.longhorn.io/create-default-disk=true --overwrite
kubectl label node k3s-worker-1 node.longhorn.io/create-default-disk=true --overwrite
kubectl label node k3s-worker-2 node.longhorn.io/create-default-disk=true --overwrite
# #Drain DaemonSets Temporarily off a Node
# kubectl taint node k3s-server- node-role.kubernetes.io/control-plane:NoExecute
# kubectl taint node k3s-server- node-role.kubernetes.io/control-plane:NoExecute-
}
k3s_setup() {
# Configure PFsense / HAProxy / DHCP / DNS
# Servers
ansible-playbook -i ansible/inventory --ask-become-pass ansible/servers.yaml --ask-pass
# Reboot All Kubernetes Nodes && For KVM Nodes Update DHCP Server With New Mac Address
# Install VMs
bash kvm_k3s.bash kvm-infra
bash kvm_k3s.bash kvm
bash kvm_k3s.bash ansible
# Reboot VMs
# Install K3s
# First Time Setup
export K3S_TOKEN=""
export RESERVED="--kubelet-arg system-reserved=cpu=250m,memory=500Mi --kubelet-arg kube-reserved=cpu=250m,memory=500Mi"
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server --cluster-init --tls-san 10.0.0.100 --tls-san k3s.<URL>.com ${RESERVED}" \
INSTALL_K3S_CHANNEL=v1.23 sh -
# Servers
export K3S_TOKEN=""
export RESERVED="--kubelet-arg system-reserved=cpu=250m,memory=500Mi --kubelet-arg kube-reserved=cpu=250m,memory=500Mi"
curl -sfL https://get.k3s.io |
INSTALL_K3S_EXEC="server --server https://10.0.0.100:6443 --disable traefik ${RESERVED}" \
K3S_URL=https://10.0.0.100:6443 INSTALL_K3S_CHANNEL=v1.23 sh -
# Agents/Workers
export K3S_TOKEN=""
export RESERVED="--kubelet-arg system-reserved=cpu=250m,memory=250Mi --kubelet-arg kube-reserved=cpu=250m,memory=250Mi"
curl -sfL https://get.k3s.io |
INSTALL_K3S_EXEC="${RESERVED}" \
K3S_URL=https://10.0.0.100:6443 INSTALL_K3S_CHANNEL=v1.23 sh -
# Label Nodes
bash kvm_k3s label_nodes
}
k3s_image_cleanup() {
export IP_LIST="5 102 103 111 112"
for IP in ${IP_LIST}; do
ssh -t arthur@10.0.0."${IP}" "sudo k3s crictl rmi --prune"
done
}
# Sandbox Functions
# SANDBOX HAPROXY: /machineConfigs/sandbox/var/etc/haproxy
# Setup DHCP Network
# virsh net-edit default
# <host mac='10:10:00:00:00:03' ip='10.10.10.3'/>
# <host mac='10:10:00:00:00:04' ip='10.10.10.4'/>
# <host mac='10:10:00:00:00:05' ip='10.10.10.5'/>
# <host mac='10:10:00:00:00:06' ip='10.10.10.6'/>
# <host mac='10:10:00:00:00:07' ip='10.10.10.7'/>
# <host mac='10:10:00:00:00:08' ip='10.10.10.8'/>
# <host mac='10:10:00:00:00:09' ip='10.10.10.9'/>
# virsh net-destroy default && virsh net-start default
export START_IP=11
export PASSWORD=${PASSWORD:-}
stop_cluster() {
echo -e "\n${BLUE}Stopping Cluster:${NC}"
for NODE in ${NODES}; do
if virsh destroy "${PREFIX}${NODE}"; then
echo "Shutting Down:${PREFIX}${NODE}"
fi
done
}
start_cluster() {
echo -e "\n${BLUE}Starting Cluster:${NC}"
for NODE in ${NODES}; do
virsh start "${PREFIX}${NODE}"
echo "Started ${PREFIX}${NODE}"
done
}
reboot_cluster() {
echo -e "\n${BLUE}Rebooting Cluster:${NC}"
for NODE in ${NODES}; do
virsh reboot "${PREFIX}${NODE}"
echo "Started ${PREFIX}${NODE}"
done
IP=${START_IP}
for NODE in ${NODES}; do
finished="0"
while [[ "${finished}" = "0" ]]; do
sleep 1
if nc -z 10.10.10."${IP}" 22; then
echo "${PREFIX}${NODE} is up."
finished=1
fi
done
((IP++))
done
}
delete_cluster() {
stop_cluster
echo -e "\n${BLUE}Deleting Cluster:${NC}"
for NODE in ${NODES}; do
echo "${NODE}"
if virsh undefine "${PREFIX}${NODE}" --remove-all-storage; then
echo "Deleting: ${PREFIX}${NODE}"
fi
done
}
ansible_sandbox() {
echo -e "\n${BLUE}Running Ansible Playbooks:${NC}"
if [[ -z "${PASSWORD}" ]]; then
echo -n Password:
read -r -s PASSWORD
echo ""
fi
IP=${START_IP}
# Create Ansible Inventory File
echo "[servers]" >/tmp/inventory
for NODE in ${NODES}; do
echo "10.10.10.${IP}" >>/tmp/inventory
((IP++))
done
# Run Playbooks On All Machines
ansible-playbook -i /tmp/inventory ansible/servers.yaml --extra-vars \
"ansible_become_pass=${PASSWORD} ansible_ssh_pass=${PASSWORD}"
}
label_vms() {
load_kubeconfig
echo -e "\n${BLUE}Labeling Nodes:${NC}"
for NODE in ${NODES}; do
if [[ "${NODE}" =~ "master" || "${NODE}" =~ "server" ]]; then
kubectl label nodes "${PREFIX}${NODE}" node-type=master --overwrite
kubectl taint node "${PREFIX}${NODE}" node-role.kubernetes.io/control-plane:NoSchedule --overwrite
elif [[ "${NODE}" =~ "worker" || "${NODE}" =~ "agent" ]]; then
kubectl label node "${PREFIX}${NODE}" node-role.kubernetes.io/worker=true --overwrite
ZONE="EVEN"
if ((${NODE##*-} % 2)); then
ZONE="ODD"
fi
kubectl label node "${PREFIX}${NODE}" topology.kubernetes.io/zone="${ZONE}" --overwrite
kubectl label node "${PREFIX}${NODE}" node.longhorn.io/create-default-disk=true --overwrite
fi
done
}
load_kubeconfig() {
echo -e "\n${BLUE}Loading KubeConfig:${NC}"
export KUBECONFIG=/mnt/storage/vm/${PREFIX}/${PREFIX}.yaml
}
uninstall_k3s() {
echo -e "\n\n${BLUE}Uninstall k3s:${NC}"
if [[ -z "${PASSWORD}" ]]; then
echo -n Password:
read -r -s PASSWORD
echo ""
fi
IP=${START_IP}
for NODE in ${NODES}; do
echo "${PREFIX}${NODE}"
if [[ "${NODE}" =~ "master" || "${NODE}" =~ "server" ]]; then
sshpass -p "${PASSWORD}" ssh -t 10.10.10."${IP}" "echo ${PASSWORD} | sudo -S /usr/local/bin/k3s-uninstall.sh"
elif [[ "${NODE}" =~ "worker" || "${NODE}" =~ "agent" ]]; then
sshpass -p "${PASSWORD}" ssh -t 10.10.10."${IP}" "echo ${PASSWORD} | sudo -S /usr/local/bin/k3s-agent-uninstall.sh"
fi
((IP++))
done
}
install_k3s() {
# Install & Setup k3s
echo -e "\n\n${BLUE}Install & Setup k3s:${NC}"
if [[ -z "${PASSWORD}" ]]; then
echo -n Password:
read -r -s PASSWORD
echo ""
fi
K3S="echo ${PASSWORD} | sudo -S ls; curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='"
K3S_RESERVED="--kubelet-arg system-reserved=cpu=125m,memory=250Mi --kubelet-arg kube-reserved=cpu=125m,memory=250Mi'"
K3S_CONFIG="K3S_TOKEN=${K3S_TOKEN} INSTALL_K3S_CHANNEL=v1.23 sh -"
INSTALL="${K3S} server --cluster-init --disable traefik ${K3S_RESERVED} ${K3S_CONFIG}"
SERVER=" ${K3S} server --server https://10.10.10.1:6443 --disable traefik ${K3S_RESERVED} ${K3S_CONFIG}"
WORKER=" ${K3S} ${K3S_RESERVED} K3S_URL=https://10.10.10.1:6443 ${K3S_CONFIG}"
IP=${START_IP}
for NODE in ${NODES}; do
KUBE=0
if [[ "${NODE}" =~ "master" || "${NODE}" =~ "server" ]]; then
if [[ ${IP} -eq ${START_IP} ]]; then
CONFIG=${INSTALL}
KUBE=1
else
CONFIG=${SERVER}
fi
elif [[ "${NODE}" =~ "worker" || "${NODE}" =~ "agent" ]]; then
CONFIG=${WORKER}
fi
echo "${PREFIX}${NODE}"
sshpass -p "${PASSWORD}" ssh -t 10.10.10."${IP}" "${CONFIG}"
if [[ ${KUBE} -eq 1 ]]; then
CMD="echo ${PASSWORD} | sudo -S cp /etc/rancher/k3s/k3s.yaml /tmp; sudo chmod 777 /tmp/k3s.yaml"
sshpass -p "${PASSWORD}" ssh -t 10.10.10."${START_IP}" "${CMD}"
sshpass -p "${PASSWORD}" scp 10.10.10."${START_IP}":/tmp/k3s.yaml "/mnt/storage/vm/${PREFIX}/${PREFIX}.yaml"
sed -i "s,127.0.0.1,10.10.10.1,g" "/mnt/storage/vm/${PREFIX}/${PREFIX}.yaml"
fi
((IP++))
done
echo -e "\n\n${BLUE}Install Complete!${NC}"
echo "export KUBECONFIG=/mnt/storage/vm/${PREFIX}/${PREFIX}.yaml"
echo "${PREFIX} k3s Install Complete!"
}
install_addons() {
# Tweak Settings from Physical HomeLab
load_kubeconfig
echo -e "\n${BLUE}Installing Cluster Addons:${NC}"
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-}
if [[ -z "${URL}" ]]; then
echo -n URL:
read -r -s URL
echo ""
fi
echo "${URL}"
rm -rf /tmp/kubernetes
cp -r kubernetes /tmp/
echo -e "\n${BLUE}Traefik:${NC}"
kubectl apply -f /tmp/kubernetes/traefik/traefik-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/traefik/traefik-dashboard-traefik.yaml
rm -f /tmp/kubernetes/traefik/cluster-wildcard-certificate.yaml
kubectl apply -f /tmp/kubernetes/traefik/traefik-crd.yaml
kubectl apply -f /tmp/kubernetes/traefik
echo -e "\n${BLUE}Kube Eagle:${NC}"
kubectl apply -f /tmp/kubernetes/kube-eagle
echo -e "\n${BLUE}Kubernetes Dashboard:${NC}"
kubectl apply -f /tmp/kubernetes/kubernetes-dashboard/kubernetes-dashboard-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/kubernetes-dashboard/kubernetes-dashboard-traefik.yaml
kubectl apply -f /tmp/kubernetes/kubernetes-dashboard
echo -e "\n${BLUE}Longhorn:${NC}"
kubectl apply -f /tmp/kubernetes/longhorn/longhorn-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/longhorn/longhorn-traefik.yaml
kubectl apply -f /tmp/kubernetes/longhorn
kubectl patch daemonset -n longhorn-system longhorn-manager --type=json -p='[{"op": "remove", "path": "/spec/template/spec/tolerations"}]'
echo -e "\n${BLUE}Waiting for Longhorn to Boot:${NC}"
local lh_count=0
lh_count=$(kubectl get pods -n longhorn-system | grep -cv Running) || { echo "Failed to get Longhorn pods count" >&2; return 1; }
while [[ "${lh_count}" -ne 1 ]]; do
sleep 1
done
echo -e "\n${BLUE}Prometheus:${NC}"
kubectl apply -f /tmp/kubernetes/prometheus/prometheus-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/prometheus/prometheus-traefik.yaml
kubectl apply -f /tmp/kubernetes/prometheus
echo -e "\n${BLUE}Grafana:${NC}"
kubectl apply -f /tmp/kubernetes/grafana/grafana-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/grafana/grafana-traefik.yaml
kubectl apply -f /tmp/kubernetes/grafana
kubectl patch deployment -n grafana grafana --type=json -p='[{"op": "remove", "path": "/spec/template/spec/containers/0/env"}]'
echo -e "\n${BLUE}Grafana Loki:${NC}"
kubectl apply -f /tmp/kubernetes/grafana-loki
echo -e "\n${BLUE}Grafana Promtail:${NC}"
kubectl apply -f /tmp/kubernetes/grafana-promtail
echo -e "\n\n${BLUE}Addons Installed!${NC}"
}
install_addons_optional() {
load_kubeconfig
echo -e "\n${BLUE}Heimdall:${NC}"
kubectl apply -f /tmp/kubernetes/heimdall/heimdall-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/heimdall/heimdall-traefik.yaml
sed -i "s/replicas: 1/replicas: 0/g" /tmp/kubernetes/heimdall/heimdall-statefulSet.yaml
kubectl apply -f /tmp/kubernetes/heimdall
kubectl patch statefulset -n heimdall heimdall --type=json -p='[{"op": "remove", "path": "/spec/template/spec/affinity"}]'
kubectl patch statefulset -n heimdall heimdall --type=json -p='[{"op": "remove", "path": "/spec/template/spec/tolerations"}]'
kubectl scale --replicas=1 statefulSet/heimdall -n heimdall
echo -e "\n${BLUE}Uptime Kuma:${NC}"
kubectl apply -f /tmp/kubernetes/uptime-kuma/uptime-kuma-namespace.yaml
sed -i "s/<URL>/${URL}/g" /tmp/kubernetes/uptime-kuma/uptime-kuma-traefik.yaml
sed -i "s/replicas: 1/replicas: 0/g" /tmp/kubernetes/uptime-kuma/uptime-kuma-statefulSet.yaml
kubectl apply -f /tmp/kubernetes/uptime-kuma
kubectl patch statefulset -n uptime-kuma uptime-kuma --type=json -p='[{"op": "remove", "path": "/spec/template/spec/affinity"}]'
kubectl patch statefulset -n uptime-kuma uptime-kuma --type=json -p='[{"op": "remove", "path": "/spec/template/spec/tolerations"}]'
kubectl scale --replicas=1 statefulSet/uptime-kuma -n uptime-kuma
}
get_dashboard_secret() {
load_kubeconfig
echo -e "\n${BLUE}Kubernetes Dashboard Secret:${NC}"
# shellcheck disable=SC2046
local sa_name=""
sa_name=$(kubectl get serviceaccount admin-user -n kubernetes-dashboard -o jsonpath="{.secrets[0].name}") || { echo "Failed to get service account secret name" >&2; return 1; }
kubectl get secret -n kubernetes-dashboard \
"${sa_name}" \
-o jsonpath="{.data.token}" | base64 --decode
}
preseed_server() {
echo -e "\n${BLUE}Loading Preseed Server:${NC}"
echo -n Password:
read -r -s PASSWORD
echo ""
# Startup Preseed Web Server
mkdir -p /tmp/preseed/
cp machineConfigs/preseed.cfg /tmp/preseed//preseed.cfg
mkpasswd -m md5 "${PASSWORD}"
local preseed_hash=""
preseed_hash=$(mkpasswd -s -m md5 "${PASSWORD}") || { echo "Failed to generate password hash" >&2; return 1; }
sed -i "s,#d-i passwd/user-password-crypted password <HASH>,d-i passwd/user-password-crypted password ${preseed_hash},g" "/tmp/preseed/preseed.cfg"
python3 -m http.server --directory /tmp/preseed/
}
install_cluster() {
echo -e "\n${BLUE}Installing Cluster:${NC}"
echo -n Password:
read -r -s PASSWORD
echo ""
# IP Counter
IP=${START_IP}
# Delete Cluster If Exists
delete_cluster
mkdir -p "/mnt/storage/vm/${PREFIX}"
for NODE in ${NODES}; do
echo -e "\n\n${BLUE}Creating: ${PREFIX}${NODE}${NC}"
if [[ "${NODE}" =~ "master" || "${NODE}" =~ "server" ]]; then
CPU=3
MEMORY=4096
DISK=15
elif [[ "${NODE}" =~ "worker" || "${NODE}" =~ "agent" ]]; then
CPU=6
MEMORY=8192
DISK=30
fi
# Install VM
#--debug \
#--wait=-1 \
# shellcheck disable=SC2140
virt-install \
--noautoconsole \
--graphics vnc \
--name="${PREFIX}${NODE}" \
--os-variant=debian11 \
--vcpus sockets=1,cores=1,threads="${CPU}" \
--ram="${MEMORY}" \
--disk "/mnt/storage/vm/${PREFIX}/${PREFIX}${NODE}".img,size="${DISK}" \
--network network=default,model=virtio,mac="10:10:00:00:00:${IP}" \
--location=http://ftp.us.debian.org/debian/dists/stable/main/installer-amd64/ \
--extra-args="\
auto=true priority=critical vga=normal hostname=${PREFIX}${NODE} \
url=http://10.10.10.1:8000/preseed.cfg"
# https://crysol.org/recipe/2012-12-25/virtual-machine-unattended-debian-installations-with-libvirt-and-d-i-preseeding.html
((IP++))
done
# Wait for Machines to Finish Installing OS, Then Boot Machines
echo -e "\n\n${BLUE}Booting Nodes:${NC}"
echo "Waiting for Installation to Finish..."
echo "http://localhost:9090/machines"
for NODE in ${NODES}; do
# https://serverfault.com/a/386867
finished="0"
while [[ "${finished}" = "0" ]]; do
sleep 5
local running=0
running=$(virsh list --all | grep "running" | grep "${PREFIX}${NODE}" | wc -c) || { echo "Failed to check VM ${NODE} status" >&2; return 1; }
if [[ "${running}" -eq 0 ]]; then
echo "Starting vm ${PREFIX}${NODE}"
sleep 1
virsh start "${PREFIX}${NODE}"
finished=1
fi
done
done
# Add Machines to Known Hosts
echo -e "\n\n${BLUE}Add SSH Known Hosts:${NC}"
IP=${START_IP}
for NODE in ${NODES}; do
finished="0"
while [[ "${finished}" = "0" ]]; do
sleep 1
if nc -z 10.10.10."${IP}" 22; then
ssh-keygen -f "${HOME}/.ssh/known_hosts" -R "10.10.10.${IP}"
ssh-keyscan 10.10.10."${IP}" >>"${HOME}/.ssh/known_hosts"
finished=1
fi
done
((IP++))
done
# Run Playbooks On All Machines
ansible_sandbox
reboot_cluster
}
install_k3s_cluster() {
export PREFIX="sk3s" # Sandbox k3s
export K3S_TOKEN=${PREFIX} # Sandbox k3s
export NODES="-master-0 -master-1 -master-2 -worker-0 -worker-1 -worker-2 -worker-3"
install_cluster
# Install & Setup k3s
install_k3s
export KUBECONFIG="/mnt/storage/vm/${PREFIX}/${PREFIX}.yaml"
local node_count=0
node_count=$(kubectl get nodes | wc -l) || { echo "Failed to get node count" >&2; return 1; }
local expected=0
expected=$(echo "${NODES}" | wc -w) || { echo "Failed to count expected nodes" >&2; return 1; }
while [[ "${node_count}" -le "${expected}" ]]; do
sleep 1
done
# Label Nodes
label_vms
# Install Addons
install_addons
install_addons_optional
get_dashboard_secret
echo " "
echo "export KUBECONFIG=/mnt/storage/vm/${PREFIX}/${PREFIX}.yaml"
echo "${PREFIX} k3s Install Complete!"
}
monitor_okd_virt() {
export HOMELAB="${PWD}"
export OKD=/tmp/okd
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
kubectl config set-cluster okd --insecure-skip-tls-verify=true
"${OKD}/openshift-install" agent wait-for bootstrap-complete --dir "${OKD}/okd/"
wait_for_install_complete "${OKD}/okd/"
}
delete_okd_virt() {
export HOMELAB="${PWD}"
export OKD=/tmp/okd
# TODO, Automate Finding Which IPs to Remove
ssh-keygen -R 10.103.10.101
ssh-keygen -R 10.103.10.102
ssh-keygen -R 10.103.10.103
echo -e "\n\n${BLUE}Delete All Existing Data:${NC}"
rm -rf \
"${OKD}/oc" \
"${OKD}/kubectl" \
"${OKD}/openshift-install" \
"${OKD}/okd"
# "${HOME}/.cache/agent" \
# ${OKD}/openshift-install-linux* \
# ${OKD}/openshift-client-linux* \
echo -e "\n\n${BLUE}Deleting VMs:${NC}"
kubectl delete -n okd-virt vm --all --ignore-not-found 2>/dev/null
kubectl delete -n okd-virt virtualmachineinstance --all --ignore-not-found 2>/dev/null
kubectl delete -n okd-virt datavolume --all --ignore-not-found 2>/dev/null
kubectl delete -f "${HOMELAB}/sandbox/kubevirt/okd/vm" --ignore-not-found 2>/dev/null
}
delete_okd_bm() {
export HOMELAB="${PWD}"
export OKD=/tmp/okd
echo -e "\n\n${BLUE}Delete All Existing Data:${NC}"
rm -rf \
"${OKD}/oc" \
"${OKD}/kubectl" \
"${OKD}/openshift-install" \
"${OKD}/okd"
# "${HOME}/.cache/agent" \
# ${OKD}/openshift-install-linux* \
# ${OKD}/openshift-client-linux* \
}
okd_bm_worker_shutdown() {
export KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/okd}"
for WORKER in worker-1 worker-2 worker-3; do
echo -e "\n${BLUE}Scaling down ${WORKER}:${NC}"
kubectl patch machineset "okd-4ww5p-${WORKER}" -n openshift-machine-api --type='merge' -p '{"spec":{"replicas":0}}'
done
}
okd_bm_worker_startup() {
export KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/okd}"
for WORKER in worker-1 worker-2 worker-3; do
echo -e "\n${BLUE}Scaling up ${WORKER}:${NC}"
kubectl patch machineset "okd-4ww5p-${WORKER}" -n openshift-machine-api --type='merge' -p '{"spec":{"replicas":1}}'
done
}
generate_node_iso() {
local arg="${1:-}"
local HOMELAB="${HOMELAB:-${PWD}}"
if [[ -z "${arg}" || "${arg}" == "all" ]]; then
local yaml_files=("${HOMELAB}"/okd/node-config/*.yaml)
elif [[ "${arg}" == "workers" ]]; then
local yaml_files=("${HOMELAB}"/okd/node-config/worker-*.yaml)
elif [[ -f "${arg}" ]]; then
local yaml_files=("${arg}")
else
echo "Error: Invalid argument '${arg}'. Usage: generate_node_iso <path> | workers | all" >&2
return 1
fi
if [[ ${#yaml_files[@]} -eq 0 ]]; then
echo "Error: No YAML files found for '${arg}'" >&2
return 1
fi
local dest_dir="${HOMELAB}/notes/iso"
mkdir -p "${dest_dir}"
export KUBECONFIG=/home/arthur/.kube/okd
for node_yaml in "${yaml_files[@]}"; do
if [[ ! -f "${node_yaml}" ]]; then
continue
fi
echo -e "\n${BLUE}Generating node ISO from: ${node_yaml}${NC}"
local node_name=""
node_name=$(grep -m1 'hostname:' "${node_yaml}" | sed -E 's/.*hostname:[[:space:]]*//; s/[[:space:]]*$//')
if [[ -z "${node_name}" ]]; then
echo " WARNING: Could not extract hostname from '${node_yaml}', skipping" >&2
continue
fi
echo " Node name: ${node_name}"
local node_ip=""
node_ip=$(grep -m1 'ip:' "${node_yaml}" | sed -E 's/.*ip:[[:space:]]*//; s/[[:space:]]*$//')
if [[ -z "${node_ip}" ]]; then
echo " WARNING: Could not extract IP from '${node_yaml}', scp output will be incomplete" >&2
fi
local tmp_dir
tmp_dir=$(mktemp -d)
cp "${node_yaml}" "${tmp_dir}/nodes-config.yaml"
echo " Node IP: ${node_ip:-?}"
if ! oc adm node-image create --dir="${tmp_dir}" -o="${node_name}.iso"; then
echo "Error: oc adm node-image create failed for ${node_yaml}" >&2
rm -rf "${tmp_dir}"
continue
fi
mv "${tmp_dir}/${node_name}.iso" "${dest_dir}/${node_name}.iso"
rm -rf "${tmp_dir}"
echo -e "${BLUE}ISO generated: ${dest_dir}/${node_name}.iso${NC}"
# Print scp command
if [[ -n "${node_ip}" ]]; then
echo " scp arthur@10.101.11.7:/home/arthur/Projects/HomeLab/notes/iso/${node_name}.iso ./"
fi
done
# Bulk pull command — run from your laptop
echo ""
echo " # Pull entire ISO folder from this machine:"
echo " scp -r arthur@10.101.11.7:/home/arthur/Projects/HomeLab/notes/iso/ ./"
}
# ==============================================================================
# MicroShift BootC Image Pull
# ==============================================================================
# Builds the microshift BootC image on the VM and outputs the scp command to
# pull the image tarball to the local laptop. Follows the same pattern as
# generate_node_iso but outputs image pull commands instead.
#
# Usage:
# main.bash microshift-bootc-pull # Pull from default VM (10.101.11.7)
#
# This function:
# 1. Builds the image (make image BUILD_MODE=release)
# 2. Saves the image to a tarball
# 3. Outputs the scp command to pull the tarball to your laptop
# ==============================================================================
microshift-bootc-pull() {
local arg="${1:-}"
local HOMELAB="${HOMELAB:-${PWD}}"
local BUILD_MODE="${BUILD_MODE:-release}"
local IMAGE_TAG
# IMAGE_NAME used below for podman tag / save
# Get version from version.txt if available
local CLONE_DIR="/tmp/microshift-main"
if [[ -f "${CLONE_DIR}/_output/rpmbuild/RPMS/version.txt" ]]; then
IMAGE_TAG=$(tr '-' '_' < "${CLONE_DIR}/_output/rpmbuild/RPMS/version.txt")
else
IMAGE_TAG="latest"
fi
echo -e "\n${BLUE}MicroShift BootC Pull Instructions:${NC}"
echo " BUILD_MODE=${BUILD_MODE}"
echo " IMAGE_TAG=${IMAGE_TAG}"
echo ""
echo " # Build image on VM (if not already built):"
echo " ssh arthur@10.101.11.7 'cd /home/arthur/Projects/HomeLab/containers/microshift-bootc && make image BUILD_MODE=${BUILD_MODE}'"
echo ""
echo " # Tag and save image on VM:"
echo " ssh arthur@10.101.11.7 'cd /home/arthur/Projects/HomeLab/containers/microshift-bootc && podman tag localhost/microshift-bootc:latest localhost/microshift-bootc:${IMAGE_TAG}'"
echo " ssh arthur@10.101.11.7 'cd /home/arthur/Projects/HomeLab/containers/microshift-bootc && podman save -o microshift-bootc-${IMAGE_TAG}.tar localhost/microshift-bootc:${IMAGE_TAG}'"
echo ""
echo " # Pull image tarball to your laptop:"
echo " scp arthur@10.101.11.7:/home/arthur/Projects/HomeLab/containers/microshift-bootc/microshift-bootc-${IMAGE_TAG}.tar ."
echo " podman load -i microshift-bootc-${IMAGE_TAG}.tar"
echo ""
echo " # Cleanup on VM:"
echo " ssh arthur@10.101.11.7 'rm -f /home/arthur/Projects/HomeLab/containers/microshift-bootc/microshift-bootc-${IMAGE_TAG}.tar'"
echo ""
echo " # Or pull just the ISO (recommended):"
echo " scp arthur@10.101.11.7:/home/arthur/Projects/HomeLab/containers/microshift-bootc/output/microshift.iso ./"
}
install_okd_prep() {
sudo mount -o remount,size=8G,noatime /tmp
sudo setquota -u "$(whoami)" 0 0 0 0 /tmp || true
export XDG_CACHE_HOME="/tmp/cache"
mkdir -p "${XDG_CACHE_HOME}"
export AVP_TYPE=vault
export AVP_AUTH_TYPE=token
if [[ -z "${VAULT_TOKEN}" ]]; then
export VAULT_TOKEN
VAULT_TOKEN=$(vault login --tls-skip-verify -address="${URL}" -method=userpass -token-only username=arthur) || { echo "Failed to login to vault" >&2; return 1; }
fi
if [[ -z "${URL}" ]]; then
echo -n URL:
read -r -s URL
echo ""
fi
echo "${URL}"
echo -e "\n\n${BLUE}Get Registry URL:${NC}"
export REGISTRY
REGISTRY=${REGISTRY:-registry.arthurvardevanyan.com}
if [[ -z "${REGISTRY}" ]]; then
echo -n REGISTRY:
read -r -s REGISTRY
echo ""
fi
echo "${REGISTRY}"
echo -e "\n\n${BLUE}Download Dependencies:${NC}"
export OKD_VERSION=${OKD_VERSION:-""}
# Auto-detect OCP vs OKD: OKD versions always contain "okd"
if [[ -n "${OKD_VERSION}" ]] && [[ "${OKD_VERSION}" != *"okd"* ]]; then
OCP=true
fi
export OCP=${OCP:-false}
if [[ "${OCP}" == "true" ]]; then
# OCP Official Releases: https://amd64.ocp.releases.ci.openshift.org/
# Channels: 4-stable, 4-dev-preview
export OCP_CHANNEL=${OCP_CHANNEL:-4-stable}
if [[ -z "${OKD_VERSION}" ]]; then
OKD_VERSION=$(curl -s "https://amd64.ocp.releases.ci.openshift.org/api/v1/releasestream/${OCP_CHANNEL}/latest" | jq -r '.name')
fi
echo "${OKD_VERSION}"
export OKD_URL="quay.io/openshift-release-dev/ocp-release:${OKD_VERSION}-x86_64"
else
# OKD Releases: https://amd64.origin.releases.ci.openshift.org/
if [[ -z "${OKD_VERSION}" ]]; then
export OKD_CHANNEL=${OKD_CHANNEL:-5-scos-stable}
# https://github.com/JaimeMagiera/oct/blob/3968059ca79d9b60245aaff659533f6090c9a722/helpers/okd-query-releases.sh#L137
OKD_VERSION=$(curl -s "https://amd64.origin.releases.ci.openshift.org/releasestream/${OKD_CHANNEL}" | grep "Accepted" -B 1 | awk 'sub(/.*release\/ */,""){f=1} f{if ( sub(/ *".*/,"") ) f=0; print}' | head -n 1)
fi
echo "${OKD_VERSION}"
if [[ ${OKD_VERSION} == *"scos"* ]]; then
export OKD_URL="quay.io/okd/scos-release:${OKD_VERSION}"
export OKD_URL_CI="registry.ci.openshift.org/origin/release-scos:${OKD_VERSION}"
else
export OKD_URL="quay.io/openshift/okd:${OKD_VERSION}"
export OKD_URL_CI="registry.ci.openshift.org/origin/release:${OKD_VERSION}"
fi
fi
mkdir -p "${OKD}"
if [[ ! -f "${OKD}/openshift-install-linux-${OKD_VERSION}.tar.gz" ]]; then
echo "Required tools not found. Downloading..."
# Clean up tarballs and binaries from any previous version
find "${OKD}" -maxdepth 1 -name 'openshift-install-linux-4*.tar.gz' -delete
find "${OKD}" -maxdepth 1 -name 'openshift-client-linux-4*.tar.gz' -delete
rm -f "${OKD}/oc" "${OKD}/openshift-install" "${OKD}/kubectl"
if ! oc adm release extract --tools "${OKD_URL}" --to="${OKD}/"; then
if [[ "${OCP}" != "true" ]]; then
echo -e "${BLUE}Trying CI Registry:${NC}"
oc adm release extract --tools "${OKD_URL_CI}" --to="${OKD}/"
else
echo "ERROR: Failed to extract tools from ${OKD_URL}" >&2
exit 1
fi
fi
else
echo "Tools already present. Skipping download."
fi
tar xvzf "${OKD}/openshift-install-linux-${OKD_VERSION}.tar.gz" -C "${OKD}"
tar xvzf "${OKD}/openshift-client-linux-${OKD_VERSION}.tar.gz" -C "${OKD}"
mkdir -p "${OKD}/vm"
echo -e "\n\n${BLUE}Create Config Files:${NC}"
# Create okd directory of openshift-install files
mkdir -p "${OKD}"/okd
# Copy the install-config.yaml
export INSTALL_CONFIG_DIR=${INSTALL_CONFIG_DIR:-"${HOMELAB}/okd/install-config.yaml"}
cp "${INSTALL_CONFIG_DIR}" "${OKD}/okd/"
PULL_SECRET=$(vault kv get -field=auth secret/docker | jq -c)
SSH=$(cat "${HOME}"/.ssh/id_ed25519.pub)
sed -i "s/<SSH>/${SSH}/g" "${OKD}/okd/install-config.yaml"
sed -i "s/<URL>/${URL}/g" "${OKD}/okd/install-config.yaml"
sed -i "s/<REGISTRY>/${REGISTRY}/g" "${OKD}/okd/install-config.yaml"
sed -i "s/<CONTROL_PLANE>/${TF_VAR_control_plane_count:-${CONTROL_PLANE_COUNT}}/g" "${OKD}/okd/install-config.yaml"
sed -i "s/<WORKERS>/${TF_VAR_worker_count:-${WORKER_COUNT}}/g" "${OKD}/okd/install-config.yaml"
local pull_secret=""
pull_secret=$(printf '%s\n' "${PULL_SECRET}" | tr -d '\n')
PULL_SECRET="${pull_secret}" python3 -c "
import json, os, sys
path = sys.argv[1]
with open(path) as f:
content = f.read()
content = content.replace('pullSecret: |\n <PULL_SECRET>', 'pullSecret: ' + json.dumps(os.environ['PULL_SECRET']), 1)
with open(path, 'w') as f:
f.write(content)
" "${OKD}/okd/install-config.yaml"
cp "${OKD}/okd/install-config.yaml" "${OKD}/okd/install-config_backup.yaml"
# export OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE=${OPENSHIFT_INSTALL_OS_IMAGE_OVERRIDE:-"https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/pre-release/latest/rhcos-live-iso.x86_64.iso"}
}
install_okd_virt() {
delete_okd_virt
export HOMELAB="${PWD}"
export OKD=/tmp/okd
export CONTROL_PLANE_COUNT
export WORKER_COUNT
CONTROL_PLANE_COUNT=${CONTROL_PLANE_COUNT:-3}
WORKER_COUNT=${WORKER_COUNT:-0}
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-virt.arthurvardevanyan.com}
export INSTALL_CONFIG_DIR="${HOMELAB}/sandbox/kubevirt/okd/configs/install-config.yaml"
install_okd_prep
cp "${HOMELAB}/sandbox/kubevirt/okd/configs/agent-config.yaml" "${OKD}/okd/"
cp "${HOMELAB}/sandbox/kubevirt/okd/configs/containerfile" "${OKD}/okd/containerfile"
"${OKD}/openshift-install" agent create cluster-manifests --dir "${OKD}/okd/"
mkdir -p "${OKD}/okd/openshift"
cp "${HOMELAB}/sandbox/kubevirt/okd/configs/ovn.yaml" "${OKD}/okd/openshift/cluster-network-03-config.yml"
cp "${HOMELAB}/sandbox/kubevirt/okd/configs/boot-disk.yaml" "${OKD}/okd/openshift/boot-disk.yaml"
cp "${HOMELAB}/sandbox/kubevirt/okd/configs/secondary-disk.yaml" "${OKD}/okd/openshift/secondary-disk.yaml"
"${OKD}/openshift-install" agent create image --dir "${OKD}/okd/"
podman build -f "${OKD}/okd/containerfile" -t "${REGISTRY}/homelab/okd-virt:latest"
podman push "${REGISTRY}/homelab/okd-virt:latest"
podman rmi -f "${REGISTRY}/homelab/okd-virt:latest"
kubectl apply -f "${HOMELAB}/sandbox/kubevirt/okd/vm"
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
kubectl config set-cluster okd --insecure-skip-tls-verify=true
"${OKD}/openshift-install" agent wait-for bootstrap-complete --dir "${OKD}/okd/"
# "${OKD}/oc" apply -f "${HOMELAB}/okd/okd-configuration/overlays/sandbox/ingress-controller.yaml"
wait_for_install_complete "${OKD}/okd/"
kubectl label node sandbox-1 topology.kubernetes.io/zone=1 --overwrite
kubectl label node sandbox-2 topology.kubernetes.io/zone=2 --overwrite
kubectl label node sandbox-3 topology.kubernetes.io/zone=3 --overwrite
}
install_okd_sno() {
delete_okd_bm
export HOMELAB="${PWD}"
export OKD=/tmp/okd
export CONTROL_PLANE_COUNT
export WORKER_COUNT
CONTROL_PLANE_COUNT=${CONTROL_PLANE_COUNT:-1}
WORKER_COUNT=${WORKER_COUNT:-0}
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-sno.arthurvardevanyan.com}
install_okd_prep
cp "${HOMELAB}/sandbox/sno/agent-config.yaml" "${OKD}/okd/"
"${OKD}/openshift-install" agent create cluster-manifests --dir "${OKD}/okd/"
"${OKD}/openshift-install" agent create image --dir "${OKD}/okd/"
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
"${OKD}/openshift-install" agent wait-for bootstrap-complete --dir "${OKD}/okd/"
# "${OKD}/oc" apply -f "${HOMELAB}/okd/okd-configuration/overlays/sandbox/ingress-controller.yaml"
wait_for_install_complete "${OKD}/okd/"
}
install_okd_bm() {
delete_okd_bm
export HOMELAB="${PWD}"
export OKD=/tmp/okd
export CONTROL_PLANE_COUNT
export WORKER_COUNT
CONTROL_PLANE_COUNT=${CONTROL_PLANE_COUNT:-3}
WORKER_COUNT=${WORKER_COUNT:-0}
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-homelab.arthurvardevanyan.com}
install_okd_prep
cp "${HOMELAB}/okd/agent-config.yaml" "${OKD}/okd/"
"${OKD}/openshift-install" agent create cluster-manifests --dir "${OKD}/okd/"
mkdir -p "${OKD}/okd/openshift"
cp "${HOMELAB}/okd/ovn.yaml" "${OKD}/okd/openshift/cluster-network-03-config.yml"
cp "${HOMELAB}/okd/disks/boot-disk.yaml" "${OKD}/okd/openshift/boot-disk.yaml"
cp "${HOMELAB}/okd/disks/secondary-disk.yaml" "${OKD}/okd/openshift/secondary-disk.yaml"
"${OKD}/openshift-install" agent create image --dir "${OKD}/okd/"
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
"${OKD}/openshift-install" agent wait-for bootstrap-complete --dir "${OKD}/okd/"
# "${OKD}/oc" apply -f "${HOMELAB}/okd/okd-configuration/overlays/sandbox/ingress-controller.yaml"
wait_for_install_complete "${OKD}/okd/"
kubectl label node server-1 topology.kubernetes.io/zone=1 --overwrite
kubectl label node server-2 topology.kubernetes.io/zone=2 --overwrite
kubectl label node server-3 topology.kubernetes.io/zone=3 --overwrite
kubectl label node worker-1 topology.kubernetes.io/zone=1 --overwrite
kubectl label node worker-2 topology.kubernetes.io/zone=2 --overwrite
kubectl label node worker-3 topology.kubernetes.io/zone=3 --overwrite
}
delete_okd() {
export HOME=/home/arthur
export HOMELAB="${PWD}"
export OKD=/mnt/storage/okd
echo -e "\n\n${BLUE}Delete OKD Install:${NC}"
echo -e "\n\n${BLUE}Delete Bootstrap:${NC}"
cd "${HOMELAB}/terraform/sandbox/cluster"
terraform init
export TF_VAR_base_volume_id
TF_VAR_base_volume_id=$(terraform output -raw base_volume_id)
export TF_VAR_pool
TF_VAR_pool=$(terraform output -raw pool)
cd "${HOMELAB}/terraform/sandbox/bootstrap"
terraform init
terraform destroy -auto-approve
echo -e "\n\n${BLUE}Delete Workers:${NC}"
cd "${HOMELAB}/terraform/sandbox/workers"
terraform init
terraform destroy -auto-approve
echo -e "\n\n${BLUE}Delete libvirt and Masters:${NC}"
cd "${HOMELAB}/terraform/sandbox/cluster"
terraform destroy -auto-approve
if swapon --show | grep -q "${SWAP_PATH}"; then
swapoff /home/swapfile.img
fi
echo -e "\n\n${BLUE}Delete All Existing Data:${NC}"
rm -rf \
"${OKD}"/openshift-install-linux* \
"${OKD}"/openshift-client-linux* \
"${OKD}"/oc \
"${OKD}"/kubectl \
"${OKD}"/openshift-install \
"${OKD}"/fedora-coreos-* \
"${OKD}"/*.qcow2 \
"${OKD}"/okd \
"${OKD}"
cd "${HOMELAB}"
}
delete_okd_agent() {
export HOME=/home/arthur
export HOMELAB="${PWD}"
export OKD=/mnt/storage/okd
# TODO, Automate Finding Which IPs to Remove
# TODO Stop Running as Sudo
sudo -H -u arthur ssh-keygen -R 10.10.10.10
sudo -H -u arthur ssh-keygen -R 10.10.10.11
sudo -H -u arthur ssh-keygen -R 10.10.10.12
echo -e "\n\n${BLUE}Delete Installation:${NC}"
cd "${HOMELAB}/terraform/agent"
terraform destroy -auto-approve
echo -e "\n\n${BLUE}Delete All Existing Data:${NC}"
rm -rf \
"${OKD}"/openshift-install-linux* \
"${OKD}"/openshift-client-linux* \
"${OKD}"/oc \
"${OKD}"/kubectl \
"${OKD}"/openshift-install \
"${OKD}"/fedora-coreos-* \
"${OKD}"/*.qcow2 \
"${OKD}"/okd
if swapon --show | grep -q "${SWAP_PATH}"; then
swapoff /home/swapfile.img
fi
cd "${HOMELAB}"
}
install_okd_libvirt_prep() {
# https://blog.maumene.org/2020/11/18/OKD-or-OpenShit-in-one-box.html
# ssh -i ~/.ssh/id_ed25519 core@10.10.10.10
# journalctl -b -f -u kubelet.service -u ciro.service
# TODO: Fix Permissions:
# sudo vi /etc/libvirt/qemu.conf
# user = "root"
# group = "root"
# sudo systemctl restart libvirtd
export HOMELAB="${PWD}"
export HOME=/home/arthur
export OKD=/mnt/storage/okd
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
export PREFIX=""
export TF_VAR_control_plane_count=3
# Expand Swap Size on Host Computer
if ! test -f "${SWAP_PATH}"; then
dd if=/dev/zero of="${SWAP_PATH}" bs=45056 count=1M
mkswap "${SWAP_PATH}"
fi
if ! (swapon --show | grep -q "${SWAP_PATH}"); then
swapon "${SWAP_PATH}"
fi
systemctl start haproxy
systemctl stop firewalld
}
install_okd() {
# Delete Cluster If Exists
delete_okd
export TF_VAR_worker_count=4
install_okd_libvirt_prep
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-sandbox.arthurvardevanyan.com}
install_okd_prep
# Create the ignition files
"${OKD}/openshift-install" create ignition-configs --dir="${OKD}/okd"
chown -R arthur:arthur "${OKD}"
chmod 777 -R "${OKD}"
echo -e "\n\n${BLUE}Start OKD Install:${NC}"
echo -e "\n\n${BLUE}Setup libvirt and Masters:${NC}"
cd "${HOMELAB}/terraform/sandbox/cluster"
mkdir -p "${OKD}/terraform/cluster"
terraform init
terraform apply -auto-approve
export TF_VAR_base_volume_id
TF_VAR_base_volume_id=$(terraform output -raw base_volume_id)
export TF_VAR_pool
TF_VAR_pool=$(terraform output -raw pool)
echo -e "\n\n${BLUE}Initialize Bootstrap:${NC}"
cd "${HOMELAB}/terraform/sandbox/bootstrap"
mkdir -p "${OKD}/terraform/bootstrap"
terraform init
terraform apply -auto-approve
echo -e "\n\n${BLUE}Waiting For Bootstrap:${NC}"
"${OKD}/openshift-install" --dir="${OKD}/okd" wait-for bootstrap-complete --log-level debug
echo -e "\n\n${BLUE}Destroy Bootstrap:${NC}"
cd "${HOMELAB}/terraform/sandbox/bootstrap"
mkdir -p "${OKD}/terraform/bootstrap"
terraform destroy -auto-approve
echo -e "\n\n${BLUE}Initialize Workers:${NC}"
cd "${HOMELAB}/terraform/sandbox/workers"
mkdir -p "${OKD}/terraform/workers"
terraform init
terraform apply -auto-approve
echo -e "\n\n${BLUE}Wait for Install To Complete:${NC}"
#${OKD}/oc apply -f "${HOMELAB}/okd/okd-configuration/overlays/sandbox/ingress-controller.yaml"
wait_for_install_complete "${OKD}/okd/"
"${OKD}/oc" apply -f "${HOMELAB}/okd/okd-configuration/base/operator-hub.yaml"
"${OKD}/oc" apply -f "${HOMELAB}/okd/okd-configuration/base/operators"
echo -e "\n\n${BLUE}Setup Image Mirroring:${NC}"
#sed 's/AllowContactingSource/NeverContactSource/' "${HOMELAB}"/okd/okd-configuration/base/image-mirror-set.yaml | kubectl apply -f -
kubectl apply -f "${HOMELAB}"/okd/okd-configuration/base/image-mirror-set.yaml
echo -e "\n\n${BLUE}Install Complete:${NC}"
echo "export KUBECONFIG=${OKD}/okd/auth/kubeconfig"
local kadmin_pass=""
kadmin_pass=$(cat "${OKD}/okd/auth/kubeadmin-password") || { echo "Failed to read kubeadmin password" >&2; return 1; }
echo "Kubeadmin Password: ${kadmin_pass}"
#single_server
}
install_okd_agent() {
# Delete Cluster If Exists
delete_okd_agent
export TF_VAR_worker_count=0
install_okd_libvirt_prep
echo -e "\n\n${BLUE}Get URL:${NC}"
export URL
URL=${URL:-sandbox.arthurvardevanyan.com}
install_okd_prep
cp "${HOMELAB}/sandbox/libvirt/configs/agent-config.yaml" "${OKD}/okd/"
"${OKD}/openshift-install" agent create cluster-manifests --dir "${OKD}/okd/"
"${OKD}/openshift-install" agent create image --dir "${OKD}/okd/"
chown -R arthur:arthur "${OKD}"
chmod 777 -R "${OKD}"
echo -e "\n\n${BLUE}Start OKD Install:${NC}"
echo -e "\n\n${BLUE}Setup libvirt and Masters:${NC}"
cd "${HOMELAB}/terraform/agent"
mkdir -p "${OKD}/terraform/agent"
terraform init
terraform apply -auto-approve
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
"${OKD}/openshift-install" agent wait-for bootstrap-complete --dir "${OKD}/okd/"
# "${OKD}/oc" apply -f "${HOMELAB}/okd/okd-configuration/overlays/sandbox/ingress-controller.yaml"
wait_for_install_complete "${OKD}/okd/"
}
single_server() {
echo -e "\n\n${BLUE}Single Server Resource Adjustments:${NC}"
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
"${OKD}/oc" scale --replicas=1 ingresscontroller/default -n openshift-ingress-operator
"${OKD}/oc" scale --replicas=1 deployment.apps/console -n openshift-console
"${OKD}/oc" scale --replicas=1 deployment.apps/downloads -n openshift-console
"${OKD}/oc" scale --replicas=1 deployment.apps/oauth-openshift -n openshift-authentication
"${OKD}/oc" scale --replicas=1 deployment.apps/packageserver -n openshift-operator-lifecycle-manager
"${OKD}/oc" scale --replicas=1 deployment.apps/prometheus-adapter -n openshift-monitoring
"${OKD}/oc" scale --replicas=1 deployment.apps/thanos-querier -n openshift-monitoring
"${OKD}/oc" scale --replicas=1 statefulset.apps/prometheus-k8s -n openshift-monitoring
"${OKD}/oc" scale --replicas=1 statefulset.apps/alertmanager-main -n openshift-monitoring
}
install_addons_okd_virt() {
# Tweak Settings from Physical HomeLab
export OKD=/tmp/okd
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
export HOMELAB="${PWD}"
kubectl config set-cluster okd --insecure-skip-tls-verify=true
echo -e "\n${BLUE}Installing Cluster Addons:${NC}"
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-virt.arthurvardevanyan.com}
if [[ -z "${URL}" ]]; then
echo -n URL:
read -r -s URL
echo ""
fi
echo "${URL}"
export AVP_TYPE=vault
export AVP_AUTH_TYPE=token
if [[ -z "${VAULT_TOKEN}" ]]; then
export VAULT_TOKEN
VAULT_TOKEN=$(vault login --tls-skip-verify -address="${URL}" -method=userpass -token-only username=arthur) || { echo "Failed to login to vault" >&2; return 1; }
fi
kubectl config set-cluster okd --insecure-skip-tls-verify=true
kubectl label node sandbox-1 topology.kubernetes.io/zone=1 --overwrite
kubectl label node sandbox-2 topology.kubernetes.io/zone=2 --overwrite
kubectl label node sandbox-3 topology.kubernetes.io/zone=3 --overwrite
echo -e "\n${BLUE}Pull Secret:${NC}"
argocd-vault-plugin generate "${HOMELAB}"/okd/okd-configuration/base/pull-secret.yaml | kubectl apply -f -
sleep 30s
local mcp_master_status=""
mcp_master_status=$(kubectl get mcp master -o yaml | yq '.status.conditions[] | select(.type == "Updating") | .status') || { echo "Failed to get MCP master status" >&2; return 1; }
while [[ "${mcp_master_status}" == "True" ]]; do
echo "Waiting for MCPs"
sleep 30s
done
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/master
echo -e "\n${BLUE}OperatorHub:${NC}"
kubectl apply -f "${HOMELAB}"/okd/okd-configuration/base/operator-hub.yaml
sleep 30s
echo -e "\n${BLUE}Waiting for MarketPlace Pods to Boot:${NC}"
local mp_count=0
mp_count=$(kubectl get pods -n openshift-marketplace | grep -v "Completed" | grep -cv Running) || { echo "Failed to get Marketplace pods count" >&2; return 1; }
while [[ "${mp_count}" -ne 1 ]]; do
sleep 1
done
echo -e "\n${BLUE}Cert Manager:${NC}"
local _cm
_cm=0
while [[ ${_cm} -lt 5 ]]; do
if kubectl kustomize "${HOMELAB}"/kubernetes/cert-manager/overlays/okd-sandbox | argocd-vault-plugin generate - | kubectl apply -f -; then
break
fi
_cm=$((_cm + 1))
sleep $((5 ** _cm))
done
echo -e "\n${BLUE}Nmstate:${NC}"
kubectl create ns openshift-nmstate
kubectl kustomize "${HOMELAB}"/kubernetes/nmstate/overlays/sandbox | kubectl apply -f -
echo -e "\n${BLUE}OKD Configuration:${NC}"
kubectl kustomize "${HOMELAB}"/okd/okd-configuration/overlays/sandbox | argocd-vault-plugin generate - | kubectl apply -f -
kubectl kustomize kubernetes/nmstate/overlays/sandbox | kubectl apply -f -
echo -e "\n${BLUE}Waiting for API Certificate to Generate:${NC}"
local api_cert_ready=""
api_cert_ready=$(kubectl get certificate -n openshift-config api-certificate -o yaml | yq '.status.conditions[] | select(.type == "Ready") | .status') || { echo "Failed to get API certificate ready status" >&2; return 1; }
while [[ "${api_cert_ready}" == "False" ]]; do
echo "Waiting for API Certificate to Generate"
sleep 30
done
echo -e "\n${BLUE}Waiting for Ingress Certificate to Generate:${NC}"
local ingress_cert_ready=""
ingress_cert_ready=$(kubectl get certificate -n openshift-ingress ingress-certificate -o yaml | yq '.status.conditions[] | select(.type == "Ready") | .status') || { echo "Failed to get Ingress certificate ready status" >&2; return 1; }
while [[ "${ingress_cert_ready}" == "False" ]]; do
echo "Waiting for Ingress Certificate to Generate"
sleep 30
done
sleep 120
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/worker
echo -e "\n\n${BLUE}Addons Installed!${NC}"
}
install_addons_okd() {
# Tweak Settings from Physical HomeLab
export OKD=/mnt/storage/okd
export KUBECONFIG="${OKD}/okd/auth/kubeconfig"
export HOMELAB="${PWD}"
echo -e "\n${BLUE}Installing Cluster Addons:${NC}"
echo -e "\n\n${BLUE}Get URL:${NC}"
URL=${URL:-arthurvardevanyan.com}
if [[ -z "${URL}" ]]; then
echo -n URL:
read -r -s URL
echo ""
fi
echo "${URL}"
export AVP_TYPE=vault
export AVP_AUTH_TYPE=token
if [[ -z "${VAULT_TOKEN}" ]]; then
export VAULT_TOKEN
VAULT_TOKEN=$(vault login --tls-skip-verify -address="${URL}" -method=userpass -token-only username=arthur)
fi
sleep 15s
local mcp_worker_status=""
mcp_worker_status=$(kubectl get mcp worker -o yaml | yq '.status.conditions[] | select(.type == "Updating") | .status') || { echo "Failed to get MCP worker status" >&2; return 1; }
while [[ "${mcp_worker_status}" == "True" ]]; do
echo "Waiting for MCPs"
sleep 30s
done
oc debug node/worker-0 -t -- chroot /host sudo mkfs.ext4 -L longhorn /dev/vdb
oc debug node/worker-1 -t -- chroot /host sudo mkfs.ext4 -L longhorn /dev/vdb
oc debug node/worker-2 -t -- chroot /host sudo mkfs.ext4 -L longhorn /dev/vdb
oc debug node/worker-3 -t -- chroot /host sudo mkfs.ext4 -L longhorn /dev/vdb
kubectl label node worker-0 topology.kubernetes.io/zone="even" --overwrite
kubectl label node worker-1 topology.kubernetes.io/zone="odd" --overwrite
kubectl label node worker-2 topology.kubernetes.io/zone="even" --overwrite
kubectl label node worker-3 topology.kubernetes.io/zone="odd" --overwrite || { echo "Warning: kubectl label failed on worker-3" >&2; }
kubectl annotate node "worker-0" node.longhorn.io/default-disks-config='[{"path":"/var/mnt/longhorn","allowScheduling":true}]' --overwrite
kubectl annotate node "worker-1" node.longhorn.io/default-disks-config='[{"path":"/var/mnt/longhorn","allowScheduling":true}]' --overwrite
kubectl annotate node "worker-2" node.longhorn.io/default-disks-config='[{"path":"/var/mnt/longhorn","allowScheduling":true}]' --overwrite
kubectl annotate node "worker-3" node.longhorn.io/default-disks-config='[{"path":"/var/mnt/longhorn","allowScheduling":true}]' --overwrite || { echo "Warning: kubectl annotate failed on worker-3" >&2; }
kubectl label node "worker-0" node.longhorn.io/create-default-disk=config --overwrite
kubectl label node "worker-1" node.longhorn.io/create-default-disk=config --overwrite
kubectl label node "worker-2" node.longhorn.io/create-default-disk=config --overwrite
kubectl label node "worker-3" node.longhorn.io/create-default-disk=config --overwrite || { echo "Warning: kubectl label failed on worker-3" >&2; }
kubectl label node "worker-0" node-role.kubernetes.io/infra="" --overwrite
kubectl label node "worker-1" node-role.kubernetes.io/infra="" --overwrite
kubectl label node "worker-2" node-role.kubernetes.io/infra="" --overwrite
kubectl label node "worker-3" node-role.kubernetes.io/infra="" --overwrite || { echo "Warning: kubectl label failed on worker-3" >&2; }
kubectl apply -f "${HOMELAB}"/okd/okd-configuration/base/mcp.yaml
yq '.spec.config.systemd.units[1].enabled=false' "${HOMELAB}"/okd/okd-configuration/overlays/sandbox/longhorn-mc.yaml | kubectl apply -f -
sleep 15s
local mcp_infra_status=""
mcp_infra_status=$(kubectl get mcp infra -o yaml | yq '.status.conditions[] | select(.type == "Updating") | .status') || { echo "Failed to get MCP infra status" >&2; return 1; }
while [[ "${mcp_infra_status}" == "True" ]]; do
echo "Waiting for MCPs"
sleep 30s
done
echo -e "\n${BLUE}Longhorn:${NC}"
local _lh
_lh=0
while [[ ${_lh} -lt 5 ]]; do
if kubectl kustomize "${HOMELAB}"/kubernetes/longhorn/overlays/okd-sandbox | argocd-vault-plugin generate - | kubectl apply -f -; then
break
fi
_lh=$((_lh + 1))
sleep $((5 ** _lh))
done
echo -e "\n${BLUE}Waiting for Longhorn to Boot:${NC}"
local lh_count2=0
lh_count2=$(kubectl get pods -n longhorn-system | grep -cv Running) || { echo "Failed to get Longhorn pods count" >&2; return 1; }
while [[ "${lh_count2}" -ne 1 ]]; do
sleep 1
done
echo -e "\n${BLUE}OKD Monitoring:${NC}"
local _om
_om=0
while [[ ${_om} -lt 5 ]]; do
local _okm
_okm=$(kubectl kustomize "${HOMELAB}"/okd/openshift-monitoring/overlays/sandbox | argocd-vault-plugin generate - | kubectl apply -f -) || { echo "Failed to apply monitoring" >&2; return 1; }
if [[ "${_okm}" == "0" ]]; then
break
fi
_om=$((_om + 1))
sleep $((5 ** _om))
done
echo -e "\n${BLUE}Cert Manager:${NC}"
local _cm
_cm=0
while [[ ${_cm} -lt 5 ]]; do
if kubectl kustomize "${HOMELAB}"/kubernetes/cert-manager/overlays/okd-sandbox | argocd-vault-plugin generate - | kubectl apply -f -; then
break
fi
_cm=$((_cm + 1))
sleep $((5 ** _cm))
done
echo -e "\n${BLUE}OKD Configuration:${NC}"
local _oc
_oc=0
while [[ ${_oc} -lt 5 ]]; do
if kubectl kustomize "${HOMELAB}"/okd/okd-configuration/overlays/sandbox | argocd-vault-plugin generate - | kubectl apply -f -; then
break
fi
_oc=$((_oc + 1))
sleep $((5 ** _oc))
done
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/worker
oc patch --type=merge --patch='{"spec":{"paused":true}}' machineconfigpool/infra
echo -e "\n${BLUE}Waiting for API Certificate to Generate:${NC}"
local api_cert_status=""
api_cert_status=$(kubectl get certificate -n openshift-config api-certificate -o yaml | yq '.status.conditions[] | select(.type == "Ready") | .status') || { echo "Failed to get API certificate status" >&2; return 1; }
while [[ "${api_cert_status}" == "False" ]]; do
echo "Waiting for API Certificate to Generate"
sleep 30
done
echo -e "\n${BLUE}Waiting for Ingress Certificate to Generate:${NC}"
local ingress_cert_status=""
ingress_cert_status=$(kubectl get certificate -n openshift-ingress ingress-certificate -o yaml | yq '.status.conditions[] | select(.type == "Ready") | .status') || { echo "Failed to get Ingress certificate status" >&2; return 1; }
while [[ "${ingress_cert_status}" == "False" ]]; do
echo "Waiting for Ingress Certificate to Generate"
sleep 30
done
sleep 120
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/master
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/worker
oc patch --type=merge --patch='{"spec":{"paused":false}}' machineconfigpool/infra
echo -e "\n\n${BLUE}Addons Installed!${NC}"
}
approve_csr() {
echo -e "\n\n${BLUE}Approving CSRs:${NC}"
while true; do
for csr in $(oc get csr 2>/dev/null | grep -w 'Pending' | awk '{print $1}'); do
echo -n ' --> Approving CSR: '
oc adm certificate approve "${csr}" 2>/dev/null || { echo "Warning: CSR ${csr} not pending or already approved" >&2; }
done
sleep 5
done
}
update_install_plans() {
echo -e "\n${BLUE}Checking for pending InstallPlans:${NC}"
PENDING=$(kubectl get installplan -A -o jsonpath='{range .items[?(@.spec.approved==false)]}{.metadata.namespace}{"\t"}{.spec.clusterServiceVersionNames[0]}{"\n"}{end}')
if [[ -z "${PENDING}" ]]; then
echo "No pending InstallPlans found."
return 0
fi
while IFS=$'\t' read -r NAMESPACE CSV; do
[[ -z "${NAMESPACE}" ]] && continue
echo -e "\nPending: ${CSV} in ${NAMESPACE}"
# Find the subscription file matching this namespace
SUB_FILE=$(grep -rl "namespace: ${NAMESPACE}$" --include="subscription.yaml" kubernetes/ 2>/dev/null | head -1)
if [[ -z "${SUB_FILE}" ]]; then
echo " WARNING: No subscription file found for namespace '${NAMESPACE}'"
continue
fi
# Get current startingCSV from the file
CURRENT_CSV=$(grep 'startingCSV:' "${SUB_FILE}" | sed 's/.*startingCSV:[[:space:]]*//' | tr -d '"' | tr -d "'" | awk '{print $1}')
if [[ "${CURRENT_CSV}" == "${CSV}" ]]; then
echo " Already up to date: ${SUB_FILE}"
continue
fi
echo " Updating ${SUB_FILE}"
echo " ${CURRENT_CSV} -> ${CSV}"
sed -i "s|startingCSV:.*|startingCSV: ${CSV}|" "${SUB_FILE}"
done <<< "${PENDING}"
echo -e "\n${BLUE}Done updating subscription files!${NC}"
}
approve_install_plans() {
echo -e "\n${BLUE}Approving pending InstallPlans:${NC}"
PENDING=$(kubectl get installplan -A -o jsonpath='{range .items[?(@.spec.approved==false)]}{.metadata.namespace}{"\t"}{.metadata.name}{"\t"}{.spec.clusterServiceVersionNames[0]}{"\n"}{end}')
if [[ -z "${PENDING}" ]]; then
echo "No pending InstallPlans found."
return 0
fi
while IFS=$'\t' read -r NAMESPACE NAME CSV; do
[[ -z "${NAMESPACE}" ]] && continue
# Strip hidden characters (CR, whitespace)
NAMESPACE=$(printf '%s' "${NAMESPACE}" | tr -d '\r' | xargs)
CSV=$(printf '%s' "${CSV}" | tr -d '\r' | xargs)
echo " Approving ${NAME} (${CSV}) in ${NAMESPACE}"
kubectl patch installplan "${NAME}" -n "${NAMESPACE}" --type merge --patch '{"spec":{"approved":true}}'
# Find and update the subscription file matching this namespace
SUB_FILE=$(grep -rl "namespace: ${NAMESPACE}" --include="subscription.yaml" kubernetes/ 2>/dev/null | head -1)
if [[ -z "${SUB_FILE}" ]]; then
echo " WARNING: No subscription file found for namespace '${NAMESPACE}'"
continue
fi
# Get current startingCSV from the file
CURRENT_CSV=$(grep 'startingCSV:' "${SUB_FILE}" | sed 's/.*startingCSV:[[:space:]]*//' | tr -d '"' | tr -d "'" | tr -d '\r' | awk '{print $1}')
if [[ "${CURRENT_CSV}" == "${CSV}" ]]; then
echo " Already up to date: ${SUB_FILE} (${CSV})"
else
echo " Updating ${SUB_FILE}"
echo " ${CURRENT_CSV} -> ${CSV}"
sed -i "s|startingCSV:.*|startingCSV: ${CSV}|" "${SUB_FILE}"
fi
done <<< "${PENDING}"
echo -e "\n${BLUE}Done approving InstallPlans!${NC}"
}
"$@"