hack/e2e-prepare-cluster.bats
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bats
# -----------------------------------------------------------------------------
# Cozystack end‑to‑end provisioning test (Bats)
# -----------------------------------------------------------------------------
@test "Required installer assets exist" {
if [ ! -f _out/assets/nocloud-amd64.raw.xz ]; then
echo "Missing: _out/assets/nocloud-amd64.raw.xz" >&2
exit 1
fi
}
@test "IPv4 forwarding is enabled" {
if [ "$(cat /proc/sys/net/ipv4/ip_forward)" != 1 ]; then
echo "IPv4 forwarding is disabled!" >&2
echo >&2
echo "Enable it with:" >&2
echo " echo 1 > /proc/sys/net/ipv4/ip_forward" >&2
exit 1
fi
}
@test "Clean previous VMs" {
kill $(cat srv1/qemu.pid srv2/qemu.pid srv3/qemu.pid 2>/dev/null) 2>/dev/null || true
rm -rf srv1 srv2 srv3
}
@test "Prepare networking and masquerading" {
ip link del cozy-br0 2>/dev/null || true
ip link add cozy-br0 type bridge
ip link set cozy-br0 up
ip address add 192.168.123.1/24 dev cozy-br0
# Masquerading rule – idempotent (delete first, then add)
iptables -t nat -D POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE 2>/dev/null || true
iptables -t nat -A POSTROUTING -s 192.168.123.0/24 ! -d 192.168.123.0/24 -j MASQUERADE
}
@test "Prepare cloud‑init drive for VMs" {
mkdir -p srv1 srv2 srv3
# Generate cloud‑init ISOs
for i in 1 2 3; do
echo "hostname: srv${i}" > "srv${i}/meta-data"
cat > "srv${i}/user-data" <<'EOF'
#cloud-config
EOF
cat > "srv${i}/network-config" <<EOF
version: 2
ethernets:
eth0:
dhcp4: false
addresses:
- "192.168.123.1${i}/26"
gateway4: "192.168.123.1"
nameservers:
search: [cluster.local]
addresses: [8.8.8.8]
EOF
( cd "srv${i}" && genisoimage \
-output seed.img \
-volid cidata -rational-rock -joliet \
user-data meta-data network-config )
done
}
@test "Use Talos NoCloud image from assets" {
if [ ! -f _out/assets/nocloud-amd64.raw.xz ]; then
echo "Missing _out/assets/nocloud-amd64.raw.xz" 2>&1
exit 1
fi
rm -f nocloud-amd64.raw
cp _out/assets/nocloud-amd64.raw.xz .
xz --decompress nocloud-amd64.raw.xz
}
@test "Prepare VM disks" {
for i in 1 2 3; do
cp nocloud-amd64.raw srv${i}/system.img
qemu-img resize srv${i}/system.img 50G
qemu-img create srv${i}/data.img 200G
done
}
@test "Create tap devices" {
for i in 1 2 3; do
ip link del cozy-srv${i} 2>/dev/null || true
ip tuntap add dev cozy-srv${i} mode tap
ip link set cozy-srv${i} up
ip link set cozy-srv${i} master cozy-br0
done
}
@test "Boot QEMU VMs" {
# cache=unsafe drops guest flush pass-through on these ephemeral CI disks.
# Otherwise every management-etcd Raft-commit fsync propagates to the
# host-backed raw files, and under the concurrent image/CDI I/O storm etcd
# stalls for seconds, the apiserver returns context-deadline-exceeded, and
# operators lose their leader-election leases together. The VMs are recreated
# per run, so the durability we trade away here is never needed.
for i in 1 2 3; do
# `debug-threads=on` names the vCPU threads (`CPU N/KVM`) so the QEMU
# thread capture on the node-join failure path can tell a vCPU from an IO
# thread; without it every thread reports the bare process name. QEMU only
# renames threads when the flag is passed, so the capture's legend and its
# fixtures are pinned to this line carrying it.
qemu-system-x86_64 -machine type=pc,accel=kvm -cpu host -smp 8 -m 24576 \
-name guest=srv${i},debug-threads=on \
-device virtio-net,netdev=net0,mac=52:54:00:12:34:5${i} \
-netdev tap,id=net0,ifname=cozy-srv${i},script=no,downscript=no \
-drive file=srv${i}/system.img,if=virtio,format=raw,cache=unsafe \
-drive file=srv${i}/seed.img,if=virtio,format=raw,cache=unsafe \
-drive file=srv${i}/data.img,if=virtio,format=raw,cache=unsafe \
-display none -daemonize -pidfile srv${i}/qemu.pid
done
# Give qemu a few seconds to start up networking
sleep 5
}
@test "Wait until Talos API port 50000 is reachable on all machines" {
timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'
}
@test "Generate Talos cluster configuration" {
# Cluster‑wide patches
cat > patch.yaml <<'EOF'
machine:
kubelet:
nodeIP:
validSubnets:
- 192.168.123.0/24
extraConfig:
maxPods: 512
kernel:
modules:
- name: openvswitch
- name: drbd
parameters:
- usermode_helper=disabled
# Talos v1.14 kernels have an empty modprobe path
# (siderolabs/pkgs#1565), so DRBD's request_module() for its transport
# on the first new-peer no longer loads anything.
- name: drbd_transport_tcp
- name: zfs
- name: spl
registries:
mirrors:
docker.io:
endpoints:
- https://mirror.gcr.io
files:
- content: |
[plugins]
[plugins."io.containerd.cri.v1.runtime"]
device_ownership_from_security_context = true
path: /etc/cri/conf.d/20-customization.part
op: create
- op: overwrite
path: /etc/lvm/lvm.conf
permissions: 0o644
content: |
backup {
backup = 0
archive = 0
}
devices {
global_filter = [ "r|^/dev/drbd.*|", "r|^/dev/dm-.*|", "r|^/dev/zd.*|", "r|^/dev/loop.*|" ]
}
cluster:
apiServer:
extraArgs:
oidc-issuer-url: "https://keycloak.example.org/realms/cozy"
oidc-client-id: "kubernetes"
oidc-username-claim: "preferred_username"
oidc-groups-claim: "groups"
network:
cni:
name: none
dnsDomain: cozy.local
# Deliberately NOT the kube-ovn POD_CIDR (10.244.0.0/16, set as
# networking.podCIDR by the install step). kube-ovn owns pod IPAM and
# allocates flat from POD_CIDR; cilium is chained behind it and allocates
# no pod addresses, but it still carves its own per-node router and
# Ingress IPs out of Node.spec.podCIDR, which kube-controller-manager
# slices from this value. Overlap and kube-ovn eventually hands a pod an
# address cilium already holds. hack/sandbox-cidr-disjoint.bats guards it.
# Picking a replacement: 100.64.0.0/16 is the kube-ovn join range and
# 100.66.0.0/16 is kilo's transit range, so the free slot in this corner of
# RFC 6598 is the one below.
podSubnets:
- 100.65.0.0/16
serviceSubnets:
- 10.96.0.0/16
EOF
# Control‑plane‑only patches
cat > patch-controlplane.yaml <<'EOF'
machine:
nodeLabels:
node.kubernetes.io/exclude-from-external-load-balancers:
$patch: delete
network:
interfaces:
- interface: eth0
vip:
ip: 192.168.123.10
cluster:
allowSchedulingOnControlPlanes: true
controllerManager:
extraArgs:
bind-address: 0.0.0.0
scheduler:
extraArgs:
bind-address: 0.0.0.0
apiServer:
certSANs:
- 127.0.0.1
proxy:
disabled: true
discovery:
enabled: false
etcd:
advertisedSubnets:
- 192.168.123.0/24
EOF
# Generate secrets once
if [ ! -f secrets.yaml ]; then
talosctl gen secrets
fi
rm -f controlplane.yaml worker.yaml talosconfig kubeconfig
# Pin Kubernetes to v1.33.12. talosctl v1.13.5 now defaults to a much newer
# k8s (1.36.x) that is outside cozystack's supported range, so e2e pins back
# to the 1.33 line. v1.33.12 also carries the fix for the
# kube-controller-manager VAP status type-checker nil-pointer panic on
# `additionalProperties: true` schemas (kubernetes/kubernetes#135155,
# backported to release-1.33 in #136958, first released in v1.33.10); without
# it, cozystack-api's aggregated Tenant schema crash-loops KCM during install.
# See cozystack#2863.
talosctl gen config --with-secrets secrets.yaml cozystack https://192.168.123.10:6443 \
--kubernetes-version v1.33.12 \
--config-patch=@patch.yaml --config-patch-control-plane @patch-controlplane.yaml
}
@test "Apply Talos configuration to the node" {
# Apply the configuration to all three nodes
for node in 11 12 13; do
talosctl apply -f controlplane.yaml -n 192.168.123.${node} -e 192.168.123.${node} -i
done
# Wait for Talos services to come up again
timeout 60 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'
}
@test "Bootstrap Talos cluster" {
# Bootstrap etcd on the first node.
# Retry for up to 60s: talosctl bootstrap refuses with "time is not in
# sync yet" until NTP converges on a freshly booted node, which on fast
# ephemeral runners can take several attempts of ~5s each.
timeout 60 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 2; done'
# Wait until etcd is healthy.
# Budget is 5m, not 3m: a 3-node etcd converges via Talos's serialized
# learner promotion (etcd admits one learner at a time), so the third member
# is only promoted after the second is fully caught up. On a loaded ephemeral
# runner this deterministic path can legitimately approach ~3m wall clock —
# a 180s ceiling photo-finishes against it and times out a cluster that is
# actually healthy. The wait itself is already event-driven (poll until
# members respond); only the ceiling needed widening.
if ! timeout 300 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 >/dev/null 2>&1; do sleep 1; done'; then
talosctl dmesg -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 || true
exit 1
fi
timeout 60 sh -ec 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.10 2>&1 | grep -q "rpc error"; do sleep 1; done'
# Retrieve kubeconfig
rm -f kubeconfig
talosctl kubeconfig kubeconfig -e 192.168.123.10 -n 192.168.123.10
# Wait until all three nodes register in Kubernetes
timeout 60 sh -ec 'until [ $(kubectl get node --no-headers | wc -l) -eq 3 ]; do sleep 1; done'
}