hack/e2e-prepare-cluster-container.bats
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bats
# -----------------------------------------------------------------------------
# Cozystack E2E provisioning — CONTAINER lane
#
# The counterpart to hack/e2e-prepare-cluster.bats. That file boots three QEMU
# guests; this one takes over after hack/e2e-container-up.sh has started three
# Talos containers on the runner's docker daemon and attached this sandbox to
# their network. Everything from bootstrap onward is deliberately the same shape
# as the QEMU lane so the two stay comparable, and both hand off an identical
# cluster to hack/e2e-install-cozystack.bats.
#
# Why the split: containers cannot nest inside the sandbox the way QEMU guests
# do, so the nodes must be siblings on the host daemon for hack/e2e-compose.yaml's
# /dev and /run/udev binds to resolve against the real host devtmpfs. Only the
# host-side half needs a docker socket; this half needs only the network.
#
# Addressing, CPU and memory match the QEMU lane exactly, so the MetalLB
# 192.168.123.200-250 pool and hack/e2e-post-install-prep.sh are unchanged.
# The one deliberate difference is the API endpoint: container mode has no ARP
# VIP, so .11 stands in for the QEMU lane's 192.168.123.10.
# -----------------------------------------------------------------------------
HACK_DIR="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")" && pwd)"
E2E_CONTAINER_UP_LIB=true . "$HACK_DIR/e2e-container-up.sh"
@test "Talos API is reachable on all three container nodes" {
# e2e-container-up.sh has already started them; this is the handoff check, and
# it is a wait rather than a probe because container start is asynchronous.
if ! timeout 180 sh -ec 'until nc -nz 192.168.123.11 50000 && nc -nz 192.168.123.12 50000 && nc -nz 192.168.123.13 50000; do sleep 1; done'; then
echo "Talos API never came up on 192.168.123.11-13:50000." >&2
echo "The sandbox reaches the nodes over the compose network it was attached to;" >&2
echo "if this times out, check that hack/e2e-container-up.sh ran and that the" >&2
echo "sandbox is on the cozy-e2e_talos network." >&2
exit 1
fi
}
@test "Host kernel modules are usable from inside the nodes" {
# THE trap of this lane. machine.kernel.modules is a silent no-op in container
# mode -- kernel_module_spec.go returns early on ModeContainer with no error
# and no event -- so a lane that carried the QEMU patch over would look like it
# worked and fail much later as a CNI or storage error. e2e-container-up.sh
# loads these on the host; this asserts the nodes actually see them, which is
# the property that matters and is not implied by the host check alone.
for mod in openvswitch zfs; do
if ! talosctl read /proc/modules -n 192.168.123.11 -e 192.168.123.11 | grep -q "^${mod} "; then
echo "Kernel module '${mod}' is not visible from srv1." >&2
echo "Container nodes share the host kernel and Talos cannot load modules in" >&2
echo "container mode, so this must be loaded on the host before they start." >&2
exit 1
fi
done
}
@test "/dev/kvm is present inside the nodes" {
# KubeVirt's device plugin needs it, and its presence is what makes the tenant
# workers run under real KVM rather than TCG emulation.
if ! talosctl ls /dev/kvm -n 192.168.123.11 -e 192.168.123.11 >/dev/null 2>&1; then
echo "/dev/kvm is missing inside srv1 — KubeVirt cannot advertise devices.kubevirt.io/kvm." >&2
echo "hack/e2e-compose.yaml binds the host /dev; check the host exposes /dev/kvm." >&2
exit 1
fi
}
@test "Bootstrap Talos cluster" {
# Retry for up to 120s: talosctl bootstrap refuses with "time is not in sync
# yet" until NTP converges on a freshly started node. This is the same
# allowance the QEMU lane makes, and it is a genuine infrastructure wait, not
# a retry around product logic.
#
# If it never converges, the usual cause is that the nodes cannot reach the
# internet at all rather than slow NTP -- most often because docker sets
# `-P FORWARD DROP` and the compose bridge's traffic is being dropped.
if ! timeout 120 sh -ec 'until talosctl bootstrap -n 192.168.123.11 -e 192.168.123.11; do sleep 2; done'; then
echo "talosctl bootstrap never succeeded." >&2
echo "If the error was 'time is not in sync yet', the nodes likely have no egress:" >&2
echo "check 'iptables -S FORWARD' on the host for a DROP policy over the compose bridge." >&2
talosctl dmesg -n 192.168.123.11 -e 192.168.123.11 || true
exit 1
fi
}
@test "Wait until etcd is healthy" {
# Same 5m budget and the same reason as the QEMU lane: a 3-node etcd converges
# through Talos's serialized learner promotion, so the third member is only
# promoted once the second has caught up.
if ! timeout 300 sh -ec 'until talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.11 >/dev/null 2>&1; do sleep 1; done'; then
talosctl dmesg -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.11 || true
exit 1
fi
timeout 60 sh -ec 'while talosctl etcd members -n 192.168.123.11,192.168.123.12,192.168.123.13 -e 192.168.123.11 2>&1 | grep -q "rpc error"; do sleep 1; done'
}
@test "Retrieve kubeconfig and register all three nodes" {
rm -f kubeconfig
# No ARP VIP in container mode, so the endpoint is a node IP rather than the
# QEMU lane's 192.168.123.10.
talosctl kubeconfig kubeconfig -e 192.168.123.11 -n 192.168.123.11
if ! timeout 120 sh -ec 'until [ $(kubectl get node --no-headers | wc -l) -eq 3 ]; do sleep 1; done'; then
echo "Fewer than three nodes registered:" >&2
kubectl get nodes -o wide >&2 || true
exit 1
fi
}
@test "Node allocatable matches the container limits" {
# Container mode exposes host-wide capacity through /proc, so without the
# systemReserved values generated by e2e-container-up.sh every one of these
# nodes advertises the runner's full 32 CPU / ~125 GiB. That lets the
# scheduler pack far beyond each 8 CPU / 24 GiB cgroup and turns later OOMs
# into apparent workload flakes. Check the scheduler-visible result, not just
# the machine-config input.
local node cpu cpu_m memory memory_kib
local node_cpu_limit="${COZY_E2E_NODE_CPUS:-8}"
local node_memory_limit_mib="${COZY_E2E_NODE_MEMORY_MIB:-24576}"
local max_cpu_m=$(( node_cpu_limit * 1000 ))
local min_cpu_m=$(( max_cpu_m - 1000 ))
local max_memory_kib=$(( node_memory_limit_mib * 1024 ))
local min_memory_kib=$(( max_memory_kib - 1024 * 1024 ))
for node in srv1 srv2 srv3; do
cpu=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.cpu}')
case "$cpu" in
*m) cpu_m=${cpu%m} ;;
*[!0-9]* | '')
echo "node $node reported an unreadable allocatable CPU quantity: '$cpu'" >&2
return 1
;;
*) cpu_m=$(( cpu * 1000 )) ;;
esac
memory=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.memory}')
if ! memory_kib=$(kubernetes_memory_to_kib "$memory"); then
echo "node $node reported an unreadable allocatable memory quantity: '$memory'" >&2
return 1
fi
if [ "$cpu_m" -lt "$min_cpu_m" ] || [ "$cpu_m" -gt "$max_cpu_m" ]; then
echo "node $node allocatable CPU $cpu is outside ${min_cpu_m}-${max_cpu_m}m" >&2
return 1
fi
if [ "$memory_kib" -lt "$min_memory_kib" ] || [ "$memory_kib" -gt "$max_memory_kib" ]; then
echo "node $node allocatable memory $memory is outside ${min_memory_kib}-${max_memory_kib}Ki" >&2
return 1
fi
echo "$node allocatable: ${cpu}/${memory}"
done
}