hack/e2e-talos-image-cache.yaml
Rama main · réplica de lectura del repositorio original.
# In-sandbox caching mirror for the Talos Image Factory worker OS disk image.
#
# Why this exists: tenant Kubernetes worker VMs boot from a Talos raw disk image
# that CDI streams over HTTP from the PUBLIC Talos Image Factory
# (https://factory.talos.dev/image/<schematicID>/<version>/openstack-amd64.raw.xz).
# That endpoint does not support byte-range requests (so CDI falls back to a full
# sequential copy-to-scratch) and its egress is intermittently flaky from the CI
# runner — a mid-transfer HTTP/2 stream reset or a stalled download makes the
# worker DataVolume import never finish, the worker VM never boots, no tenant
# node joins within the chart's 12-minute node-join deadline, and the
# `kubernetes-*` e2e tests fail (and, even when they pass, take 18-22 minutes
# each dominated by these downloads). See cozystack/cozystack#3231.
#
# This Deployment fetches the image ONCE at e2e-install time with an aggressive
# curl retry loop (decoupling the one flaky external fetch from the many
# per-worker imports), then serves it locally over plain HTTP with range support.
# Tenant `Kubernetes` CRs point `spec.talos.imageFactoryURL` at this Service
# (see hack/e2e-chainsaw/_lib/run-kubernetes.sh) so every worker imports from the
# warm local copy instead of the public internet. Tenant namespaces run under a
# default-deny Cilium egress installed by the tenant chart, whose allow rules cover
# same-namespace endpoints, the tenant tree and the `world` entity (which is why the
# public factory works) but NOT an arbitrary kube-system Service, so a worker's CDI
# importer would resolve this Service yet have its TCP connect to the ClusterIP
# silently dropped. The CiliumClusterwideNetworkPolicy at the end of this file punches a
# single tightly-scoped hole for that path. The e2e harness only points tenants
# here after a tenant-scoped probe confirms the importer path actually reaches the
# ClusterIP with a 206; otherwise it falls back to the public factory, so the
# mirror can only help, never make CI worse.
#
# The __SCHEMATIC_ID__ / __TALOS_VERSION__ placeholders are substituted at apply
# time by hack/e2e-install-cozystack.bats from packages/apps/kubernetes/values.yaml.
#
# CI band-aid, not a product artifact. The durable fix is the overridable
# `talos.imageFactoryURL` chart value this manifest exercises; a self-hosted
# Talos Image Factory or an operator's own mirror is the production path.
apiVersion: v1
kind: Service
metadata:
name: talos-image-cache
namespace: kube-system
labels:
app.kubernetes.io/name: talos-image-cache
spec:
selector:
app.kubernetes.io/name: talos-image-cache
ports:
- name: http
port: 80
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: talos-image-cache
namespace: kube-system
labels:
app.kubernetes.io/name: talos-image-cache
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: talos-image-cache
strategy:
type: Recreate
template:
metadata:
labels:
app.kubernetes.io/name: talos-image-cache
spec:
# The seed fetch and static serving use ordinary pod networking — the same
# egress path the CDI importer pods already use to reach factory.talos.dev.
initContainers:
# Fetch the compressed raw disk image ONCE, retrying hard against the
# flaky upstream. The Deployment only becomes Available after this
# succeeds, so "Available" is the e2e harness's signal that the mirror is
# warm and safe to point tenants at.
- name: seed
# Reuse the in-tree, digest-pinned image already shipped by the
# installer and pre-pulled for the cilium-leak-healer watchdog, so this
# pod does not add a fresh external image pull of its own.
image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
command: [/bin/sh, -ec]
args:
- |
dest="/data/image/__SCHEMATIC_ID__/__TALOS_VERSION__/openstack-amd64.raw.xz"
url="https://factory.talos.dev/image/__SCHEMATIC_ID__/__TALOS_VERSION__/openstack-amd64.raw.xz"
mkdir -p "$(dirname "$dest")"
echo "seeding $dest from $url"
# --retry-all-errors + a long retry budget rides out the mid-transfer
# stream resets that break the CDI importer's single attempt.
# --speed-time/--speed-limit additionally abort a transfer that
# stalls below 1 KiB/s for 60s — a frozen (not reset) download is
# not an "error", so --retry-all-errors alone would let it hang out
# the whole rollout budget; the abort is then retried like any other
# failure.
curl --fail --location --show-error \
--retry 30 --retry-all-errors --retry-delay 15 \
--connect-timeout 30 --speed-time 60 --speed-limit 1024 \
--output "$dest" \
"$url"
ls -la "$dest"
volumeMounts:
- name: data
mountPath: /data
containers:
- name: serve
image: docker.io/alpine/k8s:1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47
# Serve the seeded image over HTTP with byte-range (206) support so CDI
# can use its fast range import instead of the copy-to-scratch fallback
# the range-less public factory forces. The pinned image ships neither a
# `busybox httpd` applet nor a standalone httpd/nginx, and stock
# `python3 -m http.server` ignores Range headers (answers 200, not 206),
# so a tiny range-capable server (serve.py ConfigMap below) runs on the
# image's bundled python3 — no extra package pull that would reintroduce
# a flaky external dependency.
command: [python3, /serve/serve.py, /data, "8080"]
ports:
- name: http
containerPort: 8080
readinessProbe:
# httpGet (not tcpSocket) so readiness reflects the server actually
# answering HTTP, not merely the port being open.
httpGet:
path: /
port: http
initialDelaySeconds: 2
periodSeconds: 5
resources:
requests:
cpu: 20m
memory: 32Mi
limits:
memory: 128Mi
volumeMounts:
- name: data
mountPath: /data
- name: serve-script
mountPath: /serve
volumes:
- name: data
emptyDir: {}
- name: serve-script
configMap:
name: talos-image-cache-serve
---
apiVersion: v1
kind: ConfigMap
metadata:
name: talos-image-cache-serve
namespace: kube-system
labels:
app.kubernetes.io/name: talos-image-cache
data:
# Minimal static file server with HTTP byte-range support. Stock
# `python3 -m http.server` answers Range requests with 200 + the full body;
# this adds the single-range 206 path (Content-Range + Accept-Ranges) CDI
# needs to skip its slow copy-to-scratch import.
serve.py: |
import http.server
import os
import sys
import time
ROOT = sys.argv[1] if len(sys.argv) > 1 else "/data"
PORT = int(sys.argv[2]) if len(sys.argv) > 2 else 8080
class RangeHandler(http.server.SimpleHTTPRequestHandler):
# HTTP/1.1 keep-alive lets CDI pull many ranges over one connection.
protocol_version = "HTTP/1.1"
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=ROOT, **kwargs)
def send_head(self):
path = self.translate_path(self.path)
if os.path.isdir(path):
return super().send_head()
try:
f = open(path, "rb")
except OSError:
self.send_error(404)
return None
size = os.fstat(f.fileno()).st_size
ctype = self.guess_type(path)
rng = self.headers.get("Range", "")
if not rng.startswith("bytes="):
self._remaining = size
self._transfer_range = "full"
self.send_response(200)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(size))
self.send_header("Accept-Ranges", "bytes")
self.end_headers()
return f
start_s, _, end_s = rng[6:].split(",")[0].strip().partition("-")
start = int(start_s) if start_s else max(0, size - int(end_s))
end = int(end_s) if start_s and end_s else size - 1
if start >= size or start > end:
self.send_response(416)
self.send_header("Content-Range", "bytes */%d" % size)
self.send_header("Content-Length", "0")
self.end_headers()
f.close()
return None
end = min(end, size - 1)
f.seek(start)
self._remaining = end - start + 1
self._transfer_range = "%d-%d/%d" % (start, end, size)
self.send_response(206)
self.send_header("Content-Type", ctype)
self.send_header("Content-Range", "bytes %d-%d/%d" % (start, end, size))
self.send_header("Content-Length", str(self._remaining))
self.send_header("Accept-Ranges", "bytes")
self.end_headers()
return f
def copyfile(self, source, outputfile):
remaining = getattr(self, "_remaining", None)
if remaining is None:
super().copyfile(source, outputfile)
return
self._remaining = None
requested = remaining
sent = 0
started = time.monotonic()
try:
while remaining > 0:
chunk = source.read(min(65536, remaining))
if not chunk:
break
outputfile.write(chunk)
sent += len(chunk)
remaining -= len(chunk)
finally:
status = "complete" if sent == requested else "interrupted"
self.log_message(
"transfer-finished status=%s path=%s range=%s requested=%d sent=%d duration=%.3fs",
status,
self.path,
getattr(self, "_transfer_range", "unknown"),
requested,
sent,
time.monotonic() - started,
)
with http.server.ThreadingHTTPServer(("0.0.0.0", PORT), RangeHandler) as httpd:
print("serving %s on :%d" % (ROOT, PORT), flush=True)
httpd.serve_forever()
---
# CI-only: allow the tenant CDI importer Pods to reach the mirror.
#
# Tenant namespaces run under a default-deny Cilium egress installed by the tenant
# chart (allow-internal-communication, allow-external-communication and the
# `<tenant>-egress` CiliumClusterwideNetworkPolicy). Their union allows egress to
# same-namespace endpoints, the tenant tree and the `world` entity (so the public
# factory works), but nothing permits egress to an arbitrary kube-system Service —
# so a worker's CDI importer resolves talos-image-cache.kube-system.svc yet its TCP
# connect to the Service ClusterIP is silently dropped (dial ... i/o timeout), the
# disk import never starts, and the tenant node never joins. Cilium unions allow
# rules across policies, so this adds one tightly-scoped allow: importer Pods in
# tenant-test may egress to the mirror Pod in kube-system.
#
# Scoped to tenant-test (which the tenant chart already puts under default-deny
# egress) so it only ever ADDS an allow — it never flips another namespace from
# default-allow to default-deny. The mirror Pod is selected by no ingress policy
# (default-allow ingress), so no ingress rule is needed; replies return on the
# established conntrack entry.
#
# Applied at point-of-use by hack/e2e-chainsaw/_lib/talos-image-cache.sh, not with
# the rest of this manifest in hack/e2e-install-cozystack.bats: that runs before
# Cozystack (and thus before Cilium's CRDs) is installed, so this kind does not yet
# exist when the manifest is first applied.
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
name: e2e-talos-image-cache-importer-egress
labels:
app.kubernetes.io/name: talos-image-cache
spec:
endpointSelector:
matchLabels:
"k8s:cdi.kubevirt.io": importer
"k8s:io.kubernetes.pod.namespace": tenant-test
# No toPorts: the allow is already scoped to a single kube-system pod identity
# (CI-only, torn down with the sandbox), so an L4 port restriction adds no
# meaningful blast-radius reduction — only risk. With kube-proxy-less Cilium the
# importer's connection to the Service ClusterIP is translated to the backend
# pod port per packet in bpf_lxc.c (the chart forces bpf-lb-sock-hostns-only
# while Gateway API is on) BEFORE egress policy is evaluated, so a port literal here
# would have to track the container port (8080), not the Service port (80); a
# wrong guess silently drops the traffic and reintroduces the exact public-factory
# fallback this policy exists to prevent.
egress:
- toEndpoints:
- matchLabels:
"k8s:io.kubernetes.pod.namespace": kube-system
"k8s:app.kubernetes.io/name": talos-image-cache