deploy_rolling_workflow_test.go
Rama main · réplica de lectura del repositorio original.
package main
import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
// Guards for the multi-node rollout: the workflow deploys one node at a time,
// checks every node before touching any, and keeps probing each node, pinned
// to it, until its old slot has really stopped.
const deployNodesMatrix = `${{ fromJSON(vars.CLIRELAY_DEPLOY_NODES || '["default"]') }}`
type ghWorkflow struct {
Env map[string]string `yaml:"env"`
Jobs map[string]ghJob `yaml:"jobs"`
}
type ghJob struct {
Needs ghNeeds `yaml:"needs"`
If string `yaml:"if"`
Strategy *ghStrategy `yaml:"strategy"`
Env map[string]string `yaml:"env"`
Steps []ghStep `yaml:"steps"`
}
type ghStrategy struct {
MaxParallel *int `yaml:"max-parallel"`
FailFast *bool `yaml:"fail-fast"`
Matrix map[string]string `yaml:"matrix"`
}
type ghStep struct {
Name string `yaml:"name"`
ID string `yaml:"id"`
If string `yaml:"if"`
Uses string `yaml:"uses"`
Run string `yaml:"run"`
Env map[string]string `yaml:"env"`
}
// ghNeeds accepts both `needs: build` and `needs: [build, preflight]`.
type ghNeeds []string
func (n *ghNeeds) UnmarshalYAML(value *yaml.Node) error {
if value.Kind == yaml.ScalarNode {
*n = ghNeeds{value.Value}
return nil
}
var list []string
if err := value.Decode(&list); err != nil {
return err
}
*n = list
return nil
}
func (n ghNeeds) has(job string) bool {
for _, need := range n {
if need == job {
return true
}
}
return false
}
func loadDeployWorkflow(t *testing.T) (ghWorkflow, yaml.Node) {
t.Helper()
data, err := os.ReadFile(".github/workflows/deploy.yml")
if err != nil {
t.Fatalf("read deploy workflow: %v", err)
}
var wf ghWorkflow
if err := yaml.Unmarshal(data, &wf); err != nil {
t.Fatalf("parse deploy workflow: %v", err)
}
var root yaml.Node
if err := yaml.Unmarshal(data, &root); err != nil {
t.Fatalf("parse deploy workflow: %v", err)
}
return wf, root
}
func workflowJob(t *testing.T, wf ghWorkflow, name string) ghJob {
t.Helper()
job, ok := wf.Jobs[name]
if !ok {
t.Fatalf("deploy workflow has no %q job", name)
}
return job
}
func workflowStep(t *testing.T, job ghJob, name string) (int, ghStep) {
t.Helper()
for i, step := range job.Steps {
if step.Name == name {
return i, step
}
}
t.Fatalf("job has no step named %q", name)
return -1, ghStep{}
}
// mappingKeys yields every mapping key anywhere in a YAML document.
func mappingKeys(node *yaml.Node, visit func(key string)) {
if node.Kind == yaml.MappingNode {
for i := 0; i+1 < len(node.Content); i += 2 {
visit(node.Content[i].Value)
}
}
for _, child := range node.Content {
mappingKeys(child, visit)
}
}
func TestDeployWorkflowRollsOutOneNodeAtATime(t *testing.T) {
wf, root := loadDeployWorkflow(t)
deploy := workflowJob(t, wf, "deploy")
if deploy.Strategy == nil {
t.Fatalf("deploy job must be a matrix over the rollout nodes")
}
// max-parallel 1 is what makes the rollout rolling: every node stays
// healthy through its drain before the next one is touched.
if deploy.Strategy.MaxParallel == nil || *deploy.Strategy.MaxParallel != 1 {
t.Fatalf("deploy matrix must set max-parallel: 1, got %v", deploy.Strategy.MaxParallel)
}
// fail-fast stops the rollout at the first bad node, leaving the rest on
// the previous build instead of pushing a broken one everywhere.
if deploy.Strategy.FailFast == nil || !*deploy.Strategy.FailFast {
t.Fatalf("deploy matrix must set fail-fast: true")
}
if got := deploy.Strategy.Matrix["node"]; got != deployNodesMatrix {
t.Fatalf("deploy matrix nodes = %q, want %q so a repository without CLIRELAY_DEPLOY_NODES deploys the single default node", got, deployNodesMatrix)
}
if !deploy.Needs.has("build") || !deploy.Needs.has("preflight") {
t.Fatalf("deploy must wait for the build and for the preflight of every node, needs = %v", deploy.Needs)
}
preflight := workflowJob(t, wf, "preflight")
if preflight.Strategy == nil || preflight.Strategy.Matrix["node"] != deployNodesMatrix {
t.Fatalf("preflight must cover exactly the nodes the deploy matrix covers")
}
if preflight.Strategy.FailFast == nil || !*preflight.Strategy.FailFast {
t.Fatalf("preflight matrix must set fail-fast: true")
}
if !preflight.Needs.has("build") {
t.Fatalf("preflight must run after the build, needs = %v", preflight.Needs)
}
docker := workflowJob(t, wf, "docker")
if !docker.Needs.has("deploy") {
t.Fatalf("the image build must wait for every node to be deployed, needs = %v", docker.Needs)
}
for _, name := range []string{"preflight", "deploy", "docker"} {
if job := workflowJob(t, wf, name); !strings.Contains(job.If, "needs.build.outputs.should_deploy == 'true'") {
t.Fatalf("%s job must honour the SHOULD_DEPLOY opt-out, if = %q", name, job.If)
}
}
// A step or job that may fail without failing the run would let the
// rollout continue past a broken node.
mappingKeys(&root, func(key string) {
if key == "continue-on-error" {
t.Fatalf("deploy workflow must not use continue-on-error")
}
})
}
func TestDeployWorkflowResolvesSecretsPerNodeWithDefaultFallback(t *testing.T) {
wf, _ := loadDeployWorkflow(t)
want := map[string]string{
// A missing per-node host must never fall back to SERVER_HOST: that
// would deploy some other node twice and report the missing one done.
"NODE_HOST": `${{ matrix.node == 'default' && secrets.SERVER_HOST || secrets[format('SERVER_HOST_{0}', matrix.node)] }}`,
"NODE_PORT": `${{ matrix.node == 'default' && secrets.SERVER_PORT || secrets[format('SERVER_PORT_{0}', matrix.node)] }}`,
"NODE_SSH_KEY": `${{ secrets[format('SSH_PRIVATE_KEY_{0}', matrix.node)] || secrets.SSH_PRIVATE_KEY }}`,
"NODE_KNOWN_HOSTS": `${{ secrets[format('DEPLOY_SSH_KNOWN_HOSTS_{0}', matrix.node)] || secrets.DEPLOY_SSH_KNOWN_HOSTS }}`,
"NODE_SMOKE_IP": `${{ matrix.node == 'default' && (secrets.SMOKE_IP || vars.CLIRELAY_SMOKE_IP) || secrets[format('SMOKE_IP_{0}', matrix.node)] || vars[format('CLIRELAY_SMOKE_IP_{0}', matrix.node)] }}`,
}
for _, jobName := range []string{"preflight", "deploy"} {
_, step := workflowStep(t, workflowJob(t, wf, jobName), "Configure SSH for node")
if step.Run != "./scripts/gha-node-ssh.sh" {
t.Fatalf("%s must configure ssh with scripts/gha-node-ssh.sh, runs %q", jobName, step.Run)
}
for key, expr := range want {
if got := step.Env[key]; got != expr {
t.Fatalf("%s: %s = %q, want %q", jobName, key, got, expr)
}
}
}
// The workflow runs the helpers directly, so they must be committed executable.
for _, path := range []string{"scripts/gha-node-ssh.sh", "scripts/gha-node-verify.sh"} {
info, err := os.Stat(path)
if err != nil || info.Mode().Perm()&0o111 == 0 {
t.Fatalf("%s must exist and be executable (err=%v)", path, err)
}
}
_, plan := workflowStep(t, workflowJob(t, wf, "build"), "Record deploy mode")
for _, marker := range []string{`^[A-Za-z0-9_]+$`, `ascii_downcase`, `CLIRELAY_DEPLOY_NODES`} {
if !strings.Contains(plan.Run+plan.Env["DEPLOY_NODES"], marker) {
t.Fatalf("the build must validate node names before any node is touched; missing %q", marker)
}
}
}
func TestDeployWorkflowPreflightGatesEveryNode(t *testing.T) {
wf, _ := loadDeployWorkflow(t)
_, step := workflowStep(t, workflowJob(t, wf, "preflight"), "Check node is ready for this release")
for _, want := range []string{
`ssh deploy-target true`,
// The root scripts are synced by hand; a stale node fails here, before
// any node has been touched.
`grep -m1 '^SCRIPT_VERSION='`,
`EXPECTED_SCRIPT_VERSION`,
`/usr/local/sbin/clirelay-gha-deploy --preflight`,
`^CLIRELAY_DEPLOY_CHECK ok`,
// sudo must refuse variables outside the env_keep whitelist.
`CLIRELAY_SETENV_PROBE=1`,
`setenv-probe-accepted`,
} {
if !strings.Contains(step.Run, want) {
t.Fatalf("preflight check missing %q", want)
}
}
}
func TestDeployWorkflowProbesEachNodeAcrossItsDrain(t *testing.T) {
wf, _ := loadDeployWorkflow(t)
drain, err := strconv.Atoi(extractShellAssignment(t, "scripts/deploy-blue-green.sh", "DRAIN_SECONDS"))
if err != nil {
t.Fatalf("parse DRAIN_SECONDS: %v", err)
}
minProbe, err := strconv.Atoi(wf.Env["PROBE_MIN_SECONDS"])
if err != nil {
t.Fatalf("parse PROBE_MIN_SECONDS: %v", err)
}
margin, err := strconv.Atoi(wf.Env["PROBE_MARGIN_SECONDS"])
if err != nil {
t.Fatalf("parse PROBE_MARGIN_SECONDS: %v", err)
}
// The old slot is stopped DRAIN_SECONDS after cutover; probing for less
// declares success before the step that caused both past outages.
if minProbe < drain+30 || margin < 30 {
t.Fatalf("node probes must outlast the drain by 30s: PROBE_MIN_SECONDS=%d, PROBE_MARGIN_SECONDS=%d, DRAIN_SECONDS=%d", minProbe, margin, drain)
}
deploy := workflowJob(t, wf, "deploy")
deployIndex, _ := workflowStep(t, deploy, "Blue-green deploy via fixed root entrypoint")
verifyIndex, verify := workflowStep(t, deploy, "Verify service stays healthy through drain")
if verifyIndex <= deployIndex {
t.Fatalf("the per-node verification must run after that node's deploy")
}
if !strings.Contains(verify.Run, "./scripts/gha-node-verify.sh node") ||
verify.Env["DRAIN_SECONDS"] != "${{ steps.deploy.outputs.drain_seconds }}" ||
verify.Env["OLD_PORT"] != "${{ steps.deploy.outputs.old_port }}" {
t.Fatalf("per-node verification must use the drain and slots the node reported, got run=%q env=%v", verify.Run, verify.Env)
}
// The old slot may run for its whole TimeoutStopSec after the drain's
// SIGTERM, which is longer than its shutdown grace. Waiting only for the
// grace would fail a node whose slot is still finishing its shutdown.
_, deployStep := workflowStep(t, deploy, "Blue-green deploy via fixed root entrypoint")
if verify.Env["STOP_TIMEOUT_SECONDS"] != "${{ steps.deploy.outputs.stop_timeout_seconds }}" ||
!strings.Contains(deployStep.Run, "stop_timeout_seconds=*") {
t.Fatalf("per-node verification must wait for the stop timeout the node reported, got env=%v", verify.Env)
}
if _, ok := verify.Env["SHUTDOWN_GRACE_SECONDS"]; ok {
t.Fatalf("per-node verification must not size its wait by the shutdown grace, got env=%v", verify.Env)
}
docker := workflowJob(t, wf, "docker")
publicIndex, public := workflowStep(t, docker, "Verify public endpoint after rollout")
triggerIndex, _ := workflowStep(t, docker, "Trigger dev Docker image build")
if publicIndex >= triggerIndex || !strings.Contains(public.Run, "./scripts/gha-node-verify.sh public") {
t.Fatalf("an unpinned public probe must pass before the image build is triggered")
}
data, err := os.ReadFile("scripts/gha-node-verify.sh")
if err != nil {
t.Fatalf("read verify script: %v", err)
}
if !strings.Contains(string(data), `pin=(--resolve "${url_host}:${url_port}:${pin_addr}")`) {
t.Fatalf("per-node probes must be pinned to the node with curl --resolve")
}
window := extractShellFunction(t, "scripts/gha-node-verify.sh", "probe_window_seconds")
for drainSeconds, want := range map[string]string{"180": "210", "300": "330", "60": "210"} {
cmd := exec.Command("bash", "-c", window+"\nprobe_window_seconds")
cmd.Env = append(os.Environ(), "DRAIN_SECONDS="+drainSeconds,
"PROBE_MIN_SECONDS="+wf.Env["PROBE_MIN_SECONDS"], "PROBE_MARGIN_SECONDS="+wf.Env["PROBE_MARGIN_SECONDS"])
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("probe_window_seconds: %v\n%s", err, out)
}
if got := strings.TrimSpace(string(out)); got != want {
t.Fatalf("drain %ss: probe window = %ss, want %ss", drainSeconds, got, want)
}
}
}
// The next node may only be touched once the old slot on this one has really
// exited: until then the drain can still be refused, or still be cutting
// streams, and neither shows up in a health probe.
func TestWorkflowNodeVerifyWaitsForTheOldSlotToStop(t *testing.T) {
binDir := t.TempDir()
curlLog := filepath.Join(binDir, "curl.log")
writeExecutable(t, filepath.Join(binDir, "curl"), `#!/usr/bin/env bash
printf '%s\n' "$*" >> "$CURL_LOG"
printf 'HTTP/1.1 204 No Content\r\nx-cpa-version: %s\r\n\r\nprobe_status=204\n' "$FAKE_PUBLIC_VERSION"
`)
// FAKE_OLD_STATE is a space-separated sequence: one state per read of
// the node, the last one repeating.
writeExecutable(t, filepath.Join(binDir, "ssh"), `#!/usr/bin/env bash
cat > /dev/null
n=$(cat "$FAKE_READS" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$FAKE_READS"
set -- $FAKE_OLD_STATE
[ "$n" -le $# ] || n=$#
printf 'new_state=active old_state=%s served_version=%s\n' "${!n}" "$FAKE_SERVED_VERSION"
`)
run := func(t *testing.T, oldState, served string) (string, error) {
t.Helper()
cmd := exec.Command("bash", "scripts/gha-node-verify.sh", "node")
cmd.Env = append(os.Environ(),
"PATH="+binDir+":"+os.Getenv("PATH"),
"CURL_LOG="+curlLog,
"FAKE_READS="+filepath.Join(t.TempDir(), "reads"),
"FAKE_PUBLIC_VERSION=dev-abc1234",
"FAKE_OLD_STATE="+oldState,
"FAKE_SERVED_VERSION="+served,
"HEALTH_URL=https://relay.example.test/healthz",
"APP_VERSION=dev-abc1234",
"NODE=n43",
"NODE_SMOKE_IP=203.0.113.10",
"SLOT_SERVICE=clirelay2",
"NEW_PORT=8318",
"OLD_PORT=8319",
"DRAIN_SECONDS=1",
"STOP_TIMEOUT_SECONDS=0",
"PROBE_MIN_SECONDS=1",
"PROBE_MARGIN_SECONDS=0",
"PROBE_INTERVAL_SECONDS=1",
"STOP_WAIT_EXTRA_SECONDS=1",
"STOP_POLL_SECONDS=1",
"OLD_SLOT_HOLD_SECONDS=1",
)
out, err := cmd.CombinedOutput()
return string(out), err
}
if out, err := run(t, "inactive", "dev-abc1234"); err != nil {
t.Fatalf("a node whose old slot stopped must pass: %v\n%s", err, out)
}
logData, err := os.ReadFile(curlLog)
if err != nil {
t.Fatalf("read curl log: %v", err)
}
if !strings.Contains(string(logData), "--resolve relay.example.test:443:203.0.113.10 https://relay.example.test/healthz") {
t.Fatalf("probes were not pinned to the node:\n%s", logData)
}
out, err := run(t, "active", "dev-abc1234")
if err == nil || !strings.Contains(out, "old slot clirelay2-8319 is still active") {
t.Fatalf("an old slot that never stopped must fail the node, err=%v\n%s", err, out)
}
out, err = run(t, "inactive", "dev-0000000")
if err == nil || !strings.Contains(out, "reports build dev-0000000, expected dev-abc1234") {
t.Fatalf("a new slot serving another build must fail the node, err=%v\n%s", err, out)
}
// 2026-10-27: a hand-installed watchdog started the retired slot again
// 49 s after the drain stopped it, after this check had passed.
out, err = run(t, "inactive active", "dev-abc1234")
if err == nil || !strings.Contains(out, "old slot clirelay2-8319 stopped after the drain but is active again") {
t.Fatalf("an old slot that comes back after stopping must fail the node, err=%v\n%s", err, out)
}
// A lost ssh read after the hold is not evidence of anything.
out, err = run(t, "inactive unknown", "dev-abc1234")
if err != nil || !strings.Contains(out, "could not re-read old slot clirelay2-8319") {
t.Fatalf("an unreadable node after the hold must only warn, err=%v\n%s", err, out)
}
}
// ssh looks a host on a non-standard port up as "[host]:port"; an entry for the
// bare host only covers port 22 and the connection then fails with a message
// that does not say why.
func TestWorkflowNodeSSHRequiresKnownHostsEntryForThePort(t *testing.T) {
keyDir := t.TempDir()
keygen := exec.Command("ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", filepath.Join(keyDir, "host"))
if out, err := keygen.CombinedOutput(); err != nil {
t.Skipf("ssh-keygen unavailable: %v\n%s", err, out)
}
pub, err := os.ReadFile(filepath.Join(keyDir, "host.pub"))
if err != nil {
t.Fatalf("read host key: %v", err)
}
hostKey := strings.Join(strings.Fields(string(pub))[:2], " ")
run := func(t *testing.T, env ...string) (string, string, error) {
t.Helper()
home := t.TempDir()
githubEnv := filepath.Join(home, "github_env")
cmd := exec.Command("bash", "scripts/gha-node-ssh.sh")
cmd.Env = append(os.Environ(), append([]string{
"HOME=" + home,
"GITHUB_ENV=" + githubEnv,
"NODE_SSH_KEY=placeholder-key-material",
}, env...)...)
out, err := cmd.CombinedOutput()
config, _ := os.ReadFile(filepath.Join(home, ".ssh", "config"))
exported, _ := os.ReadFile(githubEnv)
return string(out), string(config) + string(exported), err
}
out, written, err := run(t, "NODE=n2", "NODE_HOST=203.0.113.10", "NODE_PORT=47222",
"NODE_KNOWN_HOSTS=[203.0.113.10]:47222 "+hostKey)
if err != nil {
t.Fatalf("a bracketed entry must match port 47222: %v\n%s", err, out)
}
for _, want := range []string{"Host deploy-target", "Port 47222", "User deploy", "StrictHostKeyChecking yes", "NODE_SMOKE_IP=203.0.113.10"} {
if !strings.Contains(written, want) {
t.Fatalf("ssh setup missing %q:\n%s", want, written)
}
}
out, _, err = run(t, "NODE=n2", "NODE_HOST=203.0.113.10", "NODE_PORT=47222",
"NODE_KNOWN_HOSTS=203.0.113.10 "+hostKey)
if err == nil || !strings.Contains(out, "no entry for [203.0.113.10]:47222") {
t.Fatalf("a bare-host entry must be reported for port 47222, err=%v\n%s", err, out)
}
out, written, err = run(t, "NODE=default", "NODE_HOST=203.0.113.10",
"NODE_KNOWN_HOSTS=203.0.113.10 "+hostKey, "NODE_SMOKE_IP=198.51.100.7")
if err != nil || !strings.Contains(written, "Port 22") || !strings.Contains(written, "NODE_SMOKE_IP=198.51.100.7") {
t.Fatalf("the default node on port 22 with an explicit smoke address must work, err=%v\n%s\n%s", err, out, written)
}
out, _, err = run(t, "NODE=n-156", "NODE_HOST=203.0.113.10", "NODE_KNOWN_HOSTS=203.0.113.10 "+hostKey)
if err == nil || !strings.Contains(out, "letters, digits and underscores") {
t.Fatalf("a node name that cannot be a secret suffix must be refused, err=%v\n%s", err, out)
}
out, _, err = run(t, "NODE=n2", "NODE_KNOWN_HOSTS=203.0.113.10 "+hostKey)
if err == nil || !strings.Contains(out, "secret SERVER_HOST_n2 is not set") {
t.Fatalf("a missing per-node host must name the per-node secret, err=%v\n%s", err, out)
}
// A node whose provider drops runner SSH is reached through a jump host,
// and the jump host is held to the same strict known_hosts check.
bothKeys := "[203.0.113.10]:47222 " + hostKey + "\n[198.51.100.53]:2233 " + hostKey
out, written, err = run(t, "NODE=n2", "NODE_HOST=203.0.113.10", "NODE_PORT=47222",
"NODE_KNOWN_HOSTS="+bothKeys, "NODE_JUMP=gha-jump@198.51.100.53:2233")
if err != nil {
t.Fatalf("a jump host with a known key must be accepted: %v\n%s", err, out)
}
for _, want := range []string{"Host deploy-jump", "HostName 198.51.100.53", "Port 2233", "User gha-jump", "ProxyJump deploy-jump"} {
if !strings.Contains(written, want) {
t.Fatalf("jump ssh setup missing %q:\n%s", want, written)
}
}
if !strings.Contains(out, "via gha-jump@198.51.100.53:2233") {
t.Fatalf("the setup summary must name the jump host:\n%s", out)
}
out, _, err = run(t, "NODE=n2", "NODE_HOST=203.0.113.10", "NODE_PORT=47222",
"NODE_KNOWN_HOSTS=[203.0.113.10]:47222 "+hostKey, "NODE_JUMP=gha-jump@198.51.100.53:2233")
if err == nil || !strings.Contains(out, "no entry for the jump host [198.51.100.53]:2233") {
t.Fatalf("a jump host without a known key must be refused, err=%v\n%s", err, out)
}
out, _, err = run(t, "NODE=n2", "NODE_HOST=203.0.113.10", "NODE_PORT=47222",
"NODE_KNOWN_HOSTS="+bothKeys, "NODE_JUMP=198.51.100.53")
if err == nil || !strings.Contains(out, "SSH_JUMP_n2 must be user@host[:port]") {
t.Fatalf("a jump host without a user must be refused, err=%v\n%s", err, out)
}
out, written, err = run(t, "NODE=n2", "NODE_HOST=203.0.113.10", "NODE_PORT=47222",
"NODE_KNOWN_HOSTS=[203.0.113.10]:47222 "+hostKey)
if err != nil || strings.Contains(written, "ProxyJump") {
t.Fatalf("without NODE_JUMP the node must be reached directly, err=%v\n%s\n%s", err, out, written)
}
}