hack/backup-access-preflight.bats
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bats
# Unit tests for the E2E-only S3 access gate that runs before expensive backup
# workloads. All cluster and S3 calls are shell stubs.
load test_helper
@test "preflight helper remains sourceable by the POSIX test runner" {
helper=hack/e2e-chainsaw/_lib/backup-access-preflight.sh
[ "$(sed -n '1p' "$helper")" = '#!/bin/sh' ]
if grep -nE '^[[:space:]]*(set[[:space:]].*pipefail|local([[:space:]]|$))' "$helper"; then
echo "backup preflight is sourced by /bin/sh and must not contain Bash-only shell state" >&2
exit 1
fi
sh -n "$helper"
}
@test "preflight fails closed when the BucketInfo Secret cannot be read" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*) return 1 ;;
*) echo "preflight continued after the Secret read failed: $*" >&2; return 1 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "failed BucketInfo Secret read passed the S3 preflight" >&2
exit 1
fi
grep -q 'failed to read BucketInfo Secret tenant-test/bucket-demo-backup' "$out"
if grep -q 'lacks access key' "$out"; then
echo "preflight continued into credential parsing after the Secret read failed" >&2
exit 1
fi
}
@test "preflight retries IAM convergence then proves PUT GET compare and DELETE" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
attempts=$(mktemp)
uploaded_source=$(mktemp)
calls=$(mktemp)
printf '0\n' > "$attempts"
kubectl() {
printf 'kubectl %s\n' "$*" >> "$calls"
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*)
while :; do command sleep 1; done
;;
*) echo "unexpected kubectl call: $*" >&2; return 1 ;;
esac
}
timeout() {
shift
if [ "$1" = sh ]; then
return 0
fi
"$@"
}
sleep() { :; }
mc() {
printf 'mc %s\n' "$*" >> "$calls"
case "$1" in
alias) return 0 ;;
cp)
case "$3" in
backup-preflight/*)
command cp "$(sed -n '1p' "$uploaded_source")" "$4"
;;
*)
count=$(sed -n '1p' "$attempts")
count=$(( count + 1 ))
printf '%s\n' "$count" > "$attempts"
if [ "$count" -eq 1 ]; then
echo 'AccessDenied: IAM identity not loaded yet' >&2
return 1
fi
printf '%s\n' "$3" > "$uploaded_source"
;;
esac
;;
rm) return 0 ;;
*) echo "unexpected mc call: $*" >&2; return 1 ;;
esac
}
cozy_backup_access_preflight tenant-test bucket-demo-backup 5 >/dev/null
[ "$(sed -n '1p' "$attempts")" -eq 2 ] || {
echo "preflight did not retry the not-yet-loaded IAM identity" >&2
cat "$calls" >&2
exit 1
}
[ "$(grep -c '^mc rm ' "$calls")" -eq 1 ] || {
echo "preflight did not delete its successfully verified object exactly once" >&2
cat "$calls" >&2
exit 1
}
}
@test "preflight fails before backup work when granted credentials never reach S3" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*) while :; do command sleep 1; done ;;
*) return 1 ;;
esac
}
timeout() {
shift
if [ "$1" = sh ]; then return 0; fi
"$@"
}
mc() {
case "$1" in
alias) return 0 ;;
cp) echo 'AccessDenied' >&2; return 1 ;;
*) return 0 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "unusable credentials passed the S3 preflight" >&2
exit 1
fi
grep -q 'granted but did not pass S3 PUT/GET/DELETE within 0s' "$out"
grep -q 's3-preflight: AccessDenied' "$out"
grep -q 's3-preflight: attempt 1 failed at PUT' "$out"
}
@test "preflight timeout reports the port-forward and how every attempt ended" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
now=$(mktemp)
pf_ready=$(mktemp)
attempts=$(mktemp)
uploaded_source=$(mktemp)
printf '100\n' > "$now"
printf '0\n' > "$attempts"
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*)
echo 'Forwarding from 127.0.0.1:18333 -> 8333'
printf 'ready\n' > "$pf_ready"
while :; do command sleep 1; done
;;
*) return 1 ;;
esac
}
# A stubbed clock keeps the retry budget independent of how fast the
# stubs run: only a completed PUT advances it, so the loop ends after a
# known number of attempts.
date() {
if [ "${1:-}" = +%s ]; then sed -n '1p' "$now"; else command date "$@"; fi
}
timeout() {
shift
if [ "$1" = sh ]; then
waited=0
while [ ! -s "$pf_ready" ] && [ "$waited" -lt 100 ]; do
waited=$(( waited + 1 ))
command sleep 0.1
done
return 0
fi
"$@"
}
sleep() { :; }
# The first three attempts break at a later stage than the one before, so
# the stages that follow a completed upload are told apart rather than
# merged.
mc() {
case "$1" in
alias) return 0 ;;
cp)
case "$3" in
backup-preflight/*)
case "$(sed -n '1p' "$attempts")" in
1)
echo 'mc: <ERROR> Unable to download: connection reset by peer' >&2
return 1
;;
2) printf 'truncated\n' > "$4" ;;
*) command cp "$(sed -n '1p' "$uploaded_source")" "$4" ;;
esac
;;
*)
printf '%s\n' "$3" > "$uploaded_source"
count=$(sed -n '1p' "$attempts")
printf '%s\n' "$(( count + 1 ))" > "$attempts"
stamp=$(sed -n '1p' "$now")
printf '%s\n' "$(( stamp + 10 ))" > "$now"
# Do not tidy this arm away into a clean ascending
# ladder. A ladder where every attempt fails later than
# the last can never catch a missing per-iteration
# stage reset, because the stale value is overwritten
# before anything reads it.
if [ "$count" -eq 3 ]; then
echo 'mc: <ERROR> Unable to upload: connection reset by peer' >&2
return 1
fi
echo 'source -> backup-preflight/bucket-real/object'
;;
esac
;;
rm) return 1 ;;
*) return 0 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 35 >"$out" 2>&1; then
echo "an S3 round trip that never completed passed the preflight" >&2
exit 1
fi
grep -q 'port-forward: Forwarding from 127.0.0.1:18333' "$out" || {
echo "timeout path dropped the port-forward log" >&2
cat "$out" >&2
exit 1
}
for expected in \
'attempt 1 failed at GET' \
'attempt 2 failed at compare' \
'attempt 3 failed at DELETE' \
'attempt 4 failed at PUT'; do
grep -q "s3-preflight: $expected" "$out" || {
echo "timeout path lost '$expected'" >&2
cat "$out" >&2
exit 1
}
done
}
@test "all active database backup flows gate immediately after accessGranted" {
for file in \
examples/backups/postgres/run-all.sh \
examples/backups/mariadb/run-all.sh \
examples/backups/mongodb/run-all.sh \
examples/backups/clickhouse/03-create-bucket.sh \
examples/backups/kafka/03-create-bucket.sh; do
granted=$(grep -n 'accessGranted' "$file" | head -n 1 | cut -d: -f1)
preflight=$(grep -n 'cozy_backup_access_preflight' "$file" | head -n 1 | cut -d: -f1)
coordinates=$(grep -n 'Reading bucket coordinates' "$file" | head -n 1 | cut -d: -f1)
if [ -z "$granted" ] || [ -z "$preflight" ] || [ -z "$coordinates" ] \
|| [ "$granted" -ge "$preflight" ] || [ "$preflight" -ge "$coordinates" ]; then
echo "$file must run the S3 preflight after accessGranted and before consuming bucket coordinates" >&2
exit 1
fi
done
}
@test "the five preflight-wired Chainsaw backup roundtrips enable the E2E preflight" {
for file in \
hack/e2e-chainsaw/postgres/chainsaw-test.yaml \
hack/e2e-chainsaw/mariadb/chainsaw-test.yaml \
hack/e2e-chainsaw/mongodb/chainsaw-test.yaml \
hack/e2e-chainsaw/clickhouse/chainsaw-test.yaml \
hack/e2e-chainsaw/kafka/chainsaw-test.yaml; do
grep -q 'COZY_E2E_BACKUP_PREFLIGHT=1' "$file" || {
echo "$file does not enable the backup access preflight" >&2
exit 1
}
done
}
@test "preflight reports the port-forward log when the S3 client cannot be configured" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
pf_ready=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*)
echo 'Forwarding from 127.0.0.1:18333 -> 8333'
printf 'ready\n' > "$pf_ready"
while :; do command sleep 1; done
;;
*) return 1 ;;
esac
}
timeout() {
shift
if [ "$1" = sh ]; then
waited=0
while [ ! -s "$pf_ready" ] && [ "$waited" -lt 100 ]; do
waited=$(( waited + 1 ))
command sleep 0.1
done
return 0
fi
"$@"
}
mc() {
case "$1" in
alias) echo 'mc: <ERROR> Unable to initialize new alias: not an S3 endpoint' >&2; return 1 ;;
*) echo "preflight continued after the S3 client failed: mc $*" >&2; return 1 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "a failed S3 client setup passed the preflight" >&2
exit 1
fi
grep -q 'failed to configure S3 client for BucketAccess tenant-test/bucket-demo-backup' "$out"
grep -q 'port-forward: Forwarding from 127.0.0.1:18333' "$out" || {
echo "the S3 client failure dropped the port-forward log" >&2
cat "$out" >&2
exit 1
}
}
@test "preflight cleanup deletes a failed round trip's object through a live tunnel" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
pf_state=$(mktemp)
calls=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*)
# The PID of this background subshell, which is what the
# helper kills; the delete below probes it.
sh -c 'echo $PPID' > "$pf_state"
while :; do command sleep 1; done
;;
*) return 1 ;;
esac
}
timeout() {
shift
if [ "$1" = sh ]; then
waited=0
while [ ! -s "$pf_state" ] && [ "$waited" -lt 100 ]; do
waited=$(( waited + 1 ))
command sleep 0.1
done
return 0
fi
"$@"
}
mc() {
case "$1" in
alias) return 0 ;;
cp)
case "$3" in
backup-preflight/*) echo 'mc: <ERROR> Unable to download: connection reset by peer' >&2; return 1 ;;
*) return 0 ;;
esac
;;
rm)
if kill -0 "$(cat "$pf_state")" 2>/dev/null; then
printf 'rm with tunnel alive\n' >> "$calls"
else
printf 'rm with tunnel gone\n' >> "$calls"
echo 'mc: <ERROR> dial tcp 127.0.0.1:18333: connect: connection refused' >&2
return 1
fi
;;
*) return 1 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "a failed download passed the preflight" >&2
exit 1
fi
[ "$(cat "$calls")" = "rm with tunnel alive" ] || {
echo "cleanup did not delete the uploaded object while the port-forward was still up:" >&2
cat "$calls" "$out" >&2
exit 1
}
if grep -q 'could not remove' "$out"; then
echo "cleanup warned about a delete that succeeded" >&2
cat "$out" >&2
exit 1
fi
}
@test "preflight cleanup tries the delete when the upload timed out after landing" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
pf_state=$(mktemp)
calls=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*)
printf alive > "$pf_state"
while :; do command sleep 1; done
;;
*) return 1 ;;
esac
}
# The PUT reaches the server and then outlives its bound, so timeout
# reports 124 while the object is already in the bucket.
timeout() {
shift
if [ "$1" = sh ]; then
waited=0
while [ ! -s "$pf_state" ] && [ "$waited" -lt 100 ]; do
waited=$(( waited + 1 ))
command sleep 0.1
done
return 0
fi
case "$1 $2 ${5:-}" in
"mc cp backup-preflight/"*)
printf 'put\n' >> "$calls"
return 124
;;
esac
"$@"
}
mc() {
case "$1" in
alias) return 0 ;;
rm) printf 'rm\n' >> "$calls" ;;
*) return 1 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "a timed-out upload passed the preflight" >&2
exit 1
fi
grep -qx put "$calls"
grep -qx rm "$calls" || {
echo "cleanup skipped the delete of an upload that may have landed:" >&2
cat "$calls" "$out" >&2
exit 1
}
}
@test "preflight cleanup neither deletes nor warns when every PUT was refused" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
calls=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*) while :; do command sleep 1; done ;;
*) return 1 ;;
esac
}
timeout() {
shift
if [ "$1" = sh ]; then return 0; fi
"$@"
}
mc() {
case "$1" in
alias) return 0 ;;
cp) echo 'AccessDenied' >&2; return 1 ;;
rm) printf 'rm\n' >> "$calls"; echo 'AccessDenied' >&2; return 1 ;;
*) return 1 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "a refused PUT passed the preflight" >&2
exit 1
fi
if [ -s "$calls" ] || grep -q 'could not remove' "$out"; then
echo "cleanup tried to delete an object no PUT created:" >&2
cat "$calls" "$out" >&2
exit 1
fi
}
@test "preflight cleanup names the timeout when the delete itself times out" {
. hack/e2e-chainsaw/_lib/backup-access-preflight.sh
out=$(mktemp)
kubectl() {
case "$*" in
*" get secret bucket-demo-backup "*)
printf '%s' '{"spec":{"secretS3":{"accessKeyID":"access","accessSecretKey":"secret"},"bucketName":"bucket-real"}}' | base64 | tr -d '\n'
;;
*" port-forward "*) while :; do command sleep 1; done ;;
*) return 1 ;;
esac
}
# The delete hangs until its bound kills it: exit 124, nothing written.
timeout() {
shift
if [ "$1" = sh ]; then return 0; fi
if [ "$1 $2" = "mc rm" ]; then return 124; fi
"$@"
}
mc() {
case "$1" in
alias) return 0 ;;
cp)
case "$3" in
backup-preflight/*) echo 'mc: <ERROR> Unable to download: connection reset by peer' >&2; return 1 ;;
*) return 0 ;;
esac
;;
*) return 1 ;;
esac
}
if cozy_backup_access_preflight tenant-test bucket-demo-backup 0 >"$out" 2>&1; then
echo "a failed download passed the preflight" >&2
exit 1
fi
grep -q 'could not remove failed S3 preflight object' "$out"
grep -q 's3-cleanup: mc rm timed out after 15s' "$out" || {
echo "a delete that timed out left the warning without a reason:" >&2
cat "$out" >&2
exit 1
}
}