deploy/update.sh
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bash
# Provenance: vendored verbatim from https://forgejo.home.lan/authelia/bun-templates commit 89be318daf8adab5ddca957162b3b7df8429e725
# ════════════════════════════════════════════════════════════════════════════
# update.sh — move an installed app to a newer version
#
# Usage:
# bash <state-dir>/scripts/update.sh [OPTIONS]
#
# Options:
# --version TAG Update to a specific version (default: resolve latest)
# --yes Do not prompt (required for downgrades / non-interactive)
# --help, -h Show this help
#
# Order of operations:
# compare versions -> refuse same/downgrade -> pre-update hook
# -> fetch + swap -> post-update hook -> refresh unit -> restart + health
# -> rollback hint
#
# DIST=binary swaps a release tarball's payload (executable + vec0.so +
# lib/libonnxruntime.so.1) instead of a git checkout; see
# synaptomind/adr/2026-10-01-unify-deploy-framework-adr.md.
# ════════════════════════════════════════════════════════════════════════════
set -euo pipefail
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]:-$0}")" 2>/dev/null && pwd -P)" || SCRIPT_DIR=""
load_common() {
local cand
for cand in "${SCRIPT_DIR}/lib/common.sh" "${SCRIPT_DIR}/common.sh"; do
if [ -f "$cand" ]; then
# shellcheck source=lib/common.sh
. "$cand"
return 0
fi
done
echo "[app] ERROR: cannot find lib/common.sh next to $0" >&2
exit 1
}
load_common
trap cleanup_run EXIT
ARG_VERSION=""
ASSUME_YES=false
CHECK_BACKUP=false
parse_args() {
while [ $# -gt 0 ]; do
case "$1" in
--version) ARG_VERSION="$2"; shift 2 ;;
--yes|-y) ASSUME_YES=true; shift ;;
--check-backup) CHECK_BACKUP=true; shift ;;
--help|-h) sed -n '2,16p' "$0" 2>/dev/null || echo "See the comment header of update.sh"; exit 0 ;;
*) error "unknown option: $1 (try --help)" ;;
esac
done
}
# ── Current / target version ───────────────────────────────────────────────
current_version() {
if [ "$DIST" = "binary" ]; then
local bin="${INSTALL_DIR}/${APP_NAME}"
if [ -x "$bin" ]; then app_version "$bin"; else echo "unknown"; fi
else
local v
v="$(read_package_version "${INSTALL_DIR}/package.json")"
echo "${v:-unknown}"
fi
}
# package.json version at origin/<branch> (source branch policy only).
#
# Sets REMOTE_PKG_VERSION instead of printing it. This function is called from a
# command substitution, and a command substitution is a SUBSHELL: the
# `cleanup_add` that used to sit here appended to the subshell's copy of the
# registry, so the EXIT trap never saw the temp file and it survived every
# source-mode update that resolved a branch. Same reason binary_stage_payload()
# sets STAGED_PAYLOAD and updater.sh sets STAGE_ROOT rather than printing them.
REMOTE_PKG_VERSION=""
remote_pkg_version() {
local tmp=""
mktemp_owned tmp || return 0
if git -C "$INSTALL_DIR" show "origin/${1}:package.json" > "$tmp" 2>/dev/null; then
REMOTE_PKG_VERSION="$(read_package_version "$tmp")"
fi
}
# ── Hooks ──────────────────────────────────────────────────────────────────
# The backups the pre-update hook reported, filled by collect_db_backups from the
# hook's own output. print_recovery restores exactly these.
DB_BACKUPS=()
# collect_db_backups FILE — the backup paths the pre-update hook printed.
#
# The hook's line IS the record: "… Database backed up: <path>", where <path> is
# <db>.backup/<name>.<timestamp>.bak. It is parsed rather than re-derived because
# the hook is the INSTALLED copy, possibly from an older release: those print the
# same line, while a path re-derived from this release's config.json would be
# wrong for any host whose database lives elsewhere.
collect_db_backups() {
local file="$1" line path
# Append to any backups already recorded by the built-in backup_databases
# (task #2297), deduping by canonical path.
if [ ! -f "$file" ]; then return 0; fi
while IFS= read -r line; do
case "$line" in
*"Database backed up: "*)
path="${line##*Database backed up: }"
# Strip trailing whitespace, never a leading one: a path may contain a
# space, and a mangled path is worse than none.
path="${path%"${path##*[![:space:]]}"}"
if [ -n "$path" ]; then
# dedupe by canonical path
local _dup=0 _b
for _b in "${DB_BACKUPS[@]:-}"; do
if [ "$_b" = "$path" ]; then _dup=1; break; fi
done
[ "$_dup" -eq 0 ] && DB_BACKUPS+=("$path")
fi
;;
esac
done < "$file"
return 0
}
run_hook() {
local hook="${HOOKS_DIR}/$1" rc=0 out=""
if [ -x "$hook" ]; then
info "Running ${1} hook..."
# The hook's output is DATA, not only a transcript: pre-update names the
# backup files it produced, and the recovery block restores exactly those
# (task #1079 F6 — it used to print a literal `synaptomind.db.<timestamp>.bak`,
# a path that does not exist, so the only DB restore it offered could not
# run). Teed, so a long backup still streams instead of looking hung, and the
# hook's stderr joins its stdout in that transcript. BOTH properties are
# load-bearing; neither may be traded for the exit status below.
#
# mktemp_owned, not `out="$(mktemp)"`: the bare mktemp was never registered
# with the EXIT trap, so every hook run left its transcript in TMPDIR — 91
# files per run of the deploy suite, on the same filesystem production's
# SQLite database lives on. The transcript is a temp file, not an artefact:
# nothing names it after the hook returns, so owning it is correct.
#
# WHY `${PIPESTATUS[0]}` (task #1102). The construct here used to be
# `{ "$hook" 2>&1 || rc=$?; } | tee "$out"`, and it could not report a failed
# hook: the `|| rc=$?` runs in the LEFT STAGE of a pipeline, and a pipeline
# stage is a SUBSHELL, so the assignment died there and the parent shell only
# ever saw tee's status (always 0). A pre-update hook exiting 7 therefore
# reached `if ! run_hook pre-update` as success, and the guard that a failed
# database backup must not be followed by a code swap was unreachable — while
# the comment above claimed the status "crossed the pipe" and update.sh:809
# called the hook fatal. Reproduced on this host; the fix is to read the
# stage's own status after the pipeline instead of trying to write it from
# inside one.
#
# Why not the alternatives:
# * `${PIPESTATUS[0]}` — what this uses. The hook's real status, streaming
# and capture untouched, and `|| rc=` keeps the non-zero pipeline from
# tripping `set -e`. Index 0 specifically, so a tee failure (ENOSPC — this
# host has filled / before) is NOT misreported as a hook failure.
# * plain `|| rc=$?` on the pipeline — works only because `set -o pipefail`
# is on, and then it attributes tee's failure to the hook, so a full disk
# aborts the update as if the backup had failed.
# * `set -e` on the pipeline — rejected by the AC and by the code: it would
# abort the script from inside a function whose caller is written to
# decide what a hook failure means.
# * dropping the tee — not an option at all: the transcript is what
# collect_db_backups parses, so the recovery block would name no backup.
if mktemp_owned out 2>/dev/null; then
{ "$hook" 2>&1; } | tee "$out" || rc=${PIPESTATUS[0]}
if [ "$1" = "pre-update" ]; then collect_db_backups "$out"; fi
else
out=""
# No pipe here, so `$?` is the hook's own status already (it always was on
# this branch); kept explicit so the two branches cannot drift apart again.
"$hook" || rc=$?
warn "could not capture the ${1} hook's output (no temp file) — the recovery block will name no backup path"
fi
# Not "(continuing)": whether a failed hook ends the run is the CALLER's
# decision (fatal for pre-update, a reported failure for post-update), so
# this function only reports the status and returns it.
if [ "$rc" -ne 0 ]; then warn "${1} hook failed (exit ${rc})"; fi
fi
return "$rc"
}
# ── Rollback guidance (no automatic revert) ────────────────────────────────
# shell_quote STRING — single-quoted, so a path containing a space or a quote
# survives being pasted into a shell. The printed commands are run by a human,
# and a path from config.json is operator content.
shell_quote() {
local s="$1"
printf "'%s'" "${s//\'/\'\\\'\'}"
}
# print_db_restore — the mandatory DB restore, naming the REAL backup files.
#
# The payload is three files, not one commit, and migrations are forward-only
# (src/db/init.ts:127), so a rollback MUST restore the DB backup as well
# (ADR 0001 §2.9). The old text named `data/synaptomind.db.backup/
# synaptomind.db.<timestamp>.bak` — a template, not a file, and one the operator
# had to guess a timestamp into — while claiming the hook "printed each backup
# path" instead of printing the paths it had.
print_db_restore() {
local b dir db
if [ "${#DB_BACKUPS[@]}" -eq 0 ]; then
warn " # no database backup was reported by the pre-update hook, so there is"
warn " # nothing to restore here. On a host that HAS backups, newest first:"
warn " sudo ls -lt ${INSTALL_DIR}/data/synaptomind.db.backup/*.bak ${INSTALL_DIR}/data/*.backup/*.bak"
else
for b in "${DB_BACKUPS[@]}"; do
# The hook writes <db>.backup/<name>.<timestamp>.bak (hooks/pre-update), so
# the database a backup belongs to is the DIRECTORY minus its .backup
# suffix. Deriving it from the shape the hook itself writes is what keeps
# this command runnable for a database that does not live under data/ at
# all, which a re-derived hardcoded path never was.
dir="${b%/*}"
db="${dir%.backup}"
if [ "$dir" = "$b" ] || [ "$db" = "$dir" ] || [ "${b##*/}" = "$b" ]; then
# Not that shape: name the file and say that its database cannot be
# derived, rather than printing a guess that would restore over the
# wrong file.
warn " # not a <db>.backup/<name>.bak path — which database it backs up is unknown:"
warn " # sudo cp -p $(shell_quote "$b") <that database>"
continue
fi
warn " sudo cp -p $(shell_quote "$b") $(shell_quote "$db")"
warn " sudo rm -f $(shell_quote "${db}-wal") $(shell_quote "${db}-shm")"
done
fi
warn " Restoring the DB is mandatory: migrations are forward-only (src/db/init.ts),"
warn " so a pre-upgrade binary against a post-upgrade schema is unsafe. Every"
warn " database the pre-update hook backed up is listed above — those are the paths"
warn " it reported, not a template."
warn " A .prev kept only because it was byte-identical restores THIS version rather"
warn " than an older one (see binary_keep_previous) — check it before relying on it."
}
# print_unverified — the verdict when the service never answered.
#
# Printed INSTEAD of the rollback block, which used to be printed for exactly this
# case: a service that was merely slow (or an HEALTH_TIMEOUT of 2) was handed a
# remedy that stops the unit, moves the .prev payload back and restores the
# database over the new schema's. Nothing destructive is advised for a silence —
# the payload that was just installed is not evidence of anything, and the
# rollback point stays on disk, named, for an operator who has established that
# the service really is broken (task #1079 F4).
print_unverified() {
local f port
port="$(health_url_port "${HEALTH_URL}")"
warn "the service did not confirm the new version — this update is UNVERIFIED, not failed."
if [ "${HEALTH_RECHECKED:-false}" = true ]; then
warn " ${HEALTH_URL} did not answer as ${APP_NAME} in ${HEALTH_TIMEOUT}s, nor in the"
warn " ${HEALTH_CONFIRM_TIMEOUT}s re-check that followed it."
else
warn " ${HEALTH_URL} did not answer as ${APP_NAME} within ${HEALTH_TIMEOUT}s."
fi
warn " what IS done: the payload is swapped in, the unit is refreshed and the"
warn " pre-update hook ran. Nothing was rolled back, nothing was restored."
warn " check, in this order:"
warn " sudo systemctl status ${APP_NAME}"
warn " journalctl -u ${APP_NAME} -n 100 --no-pager"
warn " curl -sS ${HEALTH_URL} # does OUR service answer?"
if [ -n "$port" ]; then
warn " sudo ss -ltnp | grep ':${port}' # or something else on that port?"
fi
warn " a slow start is the likeliest cause. Raise HEALTH_TIMEOUT (or"
warn " HEALTH_CONFIRM_TIMEOUT) in ${RUN_DIR}/scripts/app.env and re-run rather than"
warn " reverting a payload that may well be serving."
if [ "$DIST" = "binary" ]; then
warn " if the service does turn out to be broken, the rollback point is still here:"
for f in $BINARY_ROLLBACK_FILES; do
if [ -f "${INSTALL_DIR}/${f}.prev" ]; then warn " ${INSTALL_DIR}/${f}.prev"; fi
done
else
warn " if the service does turn out to be broken, the previous commit is ${PREV_REF}."
fi
if [ "${#DB_BACKUPS[@]}" -gt 0 ]; then
warn " the pre-update database backup(s) are untouched:"
for f in ${DB_BACKUPS[@]+"${DB_BACKUPS[@]}"}; do warn " ${f}"; done
fi
return 0
}
# print_recovery CONFIRMED|UNCONFIRMED
# The remedy for an OBSERVED failure only. A timeout is not observed (see
# print_unverified), so it must never reach this block.
print_recovery() {
local verdict="${1:-confirmed}"
if [ "$verdict" != "confirmed" ]; then
print_unverified
return 0
fi
warn "update did not finish cleanly — previous state:"
if [ "$DIST" = "binary" ]; then
warn " rollback:"
warn " sudo systemctl stop ${APP_NAME}"
warn " cd ${INSTALL_DIR}"
warn " for f in ${BINARY_ROLLBACK_FILES}; do [ -f \"\$f.prev\" ] && sudo mv -f \"\$f.prev\" \"\$f\"; done"
print_db_restore
warn " sudo systemctl start ${APP_NAME}"
else
# The remedy runs the SAME pair the update ran, from the same keys. It used
# to print a hardcoded `bun install --frozen-lockfile --production`, which
# reproduces the very defect gap (a) closes: an app whose build needs its
# devDependencies gets them pruned by --production, so the operator follows
# the printed remedy, gets a payload that cannot start, and has no way to
# tell that the remedy — not the update — is what broke it. `bun` and the
# flags are the operator's to type; the values are not.
# shellcheck disable=SC2086
local remedy="git -C ${INSTALL_DIR} checkout --force ${PREV_REF} && (cd ${INSTALL_DIR} && bun install ${INSTALL_FLAGS}"
[ -n "${BUILD_CMD:-}" ] && remedy="${remedy} && ${BUILD_CMD}"
warn " previous commit: ${PREV_REF}"
warn " rollback: ${remedy})"
fi
return 0
}
# ── restart policy: the source-mode half of the unit refresh ────────────────
# Deliver the template's restart policy to a SOURCE-mode host.
#
# refresh_unit() re-renders the whole unit in binary mode, which carries the
# Restart= line with it. Source mode returns early there, because a full
# re-render would clobber an operator's hand edits — and that made the policy
# UNDELIVERABLE to exactly the hosts that need it: production is DIST=source
# (ExecStart=bun run start), so after the 2026-09-30 outage (an agent's
# name-pattern pkill; the process handled SIGTERM and exited 0; Restart=
# on-failure then left it down for 12 minutes) an update would have kept
# Restart=on-failure forever.
#
# So source mode gets a SURGICAL refresh: the existing Restart= line is rewritten
# in place, and only when it differs. Every other byte of the unit — an operator's
# hand edits included — is preserved, which is the invariant the early return was
# protecting. Nothing is ever INSERTED: a unit with no Restart= line is left
# alone and warned about rather than having a directive appended into an unknown
# section.
#
# Never fatal, unlike refresh_unit: an undelivered restart policy leaves a
# running service (the pre-incident behaviour), whereas failing here would block
# updates outright on a host that is otherwise fine. Every path returns 0.
#
# The write is ATOMIC and the messages describe the unit ON DISK, never the one
# this function meant to write. Both were defects of the first version, found in
# review of d9ff4bb:
# * `cp -f "$tmp" "$unit"` opens the destination O_TRUNC and then writes, so a
# copy that died partway (ENOSPC/EIO/killed — the class of event that took
# production down on 2026-09-30) left the live unit truncated at 24 bytes,
# while the message said "unchanged ... it still says Restart=on-failure".
# Both halves were false: the file no longer contained Restart= at all, and
# the staged good copy was rm -rf'd on the way out. Now the new body is
# staged beside the unit and swapped in with rename(2), so the unit is
# either the old file or the new one — never a hybrid — and the previous
# body is kept as <unit>.bak the way the binary path keeps one.
# That staging lives in lib/common.sh's write_file_atomically(), which
# install.sh and refresh_unit() below also use: the same O_TRUNC pattern
# survived in those two entry points (task #1094) precisely because the
# mechanism was inlined here instead of shared, so it is shared now.
# * A unit's mode was forced to 644, which widened a hand-edited unit that may
# carry Environment= secrets. cp -f over an existing file does not change
# its mode, so the chmod bought nothing and only ever lost the operator's
# choice. The mode is now read from the unit and re-applied to the STAGED
# file by write_file_atomically(), which also stages that file 600 BEFORE
# any byte lands in it and widens it only after the copy succeeded — the
# staging file used to inherit the awk render's 644 (umask 022), so a failed
# stage left the start of a 600 unit, Environment= secrets included, in a
# world-readable file in the unit directory, and it was left there (task
# #1094: both findings reproduced, both now covered by the shared function).
ensure_restart_policy() {
local unit="$1" body current backup now tmpdir tmp
[ -e "$unit" ] || return 0
# Read the unit ONCE and tell a read FAILURE apart from an absent directive:
# the greps used to swallow their errors, so a mode-000 unit came back with an
# empty result and was reported as "declares no Restart= line" — with a remedy
# (edit the unit) the operator cannot apply for the very permission reason that
# made it unreadable.
if ! body="$(cat -- "$unit" 2>/dev/null)"; then
warn "Restart policy unchanged in ${unit}: it could not be READ."
warn " permissions, not a missing directive — this run wrote nothing."
warn " remedy: ls -l ${unit} && sudo chown root:root ${unit} && sudo chmod 644 ${unit}"
return 0
fi
# The EFFECTIVE directive, not the first line that matches: systemd honours the
# LAST Restart= it reads, so a unit with Restart=always followed by Restart=no
# is a unit on `no`. Matching a line left that host silently on the outage
# shape this function exists to prevent — no write, no warning, exit 0.
current="$(printf '%s\n' "$body" | grep -E '^[[:space:]]*Restart=' | tail -n 1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' || true)"
if [ "$current" = "Restart=always" ]; then
return 0
fi
if [ -z "$current" ]; then
# No Restart= line at all: systemd's default is "no restart", the same
# outage shape. Refuse to guess where the directive belongs.
warn "Restart policy unchanged in ${unit}: it declares no Restart= line."
warn " a signalled or cleanly-exited process would then NOT be restarted."
warn " remedy: add 'Restart=always' under [Service] in ${unit}, then sudo systemctl daemon-reload"
return 0
fi
if ! command -v systemctl >/dev/null 2>&1; then
warn "Restart policy unchanged in ${unit}: systemctl is not on PATH."
warn " it still says ${current}; remedy: sudo systemctl daemon-reload after editing it"
return 0
fi
if [ "$(id -u)" -ne 0 ] && ! command -v sudo >/dev/null 2>&1; then
warn "Restart policy unchanged in ${unit}: writing it needs root and sudo is not available."
warn " it still says ${current}; remedy: sudo sed -i 's/^Restart=.*/Restart=always/' ${unit} && sudo systemctl daemon-reload"
return 0
fi
# Duplicate Restart= lines are COLLAPSED onto a single Restart=always, in the
# position and indentation of the first one. Leaving them as several identical
# lines would be behaviourally correct (systemd takes the last) but leaves a
# unit whose directive count silently grew with each rewrite; and a duplicate
# set is itself how the last-wins case got here. Every other byte — an
# operator's hand edits, their Environment= lines, their comments — is carried
# through untouched.
mktemp_owned tmpdir -d || { warn "Restart policy unchanged in ${unit}: no temp dir (TMPDIR unwritable?); it still says ${current}."; return 0; }
tmp="${tmpdir}/${APP_NAME}.service"
if ! awk '
/^[[:space:]]*Restart=/ {
if (held) next # a later duplicate: drop it
held = 1
line = $0
sub(/^[[:space:]]*Restart=.*/, "Restart=always", line)
next
}
{ if (held) { print line; held = 0 }; print }
END { if (held) print line }
' "$unit" > "$tmp" 2>/dev/null; then
warn "Restart policy unchanged in ${unit}: cannot rewrite the unit (transform failed); it still says ${current}."
return 0
fi
# Stage BESIDE the unit, under a name nothing loads, and swap with rename(2).
# write_file_atomically() owns the whole sequence (empty 600 staging file, cp
# onto it, mode widened only on success, rename, staging file removed on every
# failure) because each of those steps was a separate finding: a staging file
# that inherited the awk render's 644 mode carried the first bytes of a 600
# unit — Environment= secrets included — world-readable in the unit directory,
# and a FAILED stage left that partial file behind.
#
# The previous body, kept for recovery: a swap that turns out to be wrong is
# only recoverable if the old file still exists somewhere. It goes through the
# same atomic path, so a .bak is never itself a truncated file.
backup=""
if write_file_atomically "${unit}.bak" "$unit"; then
backup="${unit}.bak"
else
warn " proceeding WITHOUT a recovery copy of the previous unit."
fi
if write_file_atomically "$unit" "$tmp"; then
# Report what is on disk NOW, not what was intended: if something replaced
# the unit underneath us (a concurrent install), say so.
now="$(unit_restart_state "$unit")"
if [ "$now" = "Restart=always" ]; then
if run_root systemctl daemon-reload; then
info "Restart policy: ${unit} now carries Restart=always (was: ${current})"
if [ -n "$backup" ]; then info " previous unit kept at ${backup}"; fi
else
warn "Restart policy written to ${unit} (it now carries Restart=always), but systemctl daemon-reload failed."
warn " systemd still has the old policy in memory until it reloads; the service keeps running."
if [ -n "$backup" ]; then warn " previous unit kept at ${backup}"; fi
fi
else
warn "Restart policy written to ${unit}, but it now says: ${now}."
warn " something replaced the file during this run; leaving it to the operator."
fi
else
# The unit is untouched, so it still says what it said: report that, not the
# directive this run meant to write.
warn "Restart policy unchanged in ${unit}: ${ATOMIC_WRITE_REASON}."
warn " it still says ${current}; the unit was NOT touched."
if [ -n "$backup" ]; then warn " previous unit kept at ${backup}"; fi
fi
return 0
}
# What the unit on disk actually says, as ONE line, for a message that must
# report the real state: the effective (last) Restart= directive, or a plain
# statement of why it cannot be read.
unit_restart_state() {
local unit="$1" body
if [ ! -e "$unit" ]; then printf 'the file is gone'; return 0; fi
if ! body="$(cat -- "$unit" 2>/dev/null)"; then printf 'it cannot be READ'; return 0; fi
local line
line="$(printf '%s\n' "$body" | grep -E '^[[:space:]]*Restart=' | tail -n 1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' || true)"
if [ -z "$line" ]; then printf 'no Restart= line'; else printf '%s' "$line"; fi
}
# ── systemd unit ────────────────────────────────────────────────────────────
# render_systemd_unit() has exactly TWO call sites in the framework: install.sh's
# install_service() and this refresh_unit(). (The note here used to say ONE and
# named install.sh alone. The invariant it argues for — one renderer, so a fix
# cannot reach one entry point and miss the other — was always intact; the COUNT
# was wrong, and it was wrong in the direction that hides work: an audit reading
# "one call site" concludes the update path cannot change the body, and stops
# looking. This function is the second site, and it is the only one that runs on
# every update.)
#
# A unit written before a line existed — the LD_LIBRARY_PATH line, and now the
# Group=/StateDirectory=/EnvironmentFile=/bounded-stop lines — therefore kept its
# old body through every update, so a host that updates INTO binary mode would
# run a binary whose embedder cannot dlopen libonnxruntime.so.1 (ADR 0001 §2.2).
# Binary mode re-renders and reinstalls the unit here.
#
# Scope is deliberately narrow: source mode returns immediately, because a full
# re-render would clobber an operator's hand edits to the unit. (The other half of
# that justification — "the rendered body is unchanged for source mode" — stopped
# being true once the template gained its Restart= line, so source mode no longer
# returns without an effect: it delegates to ensure_restart_policy above, which
# rewrites that one line and nothing else. A full re-render is still refused.)
#
# In binary mode a refresh that did NOT happen is FATAL (returns 1, and main()
# aborts before the restart). Rationale: the payload has already been swapped at
# this point, so a unit left without Environment=LD_LIBRARY_PATH is a unit that
# starts the new binary with an embedder which dies on ERR_DLOPEN_FAILED — and
# /health still answers status "ok", so nothing downstream would notice. The one
# case that stays non-fatal is a host with NO unit on disk (a --no-service or
# container install): there is nothing there to go stale.
refresh_unit() {
# UNIT_FILE is overridable so a non-standard unit path (and the deploy tests)
# need not write to /etc/systemd/system.
local unit="${UNIT_FILE:-/etc/systemd/system/${APP_NAME}.service}" tmpdir tmp saved
# Source mode takes the surgical path instead of a full re-render, so the
# template's restart policy still reaches a source host without clobbering a
# hand-edited unit. It cannot fail the update.
if [ "$DIST" != "binary" ]; then ensure_restart_policy "$unit"; return 0; fi
# Render first, unconditionally: it needs neither root nor systemd, and the
# rendered body is what the failure message has to point at.
#
# The render REFUSES (returns 1) rather than exiting, so this path returns 1
# like every other refresh failure and main() still prints the recovery block.
# It used to `error` from inside the renderer, which is exit(1): the payload
# was already swapped, and the operator got neither the rollback block nor the
# warning that a re-run will not fix it — and then could not fix it.
# The render dir is owned from creation (mktemp_owned), so every path below
# that returns or aborts releases it through the EXIT trap. The ONE exception
# is the hand-off below, which says so out loud with cleanup_release.
mktemp_owned tmpdir -d
tmp="${tmpdir}/${APP_NAME}.service"
if ! render_systemd_unit "$EXEC_START" > "$tmp"; then
unit_not_rendered "$unit"
return 1
fi
if command -v systemd-analyze >/dev/null 2>&1; then
systemd-analyze verify "$tmp" >/dev/null 2>&1 || warn "systemd-analyze verify reported issues"
fi
# Keep a copy for the remedy. A plain re-run cannot fix a failed refresh —
# main()'s "Already up to date" guard returns before refresh_unit is reached
# again — so the operator needs the rendered unit as a file to install.
# When RUN_DIR is unwritable the render dir itself must SURVIVE the EXIT trap,
# or the remedy would name a path the trap just deleted — so the ownership is
# released explicitly here rather than never taken, which is what made the
# difference between this path and a forgotten registration invisible.
saved="${RUN_DIR}/unit-refresh/${APP_NAME}.service"
if mkdir -p "${RUN_DIR}/unit-refresh" 2>/dev/null && cp -f "$tmp" "$saved" 2>/dev/null; then
: # the durable copy exists; the trap still owns the render dir
else
saved="$tmp"
cleanup_release "$tmpdir"
fi
local blocked=""
if ! command -v systemctl >/dev/null 2>&1; then
blocked="systemctl is not on PATH"
elif ! systemd_running; then
blocked="systemd is not running"
elif [ "$(id -u)" -ne 0 ] && ! command -v sudo >/dev/null 2>&1; then
blocked="sudo is not available, and writing ${unit} needs root"
fi
if [ -n "$blocked" ]; then
if [ ! -e "$unit" ]; then
info "No systemd unit at ${unit} — nothing to refresh; start ${EXEC_START} by hand."
rm -f "$saved" "$tmp"
return 0
fi
unit_not_refreshed "$unit" "$blocked" "$saved"
return 1
fi
# The previous body is kept, so a rendered unit that turns out to be wrong is
# recoverable: main()'s "Already up to date" guard returns before
# refresh_unit is reached again, so a plain re-run cannot fix it.
if [ -e "$unit" ] && ! write_file_atomically "${unit}.bak" "$unit"; then
warn " proceeding WITHOUT a recovery copy of the unit this refresh replaces."
fi
# `enable` is deliberately NOT repeated: that is install.sh's job.
# Atomic replacement, not `cp -f` + chmod 644: that pair opened the LIVE unit
# O_TRUNC, so a copy dying partway (ENOSPC, EIO, killed) left a 24-byte stub
# where a hand-edited unit was — while this very function went on to report
# the unit as "left unchanged", and while the forced 644 widened a unit that
# may carry Environment= secrets. Staged beside the unit and swapped with
# rename(2), so a failed write is a no-op and the mode is preserved.
if write_file_atomically "$unit" "$tmp"; then
# The unit is correct on disk, but until systemd has read it a restart would
# apply the OLD body — the same failure one step later, so it is fatal too.
if ! run_root systemctl daemon-reload; then
unit_not_refreshed "$unit" "systemctl daemon-reload failed after the unit was written" ""
return 1
fi
rm -f "$saved" "$tmp"
info "Refreshed ${unit}"
else
unit_not_refreshed "$unit" "the unit was NOT replaced — ${ATOMIC_WRITE_REASON}" "$saved"
return 1
fi
}
# A unit is installed that this update could not refresh, so the payload just
# swapped in would start under a stale unit. Names what failed, the line at
# stake, the difference between the two units, and a remedy. Prints only.
#
# Split in two so the remedy can differ while the CONSEQUENCE cannot: the unit
# on disk is the previous body in both cases, and only one of them has a
# rendered unit to hand over. See unit_not_rendered.
_unit_stale_warning() {
local unit="$1" reason="$2"
warn "systemd unit NOT refreshed: ${reason}"
warn " installed: ${unit} (left unchanged)"
warn " required: Environment=LD_LIBRARY_PATH=${INSTALL_DIR}/lib"
warn " the payload just installed cannot dlopen lib/libonnxruntime.so.1 without"
warn " that line: the embedder child dies on ERR_DLOPEN_FAILED while /health"
warn " still reports status ok, so nothing else in this run would notice."
}
# What the operator must not do while the unit is stale, whatever stopped the
# refresh. A re-run does not reach the refresh: the payload is already swapped,
# so main() reports the version as already up to date and returns 0.
_unit_stale_tail() {
warn " the running service still serves the PREVIOUS payload; do not restart it"
warn " until the unit is fixed, and re-running update.sh will NOT fix it (it"
warn " reports the version as already up to date before refreshing anything)."
}
unit_not_refreshed() {
local unit="$1" reason="$2" saved="$3"
_unit_stale_warning "$unit" "$reason"
if [ -n "$saved" ]; then
warn " rendered: ${saved} (this update's unit — diff it against the installed one)"
warn " remedy: sudo install -m 644 ${saved} ${unit}"
warn " sudo systemctl daemon-reload"
else
# The unit on disk is already correct; systemd simply has not read it yet.
warn " the unit on disk IS this update's unit — systemd has not reloaded it"
warn " remedy: sudo systemctl daemon-reload"
fi
warn " sudo systemctl restart ${APP_NAME}"
_unit_stale_tail
}
# The unit could not be RENDERED at all: a value in app.env would not survive
# systemd's parser (see unit_value_defect), so there is no new body to install
# and — unlike every other failure here — no rendered file to hand over. The
# remedy is therefore not "install this unit over that one": the value has to be
# fixed first, and then something has to re-render. install.sh is that something
# (it re-renders from app.env and re-installs the unit); a plain re-run of
# update.sh is not, for the reason in _unit_stale_tail.
unit_not_rendered() {
local unit="$1"
_unit_stale_warning "$unit" "the unit on disk is the PREVIOUS body — it was NOT re-rendered, because a value in app.env would fold, forge or rewrite a directive (named above)"
warn " rendered: nothing — the refusal happens before any unit text is written,"
warn " so there is no file to install over the installed one"
# No rendered file to hand over, so the remedy cannot be "install this unit
# over that one". The value has to be fixed first, and then something has to
# re-render — and that something is install.sh, not a re-run of this script
# (see _unit_stale_tail). install.sh is NOT named by path: it does not copy
# itself into ${RUN_DIR}/scripts, so a path printed here would not exist.
warn " remedy: fix the value named above in ${RUN_DIR}/scripts/app.env, then"
warn " re-run install.sh (the deploy/ copy this update came from) — it"
warn " re-renders the unit from the fixed value and installs it"
warn " or add the one line above by hand, then:"
warn " sudo systemctl daemon-reload"
warn " sudo systemctl restart ${APP_NAME}"
_unit_stale_tail
}
# ── Fetch & swap ───────────────────────────────────────────────────────────
update_source() {
local bun=""
# FIRST, before `git fetch` touches .git at all. The three ownership calls in
# this function are each load-bearing at a different point, and this one is
# about a residue the LATER two cannot reach: a tree left root-owned by an
# earlier run (#1114 left /opt/subagentix/.git/index root:root, which is the
# exact mismatch ownership_mismatch exists to catch). update.sh is run by the
# OPERATOR, not as root, so on such a tree `git fetch` below fails with an
# EACCES on an index it cannot write — and a failure there aborts before the
# post-checkout and post-build calls below ever run. Applying ownership first
# is what lets this run heal the residue it is otherwise killed by; on a clean
# tree ownership_mismatch walks it read-only and finds nothing to do.
apply_ownership
git -C "$INSTALL_DIR" fetch --tags --force origin
if [ -n "$ARG_VERSION" ]; then
TARGET_REF="$ARG_VERSION"; TARGET_KIND="tag"
else
resolve_source_ref "$INSTALL_DIR"
if [ "$TARGET_KIND" = "branch" ]; then
local head want
head="$(git -C "$INSTALL_DIR" rev-parse HEAD 2>/dev/null || true)"
want="$(git -C "$INSTALL_DIR" rev-parse "origin/${TARGET_REF}" 2>/dev/null || true)"
if [ -n "$head" ] && [ "$head" = "$want" ]; then
info "Already up to date (${TARGET_REF} @ ${head:0:8})."
exit 0
fi
fi
fi
PREV_REF="$(git -C "$INSTALL_DIR" rev-parse --short HEAD 2>/dev/null || echo unknown)"
if [ "$TARGET_KIND" = "tag" ]; then
git -C "$INSTALL_DIR" checkout --force "$TARGET_REF"
else
git -C "$INSTALL_DIR" checkout --force -B "$TARGET_REF" --track "origin/${TARGET_REF}"
fi
info "Checked out ${TARGET_REF}"
# AFTER the checkout, and this is the call that covers .git itself. The
# checkout above rewrote the working tree AND .git/index, so a run as root
# leaves the service user unable to run `git status` in its own install dir
# even though every tracked file is chowned. install.sh gets this for free
# (its clone is immediately followed by apply_ownership); update.sh had no
# equivalent, which is how the residue survived an update. It is also still
# required BEFORE the build: the build runs as TARGET_USER and writes into this
# tree, and a tree chowned only at the top directory fails with an EACCES
# from inside a bundler that names neither this nor the directory.
apply_ownership
# The SAME pair install.sh runs, through the same two functions: the same
# INSTALL_FLAGS, the same BUILD_CMD, the same BUILD_TIMEOUT, the same user. A
# source update that skipped the build (or installed production-only
# dependencies for an app whose build needs its devDependencies) would check
# out new code and then restart into a payload it cannot run — and the health
# gate would report a timeout, naming neither the build nor the flags.
install_deps_and_build
# AFTER the build, for the outputs rather than the checkout. install.sh applies
# ownership twice on the source path for exactly this reason: once before the
# build so the build can write, once after so what the build WROTE is covered.
# update.sh had only the first, so anything install_deps and the build created
# (node_modules, build/, .svelte-kit) kept the ownership of whoever created it,
# and apply_ownership at the call site (update_binary/update_source dispatch)
# runs before the post-update hook — the hook is documented as free to write
# into INSTALL_DIR or DATA_DIR — so this call also gives the hook a tree that
# is already consistent with what the service will see.
apply_ownership
}
update_binary() {
binary_install_payload
}
# ── Restart & verify ───────────────────────────────────────────────────────
# Set to true when the health verdict was re-checked, so the UNVERIFIED report
# can say that both windows were used (task #1079 F4).
HEALTH_RECHECKED=false
restart_and_verify() {
local restarted=false expected="$TARGET_VERSION"
if [ "$expected" = "unknown" ]; then expected=""; fi
# Never poll without an expectation when one can be derived. A version-less
# gate accepts any well-formed /health on the port, which is how a proxy — or
# a second instance of this app — passed as the service this update installed
# (F4). The artefact just swapped in is the authority, and it is what /health
# must report back.
if [ -z "$expected" ]; then
installed_version
expected="$INSTALLED_VERSION"
if [ -z "$expected" ]; then
warn "the target version is unknown and cannot be read from ${INSTALL_DIR} —"
warn " the health check can no longer tell this service from another one on ${HEALTH_URL}."
fi
fi
if systemd_running && systemctl is-active "$APP_NAME" >/dev/null 2>&1; then
info "Restarting ${APP_NAME}..."
run_root systemctl restart "$APP_NAME"
restarted=true
else
info "${APP_NAME} is not running under systemd — start it manually:"
info " sudo systemctl start ${APP_NAME}"
fi
if [ "$restarted" = true ]; then
if wait_health "$HEALTH_URL" "$expected" "$HEALTH_TIMEOUT"; then
return 0
fi
# A timeout is this run's own bound, not a fact about the service: with
# HEALTH_TIMEOUT=2 a healthy upgrade was declared failed — with a remedy
# that stops the unit and restores the database — while the service went on
# answering eight seconds later. Re-check before deciding (F4).
if [ "$HEALTH_FAILURE" = "timeout" ]; then
if health_recheck "$HEALTH_URL" "$expected" "$HEALTH_CONFIRM_TIMEOUT"; then
return 0
fi
HEALTH_RECHECKED=true
fi
# Only a sample that CONTRADICTED the update is a verdict; a silence is not.
if [ "$HEALTH_FAILURE" = "timeout" ]; then
print_recovery unconfirmed
else
print_recovery confirmed
fi
exit 1
fi
return 0
}
# ── Main ───────────────────────────────────────────────────────────────────
main() {
parse_args "$@"
load_app_env
[ -n "${APP_NAME:-}" ] || error "APP_NAME is not set in app.env"
detect_os
detect_arch
resolve_target_user
INSTALL_DIR="${INSTALL_DIR:-/opt/${APP_NAME}}"
PORT="${PORT:-3000}"
HEALTH_TIMEOUT="${HEALTH_TIMEOUT:-60}"
# A second health window, used only when the first one expired with no answer
# (task #1079 F4). 0 disables the re-check, which is NOT recommended: it is
# the difference between "unverified" and a destructive remedy.
HEALTH_CONFIRM_TIMEOUT="${HEALTH_CONFIRM_TIMEOUT:-30}"
HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:$(resolve_port)/health}"
HOOKS_DIR="${HOOKS_DIR:-${RUN_DIR}/hooks}"
# Before the fetch, and before the pre-update hook: an unreadable health
# contract is a configuration error, and refusing it here means the operator
# has no database backup to reason about afterwards.
require_health_contract
# Export RUN_DIR so hook scripts (pre-update / post-update) can locate
# ${RUN_DIR}/scripts/app.env even when update.sh does not pass it explicitly.
export RUN_DIR
if [ "$CHECK_BACKUP" = "true" ]; then
service_unit_context || true
service_environment || true
local tmp
tmp="$(mktemp)" || error "cannot create temp file"
if ! effective_db_paths > "$tmp" 2>"$tmp.err"; then
cat "$tmp.err" >&2 2>/dev/null || true
rm -f "$tmp" "$tmp.err"
error "backup resolution failed"
exit 1
fi
if [ -s "$tmp.err" ]; then cat "$tmp.err" >&2; fi
echo "CONTEXT_SOURCE=${CONTEXT_SOURCE:-unknown}"
echo "ENV_SOURCE=${ENV_SOURCE:-unknown}"
echo "RESOLVER_AGREEMENT=${RESOLVER_AGREEMENT:-bash-only}"
echo "EFFECTIVE_CONFIG_PATH=${EFFECTIVE_CONFIG_PATH:-}"
echo "DATABASES:"
cat "$tmp"
rm -f "$tmp" "$tmp.err"
exit 0
fi
if [ "$DIST" = "source" ]; then
[ -d "${INSTALL_DIR}/.git" ] || error "not installed at ${INSTALL_DIR} — run install.sh first"
need_cmd git
elif [ "$DIST" = "binary" ]; then
[ -x "${INSTALL_DIR}/${APP_NAME}" ] || error "not installed at ${INSTALL_DIR} — run install.sh first"
command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1 || error "need curl or wget"
else
error "DIST must be 'source' or 'binary' (got '${DIST}')"
fi
CURRENT="$(current_version)"
CURRENT="${CURRENT:-unknown}"
# -- resolve target --
if [ "$DIST" = "binary" ]; then
if [ -n "$ARG_VERSION" ]; then
TAG="$(normalize_v "$ARG_VERSION")"
else
release_resolve_tag
TAG="$RESOLVED_TAG"
fi
TARGET_VERSION="${TAG#v}"
TARGET_REF="$TAG"
# refresh_unit needs it; binary mode has no bun, so the default is the
# artefact itself (mirrors install.sh default_exec_start).
[ -n "${EXEC_START:-}" ] || EXEC_START="${INSTALL_DIR}/${APP_NAME}"
else
TARGET_VERSION=""
TARGET_REF=""
# BEFORE the fetch below, which is this run's FIRST write into INSTALL_DIR and
# so the first one a tree the caller cannot write refuses (#1139 — the residual
# #1137 flagged from #1136). update.sh is run by the OPERATOR, not as root, and
# the tree it is updating is one it may have had no hand in: a root-owned
# residue like #1114's (/opt/subagentix/.git/index root:root) makes this fetch
# die with an EACCES on an index it cannot write — and it dies BEFORE
# update_source()'s own pre-fetch call is reached, so the run is killed by the
# very residue it exists to heal. Placing it here also puts it ahead of the
# pre-update hook, whose database backup writes into DATA_DIR. On a clean tree
# ownership_mismatch walks the tree read-only, finds nothing and says nothing.
apply_ownership
git -C "$INSTALL_DIR" fetch --tags --force origin
if [ -n "$ARG_VERSION" ]; then
ARG_VERSION="$(normalize_v "$ARG_VERSION")"
TARGET_REF="$ARG_VERSION"; TARGET_KIND="tag"; TARGET_VERSION="${ARG_VERSION#v}"
else
resolve_source_ref "$INSTALL_DIR"
if [ "$TARGET_KIND" = "tag" ]; then
TARGET_VERSION="${TARGET_REF#v}"
else
remote_pkg_version "$TARGET_REF"
TARGET_VERSION="${REMOTE_PKG_VERSION:-unknown}"
fi
fi
fi
info "Current: ${CURRENT}"
info "Target: ${TARGET_VERSION:-unknown}"
# -- guards --
if [ -n "$TARGET_VERSION" ] && [ "$TARGET_VERSION" != "unknown" ] && [ "$CURRENT" != "unknown" ]; then
# BOTH operands normalised. current_version() returns app_version's
# v-prefixed output for a binary install but a bare package.json version for
# a source one, and TARGET_VERSION is bare — so without this,
# `ver_cmp "v0.8.0" "0.9.0"` answers "newer" and a plain upgrade is
# misreported as a downgrade needing confirmation (ADR 0001 §2.6).
case "$(ver_cmp "$(normalize_v "$CURRENT")" "$(normalize_v "$TARGET_VERSION")")" in
same)
info "Already up to date."
exit 0 ;;
newer)
info "installed ${CURRENT} is newer than ${TARGET_VERSION} (downgrade)."
if ! confirm "Downgrade ${CURRENT} -> ${TARGET_VERSION}?"; then
info "Aborted."
exit 0
fi ;;
esac
fi
# -- hooks around the swap --
# SynaptoMind deviation: pre-update is treated as FATAL — a failed DB backup
# must never be followed by an unchecked code swap. The upstream template
# treats all hooks as non-fatal (warn + continue).
#
# This guard was UNREACHABLE until task #1102: run_hook could not carry a hook
# status out of its tee pipeline, so the `if !` here always saw success. It is
# now live, and `error` (not a bare exit) so the operator gets the same
# "[app] ERROR:" line as every other fatal path. Aborting HERE is also the only
# safe point: nothing has been swapped, the unit has not been touched and the
# service is still running the code it was running a minute ago, so there is
# nothing to recover — print_recovery would only name backups of an install
# that is untouched.
# Built-in fail-closed backup (task #2297): runs before the hook, in the
# release code every update fetches. Aborts before any swap on failure.
if ! backup_databases; then
error "pre-update backup failed — aborting before switching code"
fi
export APP_BACKUP_ALREADY_DONE=1
if ! run_hook pre-update; then
error "pre-update hook failed — aborting before switching code"
fi
if [ "$DIST" = "binary" ]; then update_binary; else update_source; fi
# AFTER the swap, and this is the #1113 fix. update_binary() →
# binary_swap_payload() moves the payload in with `mv` as root, and
# update_source()'s `git checkout --force` rewrites files as root, so both
# modes leave root-owned files in INSTALL_DIR. update.sh is run by the
# operator (bash ${RUN_DIR}/scripts/updater.sh) — not necessarily as root —
# and only one implementation of ownership existed, in install.sh, so an update
# could not fix what it had just created: the swapped 0.1.9 binary stayed
# root-owned until an operator re-applied it by hand.
#
# Placed before the post-update hook so a hook that writes into INSTALL_DIR or
# DATA_DIR (where migrations will live) sees the same ownership the service
# will, and before refresh_unit/restart so the running service never touches a
# file whose ownership changes under it.
apply_ownership
# DELIBERATE DECISION (task #1102): a failed post-update hook does NOT abort
# mid-flight. By the time it runs the payload is already swapped, so aborting
# here would leave the host WORSE off than continuing: the unit would not be
# refreshed and the service would not be restarted, i.e. a swapped payload
# that nothing has ever executed, with no health verdict and no recovery block.
# The rest of the run (refresh_unit → restart_and_verify) is exactly the
# evidence an operator needs, so it still runs.
#
# It is also not reported as success. The status is remembered and turned into
# a non-zero exit AFTER the restart and the health gate, because by then the
# honest verdict is "the update landed, and a step after the swap did not":
# post-update is where future migrations would live (the shipped hook is a
# placeholder), and a silently-ignored failure there is how an operator ends up
# with code that reports healthy while a migration never ran.
POST_UPDATE_RC=0
run_hook post-update || POST_UPDATE_RC=$?
# -- refresh unit, restart & health --
# The unit is refreshed BEFORE the restart, so the service comes back with the
# Environment=LD_LIBRARY_PATH a binary payload needs. A refresh that failed is
# fatal in binary mode (refresh_unit returns 1): the payload is already
# swapped, so a restart here would start it under a unit that cannot load
# libonnxruntime.so.1, and the health gate cannot see that. The recovery block
# still prints, because the swap is not rolled back automatically.
if ! refresh_unit; then
print_recovery
error "aborting before the restart: the systemd unit is not what this update needs"
fi
restart_and_verify
echo ""
info "Done. Now at ${TARGET_VERSION:-${TARGET_REF}}."
# The verdict the operator gets is "landed, but a post-swap step failed", and
# it is a non-zero exit so a script or an alerting wrapper sees it. AFTER the
# health gate, deliberately: the service really is up on the new code, and an
# exit status reported before the restart would say nothing about whether the
# swap was usable.
if [ "$POST_UPDATE_RC" -ne 0 ]; then
error "update landed at ${TARGET_VERSION:-${TARGET_REF}}, but the post-update hook failed (exit ${POST_UPDATE_RC}) — check its output above; the service was restarted on the new code anyway"
fi
}
main "$@"