Ctrl K
ComunidadGuías…

install.sh

Rama main · réplica de lectura del repositorio original.

Archivo original ↗Preguntar a KiBot
#!/usr/bin/env bash
#
# NineDeploy — one-click installer
# https://github.com/NineDeploy/NineDeploy
#
# Usage:
#   curl -fsSL https://raw.githubusercontent.com/NineDeploy/NineDeploy/main/install.sh | bash
#
# Or from a clone:
#   ./install.sh
#
# Deployment modes (the installer follows what is already installed):
#   --docker        panel runs as a Docker container (no Node.js/systemd/PM2
#                   on the host; Docker/Compose deploys, managed databases,
#                   Traefik ingress and S3 backups work identically)
#   --bare-metal    hardened systemd service with direct PM2 access (default
#                   for fresh non-interactive runs)
#   NINEDEPLOY_INSTALL_MODE=docker|bare-metal works too. With no flag, an
#   existing install decides the mode; on a fresh host with a terminal the
#   installer asks. Upgrades are the same command re-run.
#
# Version selection:
#   --channel release   newest vX.Y.Z tag on GitHub (default)
#   --channel stable    the release the `stable` tag points at: a release that
#                       has been out for 7 days, or a security fix (A6)
#   --channel main      track the main branch (edge)
#   --version vX.Y.Z    pin an exact tag
#   --force             discard local modifications to tracked files and drop
#                       every build artifact before rebuilding. Use this when
#                       an upgrade left a stale panel bundle behind.
#                       NINEDEPLOY_FORCE=1 works too.
#   --private           fresh installs only: keep the panel on 127.0.0.1
#                       instead of publishing it on 0.0.0.0:3000 (reach it
#                       through an SSH tunnel or a Traefik domain).
#                       NINEDEPLOY_PRIVATE=1 works too.
#
# Panel domain (A2):
#   --domain HOST       serve the panel on HOST (NINEDEPLOY_DOMAIN); point its
#                       A record at this server. A first install with a
#                       terminal asks for it when the flag is absent.
#   --email ADDRESS     Let's Encrypt account email (NINEDEPLOY_ACME_EMAIL), so
#                       HOST gets a real certificate and becomes the public URL.
#   Either flag replaces the value in .env; without them .env keeps its own.
#
# Release integrity (r702): a release since v0.10.43 is installed from its own
# archive only when it matches the release's SHA256SUMS; with cosign on the
# host, SHA256SUMS' Sigstore signature is verified first. Either failing stops
# the installer. NINEDEPLOY_SKIP_SIGNATURE_VERIFY=1 skips the signature check
# only (e.g. no route to Sigstore) — the checksum is always enforced.
#
set -euo pipefail

BOLD='\033[1m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
RED='\033[0;31m'
CYAN='\033[0;36m'
NC='\033[0m'

info()  { echo -e "${CYAN}ℹ${NC}  $*"; }
ok()    { echo -e "${GREEN}✓${NC}  $*"; }
warn()  { echo -e "${YELLOW}⚠${NC}  $*"; }
fail()  { echo -e "${RED}✗${NC}  $*"; exit 1; }

# G4.1: a root shell on a minimal image (Alpine, some RHEL cloud images) has
# no sudo. Every privileged step below is written as `sudo …`; as root that
# is just the command (`-E` keeps the environment, which root already has).
if [ "$(id -u)" -eq 0 ] && ! command -v sudo >/dev/null 2>&1; then
  sudo() {
    while [ "$#" -gt 0 ]; do
      case "$1" in -E) shift ;; --) shift; break ;; *) break ;; esac
    done
    "$@"
  }
fi

# ── Remote code policy (L-15) ──────────────────────────────────────────────
#
# This script installs Nixpacks and Traefik by downloading a pinned artifact
# and checking its SHA-256 before use. Docker and Node.js used to be different:
# both were fetched with `curl … | sudo sh`, i.e. whatever bytes the endpoint
# returned were executed as root, unverified, with no record of what ran. A
# compromised CDN, a hijacked domain or an attacker on the path between the
# host and the endpoint got root on the machine that holds every deployment
# secret on this instance.
#
# Those two now install from their vendors' APT repositories, whose packages
# are signed and verified by apt against a key pinned into
# /etc/apt/keyrings — the vendors' own documented method, and the one that
# gives the host a verifiable chain instead of a pipe.
#
# The setup script remains as a last-resort fallback, but it is no longer
# silent: it downloads to a file, prints the URL and the digest of exactly
# what it fetched, and refuses to run without consent.
allow_unverified_scripts() { [ "${NINEDEPLOY_ALLOW_UNVERIFIED_INSTALL_SCRIPTS:-}" = "1" ]; }

# ── Long-step visibility ─────────────────────────────────────────────────────
#
# Vendor installs (Node, Docker) push hundreds of MB through apt with their
# output discarded — from the operator's chair that is indistinguishable from
# a hung installer (a multi-minute "freeze" on a slow mirror). Stream apt's
# own progress lines (Get:/Unpacking/Setting up) live, with a heartbeat when
# nothing matched for a while. The exit code is the wrapped command's own.
run_apt_step() {
  local log offset new printed elapsed rc pid
  log=$(mktemp) || return 1
  offset=1
  elapsed=0
  printed=0
  # DEBIAN_FRONTEND=noninteractive: a conffile or debconf prompt would block
  # forever on the piped installer's non-terminal stdin. </dev/null enforces
  # the same guarantee for anything that reads stdin directly.
  sudo env DEBIAN_FRONTEND=noninteractive "$@" </dev/null >"$log" 2>&1 &
  pid=$!
  while kill -0 "$pid" 2>/dev/null; do
    sleep 5
    elapsed=$((elapsed + 5))
    # "Waiting for cache lock" (unattended-upgrades holding dpkg on a fresh
    # boot) and "Processing triggers" are real progress and belong on screen —
    # hiding them is what made long apt phases look like a dead installer.
    new=$(tail -c +"$offset" "$log" | grep -E '^(Get|Hit|Fetched|Unpacking|Setting up|Selecting|Preparing|Need to get|Waiting|Processing)' || true)
    if [ -n "$new" ]; then
      printf '%s\n' "$new"
      offset=$(( $(wc -c <"$log") + 1 ))
      printed=$elapsed
    elif [ $((elapsed - printed)) -ge 20 ]; then
      printf '  … still working (%ss elapsed)\n' "$elapsed" >&2
      printed=$elapsed
    fi
  done
  wait "$pid"
  rc=$?
  new=$(tail -c +"$offset" "$log" | grep -E '^(Get|Hit|Fetched|Unpacking|Setting up|Selecting|Preparing|Waiting|Processing)' || true)
  [ -z "$new" ] || printf '%s\n' "$new"
  if [ "$rc" -ne 0 ]; then
    warn "Command failed: sudo $*"
    tail -n 15 "$log" >&2
  fi
  rm -f "$log"
  return "$rc"
}

# Heartbeat for the SILENT long steps that are not apt — pnpm install and the
# monorepo build run many minutes with no output at all on a cold host (pnpm's
# progress bars need a TTY the piped installer does not have), which reads as
# a hung installer. Capture to a log, tick every 30s, and dump the log tail on
# failure so a broken build explains itself instead of dying quietly.
run_quiet_step() {
  local label="$1" log elapsed rc pid
  shift
  log=$(mktemp) || return 1
  "$@" >"$log" 2>&1 &
  pid=$!
  elapsed=0
  while kill -0 "$pid" 2>/dev/null; do
    sleep 5
    elapsed=$((elapsed + 5))
    if [ $((elapsed % 30)) -eq 0 ]; then
      printf '  … %s — still working (%ss elapsed)\n' "$label" "$elapsed" >&2
    fi
  done
  wait "$pid"
  rc=$?
  if [ "$rc" -ne 0 ]; then
    warn "Step failed: $*"
    tail -n 30 "$log" >&2
  fi
  rm -f "$log"
  return "$rc"
}

# Fetch a vendor GPG key into /etc/apt/keyrings and register its repository.
# Args: <name> <key-url> <repo-line-without-signed-by>
add_signed_apt_repo() {
  local name="$1" key_url="$2" repo_line="$3"
  local keyring="/etc/apt/keyrings/${name}.asc"
  sudo install -m 0755 -d /etc/apt/keyrings || return 1
  curl -fsSL "$key_url" | sudo tee "$keyring" >/dev/null || return 1
  sudo chmod a+r "$keyring" || return 1
  echo "deb [arch=$(dpkg --print-architecture) signed-by=${keyring}] ${repo_line}" \
    | sudo tee "/etc/apt/sources.list.d/${name}.list" >/dev/null || return 1
  run_apt_step apt-get update -y || return 1
}

# Last resort: download a vendor setup script, show what it is, and run it
# only with explicit consent.
run_vendor_script() {
  local url="$1" stage digest
  stage=$(mktemp -d) || fail "Could not create a download workspace"
  curl -fsSL "$url" -o "$stage/setup.sh" || { rm -rf "$stage"; return 1; }
  digest=$(sha256sum "$stage/setup.sh" | awk '{print $1}')
  warn "About to run an UNVERIFIED vendor script as root:"
  warn "  url    : $url"
  warn "  sha256 : $digest"
  if ! allow_unverified_scripts; then
    if [ -t 0 ]; then
      read -r -p "Run it? [y/N] " _reply
      case "$_reply" in [yY]*) ;; *) rm -rf "$stage"; return 1 ;; esac
    else
      warn "Refusing (non-interactive). Re-run with NINEDEPLOY_ALLOW_UNVERIFIED_INSTALL_SCRIPTS=1 to accept."
      rm -rf "$stage"
      return 1
    fi
  fi
  sudo -E bash "$stage/setup.sh"
  local rc=$?
  rm -rf "$stage"
  return $rc
}

# Node.js from NodeSource's signed APT repository.
install_node_apt() {
  local major="$1"
  add_signed_apt_repo "nodesource" \
    "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" \
    "https://deb.nodesource.com/node_${major}.x nodistro main" || return 1
  run_apt_step apt-get install -y nodejs
}

# Node.js: signed repo first, consented script second.
install_node() {
  local major="${1:-24}"
  install_node_apt "$major" && return 0
  warn "NodeSource APT repository unavailable — falling back to their setup script."
  run_vendor_script "https://deb.nodesource.com/setup_${major}.x" \
    && sudo apt-get install -y nodejs
}

# Supported Node lines and their October 21, 2026 security patch floors.
supported_node_version() {
  local version="${1#v}" minor patch floor_minor floor_patch
  [[ "$version" =~ ^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]] || return 1
  minor="${BASH_REMATCH[2]}"; patch="${BASH_REMATCH[3]}"
  case "${BASH_REMATCH[1]}" in
    22) floor_minor=23; floor_patch=4 ;;
    24) floor_minor=22; floor_patch=1 ;;
    26) floor_minor=12; floor_patch=1 ;;
    *) return 1 ;;
  esac
  (( minor > floor_minor || (minor == floor_minor && patch >= floor_patch) ))
}

ensure_node() {
  local required='22.23.4+, 24.22.1+ or 26.12.1+ (supported majors only)'
  local major=24 current_version installed_version
  if command -v node &>/dev/null && supported_node_version "$(node -v)"; then
    ok "Node.js $(node -v)"
    return 0
  fi
  current_version=$(node -v 2>/dev/null || true)
  if [[ "$current_version" =~ ^v?([0-9]+)\. ]]; then
    case "${BASH_REMATCH[1]}" in
      22|24|26) major="${BASH_REMATCH[1]}" ;;
      25) major=26 ;;
      *) (( BASH_REMATCH[1] <= 26 )) || fail "Unsupported Node.js $current_version. Select $required manually; the installer will not downgrade it." ;;
    esac
  fi
  warn "Node.js is missing or below the security baseline. Installing a supported Node $major runtime: $required"
  if command -v apt-get &>/dev/null; then
    install_node "$major" || fail "Could not install a secure Node.js runtime. Install $required manually: https://nodejs.org/"
  elif command -v dnf &>/dev/null; then
    install_node_rpm "$major" || fail "Could not install a secure Node.js runtime from NodeSource's RPM repository. Install $required manually: https://nodejs.org/"
  elif [ "$(host_family)" = "suse" ]; then
    install_node_zypper "$major" || fail "Could not install a secure Node.js runtime from NodeSource's RPM repository. Install $required manually: https://nodejs.org/"
  elif [ "$(host_family)" = "arch" ]; then
    install_node_pacman "$major" || fail "Could not install a secure Node.js runtime from Arch's repository. Install $required manually: https://nodejs.org/"
  elif command -v brew &>/dev/null; then
    brew upgrade "node@$major" 2>/dev/null || brew install "node@$major"
  else
    fail "Install Node.js $required manually: https://nodejs.org/"
  fi
  # Mirrors can lag the security release; a successful package install alone
  # never proves that the executable on PATH is patched.
  installed_version=$(node -v 2>/dev/null || true)
  command -v node &>/dev/null && supported_node_version "$installed_version" && [[ "${installed_version#v}" == "$major".* ]] \
    || fail "Node.js on PATH still misses the security baseline or changed the requested major ($major). Need $required; check your package mirror and PATH."
  ok "Node.js $(node -v) meets the security baseline"
}

# Docker from Docker Inc.'s signed APT repository.
install_docker_apt() {
  local codename
  codename=$( (. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") 2>/dev/null ) || return 1
  local id
  id=$( (. /etc/os-release && echo "$ID") 2>/dev/null ) || return 1
  case "$id" in ubuntu|debian) ;; *) return 1 ;; esac
  add_signed_apt_repo "docker" \
    "https://download.docker.com/linux/${id}/gpg" \
    "https://download.docker.com/linux/${id} ${codename} stable" || return 1
  run_apt_step apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
}

# G4.1: this host's family (os_family, below) and its package manager.
host_family() { os_family /etc/os-release; }

# Install packages with the host's package manager.
pkg_install() {
  case "$(host_family)" in
    debian) run_apt_step apt-get install -y "$@" ;;
    rhel|fedora) run_quiet_step "dnf install $*" sudo dnf -y install "$@" ;;
    alpine) run_quiet_step "apk add $*" sudo apk add --no-cache "$@" ;;
    suse) run_quiet_step "zypper install $*" sudo zypper --non-interactive install --no-recommends "$@" ;;
    # --needed: never reinstall; no -y/-u here, so the installer does not upgrade the whole system.
    arch) run_quiet_step "pacman -S $*" sudo pacman -S --noconfirm --needed "$@" ;;
    *) return 1 ;;
  esac
}

# Node.js from NodeSource's signed RPM repository (the stanza their setup
# script writes; gpgcheck on, so dnf verifies every package).
install_node_rpm() {
  local major="$1" arch
  arch="$(uname -m)"
  case "$arch" in x86_64|aarch64) ;; *) return 1 ;; esac
  printf '%s\n' \
    '[nodesource-nodejs]' \
    "name=Node.js Packages for Linux RPM based distros - ${arch}" \
    "baseurl=https://rpm.nodesource.com/pub_${major}.x/nodistro/nodejs/${arch}" \
    'priority=9' 'enabled=1' 'gpgcheck=1' \
    'gpgkey=https://rpm.nodesource.com/gpgkey/ns-operations-public.key' \
    'module_hotfixes=1' \
    | sudo tee /etc/yum.repos.d/nodesource-nodejs.repo >/dev/null || return 1
  run_quiet_step "dnf install nodejs" sudo dnf -y install nodejs
}

# G4.1: the same NodeSource RPMs through zypper (openSUSE, SLES). Their
# repository index is unsigned, as it is for dnf above; every package is
# signed and zypper checks it against the key imported here first.
install_node_zypper() {
  local major="$1" arch
  arch="$(uname -m)"
  case "$arch" in x86_64|aarch64) ;; *) return 1 ;; esac
  sudo rpm --import https://rpm.nodesource.com/gpgkey/ns-operations-public.key || return 1
  printf '%s\n' \
    '[nodesource-nodejs]' \
    "name=Node.js Packages for Linux RPM based distros - ${arch}" \
    "baseurl=https://rpm.nodesource.com/pub_${major}.x/nodistro/nodejs/${arch}" \
    'enabled=1' 'autorefresh=1' 'priority=9' \
    'gpgcheck=1' 'repo_gpgcheck=0' 'pkg_gpgcheck=1' \
    'gpgkey=https://rpm.nodesource.com/gpgkey/ns-operations-public.key' \
    | sudo tee /etc/zypp/repos.d/nodesource-nodejs.repo >/dev/null || return 1
  pkg_install nodejs
}

# G4.1: Node from Arch's own repository: the LTS package of that line, or
# `nodejs` for the current one. npm comes separately; Arch ships no corepack.
install_node_pacman() {
  local pkg
  case "$1" in
    22) pkg=nodejs-lts-jod ;;
    24) pkg=nodejs-lts-krypton ;;
    *) pkg=nodejs ;;
  esac
  pkg_install "$pkg" npm || { warn "pacman could not install $pkg: update the system first (sudo pacman -Syu), remove a Node package of another line if one is installed, and re-run."; return 1; }
}

# Docker from Docker Inc.'s signed RPM repository (RHEL, CentOS, Rocky, Alma, Fedora).
install_docker_rpm() {
  local family id url
  family="$(host_family)"
  id=$( (. /etc/os-release && echo "$ID") 2>/dev/null ) || return 1
  url="$(docker_rpm_repo_url "$family" "$id")" || return 1
  curl -fsSL "$url" | sudo tee /etc/yum.repos.d/docker-ce.repo >/dev/null || return 1
  # RHEL-family hosts ship podman/buildah/runc, which conflict with docker-ce.
  sudo dnf -y remove podman-docker buildah runc >/dev/null 2>&1 || true
  run_quiet_step "dnf install docker-ce" sudo dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
}

# G4.1: Docker from the distribution's own repository on openSUSE / SLES
# (SLES needs the Containers Module), started and enabled with systemd.
install_docker_zypper() {
  if ! pkg_install docker docker-compose docker-buildx; then
    if [ "$( (. /etc/os-release && echo "$ID") 2>/dev/null)" = "sles" ]; then
      warn "On SLES, Docker comes from the Containers Module: sudo SUSEConnect -p sle-module-containers/\$(. /etc/os-release && echo \$VERSION_ID)/\$(uname -m), then re-run."
    fi
    return 1
  fi
  sudo systemctl enable --now docker >/dev/null 2>&1 || true
}

# G4.1: Docker from Arch's own repository, started and enabled with systemd.
# A stale package database makes pacman 404: then run `sudo pacman -Syu` and re-run.
install_docker_pacman() {
  pkg_install docker docker-compose docker-buildx || { warn "pacman could not install Docker; update the system first (sudo pacman -Syu) and re-run."; return 1; }
  sudo systemctl enable --now docker >/dev/null 2>&1 || true
}

# Docker from Alpine's own repository (community), started and enabled with OpenRC.
install_docker_apk() {
  # docker-openrc carries the init script; apk adds it on its own only when openrc is installed.
  run_quiet_step "apk add docker" sudo apk add --no-cache docker docker-cli-compose docker-openrc || return 1
  sudo rc-update add docker default >/dev/null 2>&1 || true
  sudo rc-service docker start >/dev/null 2>&1 || true
}

if [ -z "${NINEDEPLOY_INSTALL_DIR:-}" ]; then
  if [ -f "./package.json" ] && [ -d "./apps/server" ]; then
    INSTALL_DIR="$(pwd)"
  elif [ -d "/opt/ninedeploy" ] && [ -f "/opt/ninedeploy/package.json" ]; then
    INSTALL_DIR="/opt/ninedeploy"
  elif [ -d "$HOME/ninedeploy" ]; then
    INSTALL_DIR="$HOME/ninedeploy"
  else
    INSTALL_DIR="$HOME/ninedeploy"
  fi
else
  INSTALL_DIR="$NINEDEPLOY_INSTALL_DIR"
fi
REPO_URL="https://github.com/NineDeploy/NineDeploy.git"
# Same repository as REPO_URL, in the owner/name form the GitHub API wants.
REPO_SLUG="NineDeploy/NineDeploy"
# r263: the panel image's repository. OCI/GHCR names are lowercase-only, so
# the release workflow publishes ghcr.io/ninedeploy/ninedeploy — deriving it
# from the mixed-case slug above made the --docker image checks reject the
# compose file and `docker manifest inspect` look up an invalid reference.
IMAGE_REPO="$(printf '%s' "$REPO_SLUG" | tr '[:upper:]' '[:lower:]')"
NEEDS_CLONE=false

# ── Pure helpers (no side effects; exercised by CI via the source-only mode) ─

# Highest vX.Y.Z from a newline-separated tag list on stdin.
#
# `sort -V` is GNU-only; busybox and BSD coreutils either lack it or sort
# lexically, which ranks v0.2.9 above v0.2.36 and would pin an upgrade to a
# stale release forever. Normalise each component to a zero-padded fixed
# width first so a plain lexical `sort` is correct everywhere.
highest_semver_tag() {
  (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' || true) \
    | awk -F'[v.]' '{ printf "%010d%010d%010d %s\n", $2, $3, $4, $0 }' \
    | sort \
    | tail -1 \
    | awk '{print $2}'
}

# vX.Y.Z of a GitHub `releases/latest` JSON body on stdin (empty when none).
release_tag_from_json() {
  (grep -o '"tag_name"[[:space:]]*:[[:space:]]*"[^"]*"' || true) \
    | head -1 | sed 's/.*"\([^"]*\)"$/\1/' | highest_semver_tag
}

# Highest vX.Y.Z of a GitHub `tags` JSON body on stdin (empty when none).
highest_tag_from_tags_json() {
  (grep -o '"name"[[:space:]]*:[[:space:]]*"[^"]*"' || true) \
    | sed 's/.*"\([^"]*\)"$/\1/' | highest_semver_tag
}

# Newest vX.Y.Z release tag, asked of GitHub directly. Three independent
# sources, because a single one is not reliable enough to decide what an
# operator's `--channel release` upgrade actually installs:
#
#   1. The releases API — authoritative: release-publish.yml creates the
#      GitHub Release only AFTER every gate passed and the multi-arch image
#      was pushed, so a published release is a tag whose pipeline is green.
#   2. The tags API — fallback when releases/latest is unreachable or 404s
#      (no published release yet) but tags exist.
#   3. `git ls-remote` — last resort: works behind a git-only proxy, no API
#      rate limit, but it also lists tags whose release pipeline failed.
#
# r571: git ls-remote used to be asked FIRST, so a tag pushed a minute ago —
# release pipeline still running, or failed — was installed (and, in Docker
# mode, its compose file fetched) by every `--channel release` run. Its
# fallback warnings also went to STDOUT, which `REF=$(latest_tag)` captured:
# whenever the first source came back empty, REF became "⚠ …\nvX.Y.Z" and the
# tarball URL, the compose fetch and the git checkout all failed on it.
#
# The first source that yields a well-formed tag wins.
latest_tag() {
  local _tag=""

  _tag=$(
    (curl -fsSL -m 15 --retry 3 --retry-delay 2 -H 'Accept: application/vnd.github+json' \
      "https://api.github.com/repos/${REPO_SLUG}/releases/latest" 2>/dev/null || true) \
      | release_tag_from_json
  )
  if [ -n "$_tag" ]; then printf '%s' "$_tag"; return 0; fi

  warn "The GitHub releases API returned no published release — falling back to the tags API" >&2
  _tag=$(
    (curl -fsSL -m 15 --retry 3 --retry-delay 2 -H 'Accept: application/vnd.github+json' \
      "https://api.github.com/repos/${REPO_SLUG}/tags?per_page=100" 2>/dev/null || true) \
      | highest_tag_from_tags_json
  )
  if [ -n "$_tag" ]; then printf '%s' "$_tag"; return 0; fi

  warn "The GitHub API is unreachable — falling back to git ls-remote (may include a tag whose release is not published yet)" >&2
  _tag=$(
    (git ls-remote --tags --refs "$REPO_URL" 2>/dev/null || true) \
      | awk -F/ '{print $NF}' | highest_semver_tag
  )
  printf '%s' "$_tag"
}

# A6: the vX.Y.Z release the `stable` tag points at, from `git ls-remote
# --tags` output on stdin (annotated tags peeled through their ^{} line).
# Empty when there is no stable tag or no release tag on its commit.
stable_tag_from_ls_remote() {
  awk -F'\t' '
    { ref = $2; sub("^refs/tags/", "", ref)
      if (ref ~ /\^\{\}$/) { sub(/\^\{\}$/, "", ref); peeled[ref] = $1 }
      else if (!(ref in direct)) { direct[ref] = $1 } }
    END { for (r in direct) if (!(r in peeled)) peeled[r] = direct[r]
          s = peeled["stable"]; if (s == "") exit
          for (r in peeled) if (r ~ /^v[0-9]+\.[0-9]+\.[0-9]+$/ && peeled[r] == s) print r }
  ' | highest_semver_tag
}

# A6: the stable channel's release. The daily stable-channel.yml workflow
# moves the lightweight `stable` tag to a release that has been out for seven
# days (or at once for a security fix). git ls-remote answers with every tag
# and its commit in one request; the tags API is the fallback.
stable_tag() {
  local _tag=""
  _tag=$( (git ls-remote --tags "$REPO_URL" 2>/dev/null || true) | stable_tag_from_ls_remote )
  if [ -n "$_tag" ]; then printf '%s' "$_tag"; return 0; fi
  warn "git ls-remote found no stable release — asking the GitHub API" >&2
  local _sha=""
  _sha=$( (curl -fsSL -m 15 --retry 3 --retry-delay 2 -H 'Accept: application/vnd.github+json' \
      "https://api.github.com/repos/${REPO_SLUG}/git/ref/tags/stable" 2>/dev/null || true) \
    | grep -o '"sha"[[:space:]]*:[[:space:]]*"[0-9a-f]*"' | head -1 | sed 's/.*"\([0-9a-f]*\)"$/\1/' )
  [ -n "$_sha" ] || return 0
  (curl -fsSL -m 15 --retry 3 --retry-delay 2 -H 'Accept: application/vnd.github+json' \
      "https://api.github.com/repos/${REPO_SLUG}/tags?per_page=100" 2>/dev/null || true) \
    | tr -d '\n' | sed 's/},{/}\n{/g' | grep "\"sha\"[[:space:]]*:[[:space:]]*\"$_sha\"" \
    | grep -o '"name"[[:space:]]*:[[:space:]]*"[^"]*"' | sed 's/.*"\([^"]*\)"$/\1/' | highest_semver_tag
}

# Docker mode: the panel image tag for the resolved ref. A pinned
# `--version vX.Y.Z` pulls `:vX.Y.Z` (r263); the main channel pulls `:edge`
# (set by ci.yml on every push to main). r571: a release-channel run pulls
# the tag it RESOLVED, not `:latest` — the compose file is fetched at that
# tag, and `:latest` is whatever was pushed last (a different release, or one
# whose GitHub Release never published). `:latest` remains only for the
# no-tags fallback (REF=main on the release channel).
#   $1 pinned version (may be empty)  $2 channel  $3 resolved ref
docker_image_tag() {
  if [ -n "$1" ]; then
    printf '%s' "$1"
  elif [ "$2" = "main" ]; then
    printf 'edge'
  elif printf '%s' "$3" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
    printf '%s' "$3"
  else
    printf 'latest'
  fi
}

# Nixpacks → SHA-256, keyed by <version>:<arch>. r574: the table used to be
# keyed by architecture only, so an overridden NINEDEPLOY_NIXPACKS_VERSION
# looked up the 1.41.0 digest and died on a generic checksum mismatch — the
# "not in the verified-checksum table" refusal below was unreachable.
# Source: https://github.com/railwayapp/nixpacks/releases — every entry was
# cross-checked against the GitHub-published `sha256sum -c *.sha256`.
nixpacks_sha256() { # <version> <x86_64|aarch64>
  case "$1:$2" in
    1.41.0:x86_64)  printf '%s' 0f55de7874507b9cf7502113120bd96f2ab6979f78d10eaf2eb2ade9207b3af6 ;;
    1.41.0:aarch64) printf '%s' 912bd02dd2bb6f9c3a9ed965fe8a68b4aa318dc7a2546e2eca6f2806a894ba39 ;;
    *) printf '' ;;
  esac
}

# The absolute data directory the systemd unit is rendered with (created).
# Environment files commonly use NINEDEPLOY_DATA_DIR=./.data. systemd
# requires every ReadWritePaths= operand to be absolute, so resolve the
# configured directory against the current directory (section 5 runs from
# the install dir) before rendering the unit.
#   $1 install dir
resolve_data_dir() {
  local DATA_DIR_SETTING DATA_DIR
  DATA_DIR_SETTING="${NINEDEPLOY_DATA_DIR:-$1/.data}"
  mkdir -p "$DATA_DIR_SETTING" || return 1
  DATA_DIR=$(cd "$DATA_DIR_SETTING" && pwd -P) || return 1
  printf '%s' "$DATA_DIR"
}

# ── Panel reachability (0.16) ───────────────────────────────────────────────
#
# A fresh install publishes the panel on 0.0.0.0 (both modes) so the operator
# can open http://<server-ip>:3000 straight away; `--private` (or
# NINEDEPLOY_PRIVATE=1) keeps it on loopback for tunnel/Traefik-only access.
# Upgrades never change an existing bind. Because an exposed, empty panel
# hands the instance-operator account to whoever registers first, the
# installer also writes a one-time NINEDEPLOY_SETUP_TOKEN that the first-run
# form requires (apps/server/src/lib/setupToken.ts).

# The address this host is reached on: NINEDEPLOY_PUBLIC_IP, else the source
# address of the default route (no packet is sent), else the first
# `hostname -I` address. Empty when none can be found (macOS, no network).
detect_server_ip() {
  local _ip="${NINEDEPLOY_PUBLIC_IP:-}"
  if [ -z "$_ip" ] && command -v ip >/dev/null 2>&1; then
    _ip="$(ip -4 route get 1.1.1.1 2>/dev/null | sed -n 's/.* src \([0-9.]*\).*/\1/p' | head -1)"
  fi
  if [ -z "$_ip" ]; then
    _ip="$(hostname -I 2>/dev/null | awk '{print $1}')"
  fi
  printf '%s' "$_ip"
}

# http://<ip>:<port> for a wildcard bind, http://localhost:<port> otherwise.
#   $1 bind address  $2 port
panel_default_url() {
  local _ip=""
  case "$1" in
    0.0.0.0|::|"[::]") _ip="$(detect_server_ip)" ;;
    127.0.0.1|localhost|"") ;;
    *) _ip="$1" ;;
  esac
  printf 'http://%s:%s' "${_ip:-localhost}" "$2"
}

# True when the release being installed gates its first admin on the setup
# token (0.16+, or the main channel). The installer on main is what
# `curl … main/install.sh` runs, so it also installs OLDER pinned releases —
# those must not get the 0.0.0.0 bind without the gate behind it.
#   $1 ref (main or vX.Y.Z)
ref_has_setup_token() {
  [ "$1" = "main" ] && return 0
  printf '%s' "$1" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || return 1
  local _major _minor
  _major="$(printf '%s' "${1#v}" | cut -d. -f1)"
  _minor="$(printf '%s' "${1#v}" | cut -d. -f2)"
  [ "$_major" -gt 0 ] || [ "$_minor" -ge 16 ]
}

gen_setup_token() {
  if command -v openssl >/dev/null 2>&1; then openssl rand -hex 18
  else head -c 18 /dev/urandom | od -An -tx1 | tr -d ' \n'; fi
}

# Firewall hint: an active ufw that does not allow the panel port makes the
# new 0.0.0.0 bind look broken from outside.
#   $1 port
warn_if_firewalled() {
  command -v ufw >/dev/null 2>&1 || return 0
  local _status
  _status="$(ufw status 2>/dev/null || true)"
  printf '%s' "$_status" | grep -q '^Status: active' || return 0
  printf '%s' "$_status" | grep -Eq "^$1(/tcp)?[[:space:]]" && return 0
  warn "ufw is active and does not allow port $1 — open it with: sudo ufw allow $1/tcp"
}

# Ask the panel whether it still needs its first admin, and if so print the
# setup token with a link that pre-fills it (a URL fragment, so it never
# reaches a request log).
#   $1 probe base (http://127.0.0.1:<port>)  $2 public URL  $3 token
print_setup_token_hint() {
  local _status
  [ -n "$3" ] || return 0
  _status="$(curl -fsS -m 5 "$1/v1/auth/status" 2>/dev/null || true)"
  printf '%s' "$_status" | grep -q '"initialized":false' || return 0
  echo -e "  ${CYAN}Setup token:${NC} $3"
  echo -e "  ${CYAN}First run:${NC}  $2/login#setup-token=$3"
  # A2: with a panel domain, the domain link works once its certificate is
  # issued; the direct address works at once.
  if [ -n "${4:-}" ] && [ "$4" != "$2" ]; then
    echo -e "              or, until the domain answers: $4/login#setup-token=$3"
  fi
  echo -e "              (the first account created becomes the instance operator)"
}

# ── Panel domain (A2) ───────────────────────────────────────────────────────
#
# `--domain panel.example.com --email you@example.com` serves the panel on its
# own hostname with a Let's Encrypt certificate from the first boot: both go
# into .env (NINEDEPLOY_DOMAIN, NINEDEPLOY_ACME_EMAIL), the panel's Traefik
# router does the rest. Bare metal SOURCES .env (`set -a; . ./.env`), so the
# validators accept plain hostnames and addresses only, never a character a
# shell would act on (no `$`, quotes, spaces or newlines).

# A lowercase DNS hostname with at least two labels.
valid_panel_domain() {
  case "$1" in "" | *[!a-z0-9.-]*) return 1 ;; esac
  [ "${#1}" -le 253 ] || return 1
  printf '%s\n' "$1" | grep -Eq '^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]([a-z0-9-]{0,61}[a-z0-9])?$'
}

valid_acme_email() {
  case "$1" in "" | *[!A-Za-z0-9._%+@-]*) return 1 ;; esac
  printf '%s\n' "$1" | grep -Eq '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'
}

# Set KEY=VALUE in ./.env, replacing an existing line. Filter-and-append, so
# the value never rides through a sed replacement (r449).
#   $1 key  $2 value
env_file_set() {
  local _tmp
  if [ -f .env ] && grep -q "^$1=" .env; then
    _tmp="$(mktemp)"
    { grep -v "^$1=" .env; printf '%s=%s\n' "$1" "$2"; } > "$_tmp" && cat "$_tmp" > .env && rm -f "$_tmp"
  else
    printf '%s=%s\n' "$1" "$2" >> .env
  fi
}

# On a first install with a terminal, ask for the panel domain (and, with one,
# the Let's Encrypt email) unless the flags gave them. Never on a re-run, and
# never during the panel's self-update (no terminal there).
prompt_panel_domain() {
  [ -z "$PANEL_DOMAIN_ARG" ] || return 0
  [ -z "${ND_SELF_UPDATE_TARGET:-}" ] || return 0
  { : <>/dev/tty; } 2>/dev/null || return 0
  local _in=""
  { printf "Panel domain, e.g. panel.example.com (Enter to use the server's IP address): " >/dev/tty; } 2>/dev/null || return 0
  IFS= read -r _in </dev/tty 2>/dev/null || _in=""
  _in="$(printf '%s' "$_in" | tr 'A-Z' 'a-z')"
  [ -n "$_in" ] || return 0
  if ! valid_panel_domain "$_in"; then
    warn "'$_in' is not a hostname; continuing without a panel domain (set one later in Settings -> Security)."
    return 0
  fi
  PANEL_DOMAIN_ARG="$_in"
  if [ -z "$ACME_EMAIL_ARG" ]; then
    ACME_PROMPTED=1
    { printf "Let's Encrypt account email for its certificate (Enter to skip): " >/dev/tty; } 2>/dev/null || return 0
    IFS= read -r _in </dev/tty 2>/dev/null || _in=""
    if [ -n "$_in" ] && valid_acme_email "$_in"; then
      ACME_EMAIL_ARG="$_in"
    elif [ -n "$_in" ]; then
      warn "The supplied ACME email is invalid and was not saved."
    fi
  fi
}

# Warn (never fail) when the domain does not point at this host yet: Let's
# Encrypt issues the certificate only once it does.
#   $1 domain
warn_domain_dns() {
  command -v getent >/dev/null 2>&1 || return 0
  local _ips _me
  # `getent` exits 2 for a name that does not resolve (or has AAAA only):
  # under `set -euo pipefail` that would end the run, so it must not fail.
  _ips="$({ getent ahostsv4 "$1" 2>/dev/null || true; } | awk '{print $1}' | sort -u | tr '\n' ' ')"
  _ips="${_ips% }"
  if [ -z "$_ips" ]; then
    warn "$1 does not resolve yet. Point its A record at this server; the certificate is issued once it does."
    return 0
  fi
  _me="$(detect_server_ip)"
  case " $_ips " in
    *" $_me "*) ok "$1 resolves to this server ($_me)" ;;
    *) warn "$1 resolves to $_ips, while this server's address looks like ${_me:-unknown}. Behind NAT that can be right; otherwise point the A record here first." ;;
  esac
}

# ── Release integrity (r702) ────────────────────────────────────────────────
#
# Since v0.10.43 every GitHub Release carries, besides CHANGELOG.md:
#   ninedeploy-vX.Y.Z.tar.gz    `git archive` of the tag (one top-level dir)
#   install.sh                  the tag's installer
#   SHA256SUMS                  sha256 of both, `sha256sum` format
#   SHA256SUMS.sigstore.json    Sigstore bundle: SHA256SUMS signed keyless by
#                               release-publish.yml (GitHub OIDC → Fulcio cert,
#                               logged in Rekor)
# release-publish.yml builds them in promote-release, verifies the bundle with
# verify_checksums_signature below (so these exact constants are proven against
# the real certificate before anything is published) and attaches them to a
# DRAFT release it only publishes once all four are present.
CHECKSUMS_SINCE="v0.10.43"
RELEASE_SIGNER_ISSUER="https://token.actions.githubusercontent.com"
# The certificate SAN is the signing workflow's ref: a tag push, or a manual
# re-run dispatched from main. Owner/name match case-insensitively (GitHub
# names do); everything else is literal and the expression is anchored.
RELEASE_SIGNER_IDENTITY_RE='^https://github\.com/(?i:ninedeploy/ninedeploy)/\.github/workflows/release-publish\.yml@refs/(tags/v[0-9]+\.[0-9]+\.[0-9]+|heads/main)$'

# Zero-padded sort key of a vX.Y.Z tag (see highest_semver_tag).
semver_key() {
  printf '%s\n' "$1" | awk -F'[v.]' '{ printf "%010d%010d%010d", $2, $3, $4 }'
}

# 0 when release $1 is one that publishes SHA256SUMS (vX.Y.Z >= CHECKSUMS_SINCE).
release_has_checksums() {
  printf '%s' "$1" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || return 1
  [ ! "$(semver_key "$1")" \< "$(semver_key "$CHECKSUMS_SINCE")" ]
}

# Asset name of the source archive release-publish.yml attaches for tag $1.
release_archive_name() {
  printf 'ninedeploy-%s.tar.gz' "$1"
}

# Download URL of release asset $2 of tag $1.
release_asset_url() {
  printf 'https://github.com/%s/releases/download/%s/%s' "$REPO_SLUG" "$1" "$2"
}

# The sha256 that a SHA256SUMS body on stdin lists for exactly the file name $1
# (`<64 hex>  <name>` or `<64 hex> *<name>`). Non-zero — and nothing printed —
# when the name is absent or listed with two different digests.
sha256sums_lookup() {
  local _digests
  _digests=$(tr -d '\r' | awk -v n="$1" '
    length($0) > 66 && substr($0, 1, 64) ~ /^[0-9a-f]+$/ && substr($0, 65, 1) == " " \
      && (substr($0, 66, 1) == " " || substr($0, 66, 1) == "*") && substr($0, 67) == n {
      print substr($0, 1, 64)
    }' | sort -u)
  [ -n "$_digests" ] || return 1
  [ "$(printf '%s\n' "$_digests" | wc -l)" -eq 1 ] || return 1
  printf '%s' "$_digests"
}

# Verify the Sigstore bundle $2 over SHA256SUMS $1 against the release
# workflow's identity. Returns 0 verified, 1 verification FAILED (cosign's
# output on stderr), 2 no cosign on this host, 3 a cosign too old to read
# Sigstore bundles (they need v2.4+). Callers decide what each means.
verify_checksums_signature() {
  local _help _out
  command -v cosign >/dev/null 2>&1 || return 2
  _help=$(cosign verify-blob --help 2>&1 || true)
  case "$_help" in *--new-bundle-format*) ;; *) return 3 ;; esac
  if _out=$(cosign verify-blob --new-bundle-format \
      --bundle "$2" \
      --certificate-identity-regexp "$RELEASE_SIGNER_IDENTITY_RE" \
      --certificate-oidc-issuer "$RELEASE_SIGNER_ISSUER" \
      "$1" 2>&1); then
    return 0
  fi
  printf '%s\n' "$_out" >&2
  return 1
}

# G4.1: the distribution family from an os-release file: debian (Debian,
# Ubuntu, Raspbian), rhel (RHEL, CentOS, Rocky, Alma, Oracle), fedora, alpine,
# suse (openSUSE Leap/Tumbleweed, SLES), arch (Arch, Manjaro, EndeavourOS),
# or unknown. ID decides first, then ID_LIKE. Amazon Linux is not claimed:
# Docker's RPM repository does not serve it.
os_family() {
  local file="${1:-/etc/os-release}" id="" like="" word
  [ -r "$file" ] || { echo unknown; return 0; }
  id=$( (. "$file" && printf '%s' "${ID:-}") 2>/dev/null ) || id=""
  like=$( (. "$file" && printf '%s' "${ID_LIKE:-}") 2>/dev/null ) || like=""
  case "$id" in amzn) echo unknown; return 0 ;; fedora) echo fedora; return 0 ;; esac
  for word in $id $like; do
    case "$word" in
      debian|ubuntu|raspbian) echo debian; return 0 ;;
      rhel|centos|rocky|almalinux|ol) echo rhel; return 0 ;;
      alpine) echo alpine; return 0 ;;
      suse|opensuse|opensuse-leap|opensuse-tumbleweed|sles|sled|sles_sap) echo suse; return 0 ;;
      arch|archarm|manjaro|endeavouros) echo arch; return 0 ;;
    esac
  done
  echo unknown
}

# G4.1: Docker Inc.'s RPM repository file for a family and os-release ID.
# RHEL has its own; CentOS, Rocky, Alma and Oracle use the CentOS one.
docker_rpm_repo_url() {
  case "$1" in
    fedora) echo "https://download.docker.com/linux/fedora/docker-ce.repo" ;;
    rhel) if [ "$2" = "rhel" ]; then echo "https://download.docker.com/linux/rhel/docker-ce.repo"; else echo "https://download.docker.com/linux/centos/docker-ce.repo"; fi ;;
    *) return 1 ;;
  esac
}

# r576: `NINEDEPLOY_INSTALL_SOURCE_ONLY=1 . ./install.sh` defines the helpers
# above and returns before anything touches the host — CI exercises the real
# functions under the script's own `set -euo pipefail` instead of re-typing
# them. Only honoured when SOURCED (`(return 0)` fails in an executed script),
# so a stray variable can never turn a real `curl … | bash` run into a no-op.
if [ "${NINEDEPLOY_INSTALL_SOURCE_ONLY:-}" = "1" ] && (return 0 2>/dev/null); then
  return 0
fi

echo ""
echo -e "${BOLD}╔══════════════════════════════════════════╗${NC}"
echo -e "${BOLD}║       NineDeploy Installer               ║${NC}"
echo -e "${BOLD}║       Self-hosted PaaS                   ║${NC}"
echo -e "${BOLD}╚══════════════════════════════════════════╝${NC}"
echo ""

# ── 0. Arguments & deployment mode ────────────────────────────────────────

CHANNEL="${NINEDEPLOY_CHANNEL:-release}"
PINNED_VERSION="${NINEDEPLOY_VERSION:-}"
FORCE_REFRESH="${NINEDEPLOY_FORCE:-0}"
INSTALL_MODE_FLAG=""
PANEL_PRIVATE="${NINEDEPLOY_PRIVATE:-0}"
# A2: set by --domain / --email (or the first-install prompt). An explicit flag
# replaces the value in .env; without one, .env keeps what it has.
PANEL_DOMAIN_ARG=""
ACME_EMAIL_ARG=""

while [ "$#" -gt 0 ]; do
  case "$1" in
    --domain=*)
      PANEL_DOMAIN_ARG="${1#--domain=}"
      shift
      ;;
    --domain)
      [ "$#" -ge 2 ] || fail "--domain requires a hostname, e.g. --domain panel.example.com"
      PANEL_DOMAIN_ARG="$2"
      shift 2
      ;;
    --email=*)
      ACME_EMAIL_ARG="${1#--email=}"
      shift
      ;;
    --email)
      [ "$#" -ge 2 ] || fail "--email requires the Let's Encrypt account email"
      ACME_EMAIL_ARG="$2"
      shift 2
      ;;
    --docker)
      INSTALL_MODE_FLAG="docker"
      shift
      ;;
    --bare-metal)
      INSTALL_MODE_FLAG="bare-metal"
      shift
      ;;
    --version=*)
      PINNED_VERSION="${1#--version=}"
      shift
      ;;
    --version)
      [ "$#" -ge 2 ] || fail "--version requires a vX.Y.Z value"
      PINNED_VERSION="$2"
      shift 2
      ;;
    --channel=*)
      CHANNEL="${1#--channel=}"
      shift
      ;;
    --channel)
      [ "$#" -ge 2 ] || fail "--channel requires release, stable or main"
      CHANNEL="$2"
      shift 2
      ;;
    --force|-f)
      FORCE_REFRESH=1
      shift
      ;;
    --private)
      PANEL_PRIVATE=1
      shift
      ;;
    v[0-9]*)
      [ -z "$PINNED_VERSION" ] && PINNED_VERSION="$1"
      shift
      ;;
    *)
      fail "Unknown installer argument: $1"
      ;;
  esac
done

case "$CHANNEL" in
  release|stable|main) ;;
  *) fail "Unsupported channel '$CHANNEL' (expected release, stable or main)" ;;
esac

PANEL_DOMAIN_ARG="$(printf '%s' "$PANEL_DOMAIN_ARG" | tr 'A-Z' 'a-z')"
if [ -n "$PANEL_DOMAIN_ARG" ] && ! valid_panel_domain "$PANEL_DOMAIN_ARG"; then
  fail "Invalid --domain '$PANEL_DOMAIN_ARG' (expected a hostname such as panel.example.com)"
fi
if [ -n "$ACME_EMAIL_ARG" ] && ! valid_acme_email "$ACME_EMAIL_ARG"; then
  fail "Invalid --email '$ACME_EMAIL_ARG' (expected an address such as you@example.com)"
fi

if [ -n "$PINNED_VERSION" ] && ! printf '%s' "$PINNED_VERSION" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$'; then
  fail "Invalid version '$PINNED_VERSION' (expected vX.Y.Z)"
fi

# Deployment mode: explicit flag > env > whatever is already installed. A
# fresh host with a terminal gets asked; piped/non-interactive runs default
# to bare-metal so the documented one-liner keeps its current behaviour.
DOCKER_INSTALL_DIR="${NINEDEPLOY_DOCKER_INSTALL_DIR:-/opt/ninedeploy-docker}"
BARE_METAL_UNIT_FILE="/etc/systemd/system/ninedeploy.service"

bare_metal_present() {
  [ -f "$BARE_METAL_UNIT_FILE" ] && return 0
  # A repo checkout alone is not an install; the installer's own .env plus
  # data directory are the footprint it leaves behind on this machine.
  [ -f "$INSTALL_DIR/package.json" ] && [ -f "$INSTALL_DIR/.env" ] && [ -d "$INSTALL_DIR/.data" ] && return 0
  return 1
}

docker_install_present() { [ -f "$DOCKER_INSTALL_DIR/docker-compose.yml" ]; }

if [ -n "$INSTALL_MODE_FLAG" ]; then
  INSTALL_MODE="$INSTALL_MODE_FLAG"
elif [ -n "${NINEDEPLOY_INSTALL_MODE:-}" ]; then
  INSTALL_MODE="$NINEDEPLOY_INSTALL_MODE"
elif bare_metal_present; then
  INSTALL_MODE="bare-metal"
elif docker_install_present; then
  INSTALL_MODE="docker"
elif [ -t 0 ]; then
  echo "Deployment mode:"
  echo "  1) Docker container   — panel runs as a container; no Node.js/PM2/systemd on the host"
  echo "  2) Bare-metal systemd — full feature set incl. PM2 services and UFW management"
  read -r -p "Choose [1/2, Enter=2]: " _mode_reply
  case "$_mode_reply" in
    1) INSTALL_MODE="docker" ;;
    *) INSTALL_MODE="bare-metal" ;;
  esac
else
  INSTALL_MODE="bare-metal"
fi

case "$INSTALL_MODE" in
  docker|bare-metal) ;;
  *) fail "Unsupported install mode '$INSTALL_MODE' (expected docker or bare-metal)" ;;
esac

# G4.1: the bare-metal panel is a systemd service. A Linux host without
# systemd (Alpine's OpenRC) gets the Docker mode; asking for bare-metal there
# explicitly is refused rather than leaving a panel nothing starts.
if [ "$INSTALL_MODE" = "bare-metal" ] && [ "$(uname -s)" = "Linux" ] && ! command -v systemctl >/dev/null 2>&1 && ! bare_metal_present; then
  if [ -n "$INSTALL_MODE_FLAG" ] || [ -n "${NINEDEPLOY_INSTALL_MODE:-}" ]; then
    fail "The bare-metal mode needs systemd, which this host does not have. Install with --docker instead."
  fi
  warn "This host has no systemd: installing in Docker mode (the panel runs as a container)."
  INSTALL_MODE="docker"
fi

if [ "$INSTALL_MODE" = "docker" ] && bare_metal_present; then
  fail "A bare-metal installation is already present. Upgrade it in place (re-run without --docker) or uninstall it first — running both would fight over the same ports and the Traefik ingress."
fi
# The symmetric case: an explicit --bare-metal onto a docker-mode host would
# build and start a SECOND panel (fresh .data, new master key) that fights the
# container over port 3000 and the shared ninedeploy-traefik ingress.
if [ "$INSTALL_MODE" = "bare-metal" ] && docker_install_present; then
  fail "A Docker installation is already present at $DOCKER_INSTALL_DIR. Upgrade it in place (re-run with --docker) or remove it first — running both would fight over the same ports and the Traefik ingress."
fi

# r452: running the installer from a DIFFERENT clone silently re-points the
# panel at the clone (fresh .env with a new JWT secret, empty .data) and
# orphans the production tree — the cwd-checkout preference above wins over
# the live unit's directory. The panel's self-update passes
# NINEDEPLOY_INSTALL_DIR explicitly and is immune to this check.
if [ "$INSTALL_MODE" = "bare-metal" ] && [ -f "$BARE_METAL_UNIT_FILE" ]; then
  _unit_dir="$(sed -n 's/^WorkingDirectory=//p' "$BARE_METAL_UNIT_FILE" | tail -1)"
  if [ -n "$_unit_dir" ] && [ "$_unit_dir" != "$INSTALL_DIR" ]; then
    fail "This machine's ninedeploy service runs from $_unit_dir, but the installer would operate on $INSTALL_DIR. Re-run from the installed checkout, or set NINEDEPLOY_INSTALL_DIR=$_unit_dir to upgrade the live install — proceeding from a different clone would create a fresh panel and orphan the existing data."
  fi
fi

# One installer at a time per target dir: a self-update (launched via
# systemd-run by the panel) racing an operator's SSH run interleaves
# rm -rf / tar -x / pnpm install / migrations on ONE tree, and on failure each
# EXIT trap rolls back from its own manifest over whatever the other left.
# Wait up to 30 minutes (a full update takes minutes), then give up loudly.
if command -v flock >/dev/null 2>&1; then
  case "$INSTALL_MODE" in
    docker) _lock_key="$DOCKER_INSTALL_DIR" ;;
    *)      _lock_key="$INSTALL_DIR" ;;
  esac
  _lock_key="$(printf '%s' "$_lock_key" | cksum | tr -d ' ')"
  exec 9>>"/tmp/ninedeploy-install-${_lock_key}.lock"
  # G4.1: BusyBox's flock (Alpine before util-linux is installed) has no -w;
  # it would refuse the option at once and read as a held lock.
  if flock --help 2>&1 | grep -q -- '--wait'; then
    if ! flock -w 1800 9; then
      fail "Another installer run has held the lock for over 30 minutes — investigate that run before starting a new one."
    fi
  elif ! flock 9; then
    fail "Could not take the installer lock."
  fi
fi
ok "Deployment mode: $INSTALL_MODE"

# ── 1. Prerequisites ───────────────────────────────────────────────────────

# Base system packages (G4.1: Debian/Ubuntu, RHEL/Rocky/Alma/Fedora, Alpine, openSUSE/SLES, Arch)
if [ "$(uname -s)" = "Linux" ]; then
  if ! command -v curl &>/dev/null || ! command -v git &>/dev/null || ! command -v tar &>/dev/null || ! command -v sha256sum &>/dev/null; then
    info "Installing base system packages (curl, git, ca-certificates, tar, coreutils)…"
    case "$(host_family)" in
      debian)
        run_apt_step apt-get update -y || true
        pkg_install curl git ca-certificates tar gzip coreutils || true
        ;;
      # curl-minimal stays: swapping it for curl conflicts on RHEL 9.
      rhel|fedora) pkg_install git ca-certificates tar gzip coreutils || true ;;
      alpine) pkg_install curl git ca-certificates tar gzip coreutils util-linux || true ;;
      suse|arch) pkg_install curl git ca-certificates tar gzip coreutils || true ;;
      *) warn "Unknown distribution: install curl, git, tar and coreutils yourself if a later step fails." ;;
    esac
  fi
fi

# Node.js, pnpm and the host Nixpacks CLI drive bare-metal builds and PM2
# services. In Docker mode the panel container bundles its own Node runtime
# and Nixpacks, so none of these are needed on the host.
if [ "$INSTALL_MODE" != "docker" ]; then

# Enforce the supported Node security patch baseline, including after upgrades.
ensure_node

# r570: when packageManager pins a pnpm the host does not have yet, a
# corepack-managed pnpm downloads it on first use — behind an interactive
# "Do you want to continue? [Y/n]" prompt that the panel's self-update (no
# terminal, stdin /dev/null) can never answer. Accept the pinned download.
export COREPACK_ENABLE_DOWNLOAD_PROMPT=0

# pnpm
if ! command -v pnpm &>/dev/null; then
  warn "pnpm not found. Installing pnpm…"
  PNPM_VERSION="11.23.0"
  if [ -f "$INSTALL_DIR/package.json" ]; then
    PNPM_VERSION=$(node -p "require('${INSTALL_DIR}/package.json').packageManager.replace(/^pnpm@/, '').split('+')[0]" 2>/dev/null || echo "11.23.0")
  fi
  (corepack enable 2>/dev/null && corepack prepare "pnpm@${PNPM_VERSION}" --activate 2>/dev/null) || sudo npm install -g "pnpm@${PNPM_VERSION}" || npm install -g "pnpm@${PNPM_VERSION}"
fi
ok "pnpm $(pnpm -v 2>/dev/null || echo 'installed')"

# Nixpacks CLI — required for source repositories without a Dockerfile.
# ghcr.io/railwayapp/nixpacks is a build base image, not a runnable CLI image,
# so install the pinned upstream binary and verify it before exposing it.
#
# Override at install time with: NINEDEPLOY_NIXPACKS_VERSION=1.40.0 ./install.sh
# The version-keyed table (nixpacks_sha256, near the top of this script) only
# carries checksums for versions we know about; an unknown version is rejected
# rather than silently fetched (defence-in-depth against tampered release
# artefacts). To add a new release, drop its checksums in there from the
# official GitHub release page.
NIXPACKS_VERSION="${NINEDEPLOY_NIXPACKS_VERSION:-1.41.0}"

install_nixpacks() {
  case "$(uname -m)" in
    x86_64|amd64)
      NIXPACKS_TARGET="x86_64-unknown-linux-musl"
      NIXPACKS_SHA256="$(nixpacks_sha256 "$NIXPACKS_VERSION" x86_64)"
      ;;
    aarch64|arm64)
      NIXPACKS_TARGET="aarch64-unknown-linux-musl"
      NIXPACKS_SHA256="$(nixpacks_sha256 "$NIXPACKS_VERSION" aarch64)"
      ;;
    *) fail "Nixpacks ${NIXPACKS_VERSION} has no verified binary for architecture $(uname -m)" ;;
  esac

  if [ -z "$NIXPACKS_SHA256" ]; then
    # r574: reachable now that the table is keyed by version (fail-closed).
    fail "Nixpacks ${NIXPACKS_VERSION} is not in the installer's verified-checksum table. Unset NINEDEPLOY_NIXPACKS_VERSION (default 1.41.0) or set it to a known release, or add its checksums to nixpacks_sha256 in install.sh after auditing the GitHub release."
  fi

  NIXPACKS_ASSET="nixpacks-v${NIXPACKS_VERSION}-${NIXPACKS_TARGET}.tar.gz"
  NIXPACKS_STAGE=$(mktemp -d) || fail "Could not create Nixpacks installation workspace"
  curl -fsSL "https://github.com/railwayapp/nixpacks/releases/download/v${NIXPACKS_VERSION}/${NIXPACKS_ASSET}" \
    -o "$NIXPACKS_STAGE/$NIXPACKS_ASSET" || { rm -rf "$NIXPACKS_STAGE"; fail "Could not download Nixpacks ${NIXPACKS_VERSION}"; }
  NIXPACKS_ACTUAL_SHA=$(sha256sum "$NIXPACKS_STAGE/$NIXPACKS_ASSET" | awk '{print $1}')
  if [ "$NIXPACKS_ACTUAL_SHA" != "$NIXPACKS_SHA256" ]; then
    rm -rf "$NIXPACKS_STAGE"
    fail "Nixpacks checksum verification failed; refusing to install an unverified build tool"
  fi
  tar -xzf "$NIXPACKS_STAGE/$NIXPACKS_ASSET" -C "$NIXPACKS_STAGE" nixpacks || { rm -rf "$NIXPACKS_STAGE"; fail "Could not extract Nixpacks"; }
  sudo install -m 0755 "$NIXPACKS_STAGE/nixpacks" /usr/local/bin/nixpacks || { rm -rf "$NIXPACKS_STAGE"; fail "Could not install Nixpacks"; }
  rm -rf "$NIXPACKS_STAGE"
}

if command -v nixpacks &>/dev/null && nixpacks --version 2>/dev/null | grep -q "${NIXPACKS_VERSION}"; then
  ok "Nixpacks $(nixpacks --version 2>/dev/null)"
else
  info "Installing checksum-verified Nixpacks ${NIXPACKS_VERSION} for source builds…"
  install_nixpacks
  nixpacks --version 2>/dev/null | grep -q "${NIXPACKS_VERSION}" || fail "Nixpacks installation verification failed"
  ok "Nixpacks $(nixpacks --version 2>/dev/null)"
fi

# Railpack — Railway's next-gen buildpack, the optional second source-builder
# (buildPack: 'railpack'). Installed the same checksum-verified way so an
# unverified binary can never root-run on this host. Optional: services keep
# working on nixpacks/dockerfile unless the operator picks railpack.
RAILPACK_VERSION="${NINEDEPLOY_RAILPACK_VERSION:-0.39.0}"
RAILPACK_SHA_AMD64_x86_64="728407f5cdb9e9bc1cdd07f568419344a20e71b0a5a9fd90a9cfbaca0a6c94f7"
RAILPACK_SHA_ARM64_aarch64="42eb3fa68e38f44be3610a7d74f714ec0d808d70c105fbe35e053b6e6cfb20be"

install_railpack() {
  case "$(uname -m)" in
    x86_64|amd64)
      RAILPACK_TARGET="x86_64-unknown-linux-musl"
      RAILPACK_SHA256="$RAILPACK_SHA_AMD64_x86_64"
      ;;
    aarch64|arm64)
      RAILPACK_TARGET="arm64-unknown-linux-musl"  # r584: railpack names it arm64 (Nixpacks says aarch64)
      RAILPACK_SHA256="$RAILPACK_SHA_ARM64_aarch64"
      ;;
    *) return 1 ;;
  esac

  RAILPACK_ASSET="railpack-v${RAILPACK_VERSION}-${RAILPACK_TARGET}.tar.gz"
  RAILPACK_STAGE=$(mktemp -d) || return 1
  curl -fsSL "https://github.com/railwayapp/railpack/releases/download/v${RAILPACK_VERSION}/${RAILPACK_ASSET}" \
    -o "$RAILPACK_STAGE/$RAILPACK_ASSET" || { rm -rf "$RAILPACK_STAGE"; return 1; }
  RAILPACK_ACTUAL_SHA=$(sha256sum "$RAILPACK_STAGE/$RAILPACK_ASSET" | awk '{print $1}')
  if [ "$RAILPACK_ACTUAL_SHA" != "$RAILPACK_SHA256" ]; then
    rm -rf "$RAILPACK_STAGE"
    return 1
  fi
  tar -xzf "$RAILPACK_STAGE/$RAILPACK_ASSET" -C "$RAILPACK_STAGE" railpack || { rm -rf "$RAILPACK_STAGE"; return 1; }
  sudo install -m 0755 "$RAILPACK_STAGE/railpack" /usr/local/bin/railpack || { rm -rf "$RAILPACK_STAGE"; return 1; }
  rm -rf "$RAILPACK_STAGE"
}

if command -v railpack &>/dev/null; then
  ok "Railpack $(railpack --version 2>/dev/null | head -1) available (optional railpack buildPack)"
elif install_railpack; then
  ok "Railpack ${RAILPACK_VERSION} installed (optional railpack buildPack)"
else
  info "Railpack optional binary not installed — the railpack build pack stays unavailable (harmless)."
fi

# G4.4: the pack CLI (Cloud Native Buildpacks) for the optional paketo and
# heroku build packs. Same checksum-verified install as Railpack; keep the
# version and SHA-256 pins in sync with the Dockerfile.
PACK_VERSION="${NINEDEPLOY_PACK_VERSION:-0.40.9}"
PACK_SHA_AMD64="dc0ee1e931cf8a106d7555a01a214864f9acb60b77adf15d69b74df4404758e9"
PACK_SHA_ARM64="091ccb213823656c727731537ef8f1000eb4dc3ec61641506653e7f9d6da0c5e"

install_pack() {
  case "$(uname -m)" in
    x86_64|amd64)
      PACK_ASSET="pack-v${PACK_VERSION}-linux.tgz"
      PACK_SHA256="$PACK_SHA_AMD64"
      ;;
    aarch64|arm64)
      PACK_ASSET="pack-v${PACK_VERSION}-linux-arm64.tgz"
      PACK_SHA256="$PACK_SHA_ARM64"
      ;;
    *) return 1 ;;
  esac
  PACK_STAGE=$(mktemp -d) || return 1
  curl -fsSL "https://github.com/buildpacks/pack/releases/download/v${PACK_VERSION}/${PACK_ASSET}" \
    -o "$PACK_STAGE/$PACK_ASSET" || { rm -rf "$PACK_STAGE"; return 1; }
  PACK_ACTUAL_SHA=$(sha256sum "$PACK_STAGE/$PACK_ASSET" | awk '{print $1}')
  if [ "$PACK_ACTUAL_SHA" != "$PACK_SHA256" ]; then
    rm -rf "$PACK_STAGE"
    return 1
  fi
  tar -xzf "$PACK_STAGE/$PACK_ASSET" -C "$PACK_STAGE" pack || { rm -rf "$PACK_STAGE"; return 1; }
  sudo install -m 0755 "$PACK_STAGE/pack" /usr/local/bin/pack || { rm -rf "$PACK_STAGE"; return 1; }
  rm -rf "$PACK_STAGE"
}

if command -v pack &>/dev/null; then
  ok "pack $(pack version 2>/dev/null | head -1) available (optional paketo / heroku buildPack)"
elif install_pack; then
  ok "pack ${PACK_VERSION} installed (optional paketo / heroku buildPack)"
else
  info "pack optional binary not installed — the paketo and heroku build packs stay unavailable (harmless)."
fi

fi # end bare-metal-only prerequisites (Node, pnpm, Nixpacks)

# Docker
if ! command -v docker &>/dev/null; then
  if ! command -v curl &>/dev/null; then
    warn "Install Docker manually: https://docs.docker.com/engine/install/"
    fail "Docker is required."
  fi
  _family="$(host_family)"
  info "Docker not found. Installing Docker Engine for this ${_family} host…"
  info "This is the largest download of the install (~150 MB) and can take several minutes."
  if [ "$_family" = "debian" ] && install_docker_apt; then
    ok "Installed Docker from Docker Inc.'s signed APT repository"
  elif { [ "$_family" = "rhel" ] || [ "$_family" = "fedora" ]; } && install_docker_rpm; then
    ok "Installed Docker from Docker Inc.'s signed RPM repository"
  elif [ "$_family" = "alpine" ] && install_docker_apk; then
    ok "Installed Docker from Alpine's repository"
  elif [ "$_family" = "suse" ] && install_docker_zypper; then
    ok "Installed Docker from the openSUSE/SLES repository"
  elif [ "$_family" = "arch" ] && install_docker_pacman; then
    ok "Installed Docker from Arch's repository"
  else
    warn "No signed Docker repository for this host — falling back to get.docker.com."
    run_vendor_script "https://get.docker.com" \
      || fail "Docker is required. Install it manually: https://docs.docker.com/engine/install/"
  fi
fi
if ! docker info &>/dev/null 2>&1; then
  if command -v systemctl &>/dev/null; then
    warn "Starting Docker service…"
    sudo systemctl start docker || true
  elif command -v rc-service &>/dev/null; then
    warn "Starting Docker service (OpenRC)…"
    sudo rc-service docker start || true
    # dockerd takes a moment to open its socket after the init script returns.
    for _i in 1 2 3 4 5 6 7 8 9 10; do docker info &>/dev/null 2>&1 || sudo docker info &>/dev/null 2>&1 && break; sleep 1; done
  fi
fi
if [ "$(uname -s)" = "Linux" ] && [ "$(id -u)" -ne 0 ]; then
  CURRENT_USER="$(id -un)"
  if ! groups "$CURRENT_USER" 2>/dev/null | grep -q '\bdocker\b'; then
    info "Adding user $CURRENT_USER to docker group…"
    sudo usermod -aG docker "$CURRENT_USER" 2>/dev/null || true
  fi
fi
if docker info &>/dev/null 2>&1; then
  DOCKER=(docker)
elif sudo docker info &>/dev/null 2>&1; then
  # A just-added docker group is not visible in the current shell. Keep this
  # one installer run reliable; the systemd service starts with its own groups.
  DOCKER=(sudo docker)
else
  warn "Docker daemon not running. Start it and re-run."
  fail "Docker daemon must be running."
fi
docker_cmd() { "${DOCKER[@]}" "$@"; }
ok "Docker $(docker_cmd --version 2>/dev/null | awk '{print $3}' | tr -d ',' || echo 'installed')"

# Boot resilience: a Docker daemon that is merely running NOW but not enabled
# in systemd silently stays down after the next reboot. Docker's restart
# policies then never fire, so Traefik and every deployed container stay dead,
# and the NineDeploy unit (Requires=docker.service) goes down with them. Ensure
# boot enablement unconditionally — including when this installer did not
# install Docker itself.
if [ "$(uname -s)" = "Linux" ] && command -v systemctl &>/dev/null; then
  sudo systemctl enable docker.service >/dev/null 2>&1 || true
  sudo systemctl enable docker.socket >/dev/null 2>&1 || true
  sudo systemctl enable containerd.service >/dev/null 2>&1 || true
  if systemctl is-enabled --quiet docker.service 2>/dev/null \
    || systemctl is-enabled --quiet docker.socket 2>/dev/null; then
    ok "Docker is enabled at boot (systemd)"
  else
    # Not a hard failure: snap-managed or socket-activated Docker installs can
    # be boot-persistent without these exact unit names. Warn loudly instead.
    warn "Could not verify Docker boot enablement via systemd units."
    warn "If Docker is not managed by snap, run: sudo systemctl enable docker.service"
  fi
elif [ "$(uname -s)" = "Linux" ] && command -v rc-update &>/dev/null; then
  # G4.1: OpenRC (Alpine).
  sudo rc-update add docker default >/dev/null 2>&1 || true
  if rc-update show default 2>/dev/null | grep -q '^ *docker '; then
    ok "Docker is enabled at boot (OpenRC)"
  else
    warn "Could not verify Docker boot enablement. Run: rc-update add docker default"
  fi
fi

# Swap space (Linux)
# Low-memory VPS nodes (<= 4GB RAM) without swap risk OOM-kills when pulling
# or unpacking large Docker images (e.g. n8n, Supabase, Postgres).
if [ "$(uname -s)" = "Linux" ]; then
  SWAP_TOTAL=$(free -m 2>/dev/null | awk '/^Swap:/ {print $2}' || echo "0")
  if [ -n "$SWAP_TOTAL" ] && [ "${SWAP_TOTAL:-0}" -lt 1024 ]; then
    MEM_TOTAL=$(free -m 2>/dev/null | awk '/^Mem:/ {print $2}' || echo "2048")
    if [ "${MEM_TOTAL:-2048}" -le 4096 ]; then
      info "Low swap detected (${SWAP_TOTAL:-0}MB on ${MEM_TOTAL:-2048}MB RAM). Configuring 2GB swapfile for reliable Docker operations…"
      if [ ! -f /swapfile ]; then
        (sudo fallocate -l 2G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=2048) 2>/dev/null || true
        sudo chmod 600 /swapfile 2>/dev/null || true
        sudo mkswap /swapfile >/dev/null 2>&1 || true
      fi
      if sudo swapon /swapfile >/dev/null 2>&1; then
        ok "2GB swapfile activated"
        if ! grep -Eq '^[^#]*/swapfile' /etc/fstab 2>/dev/null; then
          echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab >/dev/null 2>&1 || true
        fi
      else
        warn "Could not activate swapfile (may be running inside container or non-root) — continuing."
      fi
    fi
  elif [ -n "$SWAP_TOTAL" ] && [ "${SWAP_TOTAL:-0}" -ge 1024 ]; then
    ok "Swap space (${SWAP_TOTAL}MB)"
  fi
fi

# Network & Ingress Prerequisites
info "Preparing Docker network and Traefik proxy…"
if ! docker_cmd network inspect ninedeploy >/dev/null 2>&1; then
  docker_cmd network create ninedeploy >/dev/null || fail "Could not create the required Docker network 'ninedeploy'"
fi

# A healthy containerd overlayfs store always has this directory alongside
# metadata.db. Some interrupted Docker 29 cleanups leave the metadata database
# behind but remove the physical snapshot root, making every pull/import fail.
# Restore only the missing directory; never remove or rewrite snapshot data.
if [ -S /var/run/docker/containerd/containerd.sock ]; then
  CONTAINERD_OVERLAY_ROOT="/var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.overlayfs"
else
  CONTAINERD_OVERLAY_ROOT="/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs"
fi
CONTAINERD_SNAPSHOT_DIR="$CONTAINERD_OVERLAY_ROOT/snapshots"
DOCKER_STORAGE_DRIVER=$(docker_cmd info --format '{{.Driver}}' 2>/dev/null || true)
if [ "$DOCKER_STORAGE_DRIVER" = "overlayfs" ] && [ ! -d "$CONTAINERD_SNAPSHOT_DIR" ]; then
  warn "Docker's containerd overlayfs snapshot directory is missing; restoring the required host directory…"
  sudo install -d -o root -g root -m 0700 "$CONTAINERD_SNAPSHOT_DIR" || fail "Could not restore $CONTAINERD_SNAPSHOT_DIR"
  [ -d "$CONTAINERD_SNAPSHOT_DIR" ] || fail "Containerd snapshot directory is still unavailable after repair"
  ok "Containerd overlayfs snapshot directory restored"
fi

# Docker 29's containerd store can retain a broken Alpine snapshot forever.
# Do not loop, prune, restart Docker, or mutate containerd metadata here. A
# single registry attempt is enough; the verified binary fallback below has no
# dependency on the conflicting Alpine layer.
traefik_image_usable() {
  docker_cmd image inspect traefik:3 >/dev/null 2>&1 || return 1
  IMAGE_VERSION=$(docker_cmd run --rm traefik:3 version 2>/dev/null || true)
  printf '%s\n' "$IMAGE_VERSION" | grep -Eq 'Version:[[:space:]]+3\.'
}

build_traefik_fallback_image() {
  TRAEFIK_RELEASE="v3.7.11"
  case "$(uname -m)" in
    x86_64|amd64) TRAEFIK_ARCH="amd64" ;;
    aarch64|arm64) TRAEFIK_ARCH="arm64" ;;
    armv7l) TRAEFIK_ARCH="armv7" ;;
    armv6l) TRAEFIK_ARCH="armv6" ;;
    *) warn "No verified Traefik binary fallback is available for architecture $(uname -m)."; return 1 ;;
  esac

  TRAEFIK_ASSET="traefik_${TRAEFIK_RELEASE}_linux_${TRAEFIK_ARCH}.tar.gz"
  TRAEFIK_RELEASE_URL="https://github.com/traefik/traefik/releases/download/${TRAEFIK_RELEASE}"
  TRAEFIK_STAGE=$(mktemp -d) || return 1
  mkdir -p "$TRAEFIK_STAGE/rootfs/etc/ssl/certs" "$TRAEFIK_STAGE/rootfs/tmp" || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  chmod 1777 "$TRAEFIK_STAGE/rootfs/tmp" || { rm -rf "$TRAEFIK_STAGE"; return 1; }

  info "Docker's Alpine snapshot is unusable; building a minimal Traefik ${TRAEFIK_RELEASE} image from the official release assets…"
  curl -fsSL "$TRAEFIK_RELEASE_URL/$TRAEFIK_ASSET" -o "$TRAEFIK_STAGE/$TRAEFIK_ASSET" || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  curl -fsSL "$TRAEFIK_RELEASE_URL/traefik_${TRAEFIK_RELEASE}_checksums.txt" -o "$TRAEFIK_STAGE/checksums.txt" || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  EXPECTED_SHA=$(awk -v asset="$TRAEFIK_ASSET" '$2 == asset { print $1; exit }' "$TRAEFIK_STAGE/checksums.txt")
  ACTUAL_SHA=$(sha256sum "$TRAEFIK_STAGE/$TRAEFIK_ASSET" | awk '{print $1}')
  if [ -z "$EXPECTED_SHA" ] || [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
    warn "Traefik release checksum verification failed; refusing to create a local image."
    rm -rf "$TRAEFIK_STAGE"
    return 1
  fi

  tar -xzf "$TRAEFIK_STAGE/$TRAEFIK_ASSET" -C "$TRAEFIK_STAGE" traefik || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  install -m 0755 "$TRAEFIK_STAGE/traefik" "$TRAEFIK_STAGE/rootfs/traefik" || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  if [ -f /etc/ssl/certs/ca-certificates.crt ]; then
    install -m 0644 /etc/ssl/certs/ca-certificates.crt "$TRAEFIK_STAGE/rootfs/etc/ssl/certs/ca-certificates.crt" || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  else
    warn "Host CA certificate bundle is unavailable; refusing to create a TLS proxy image."
    rm -rf "$TRAEFIK_STAGE"
    return 1
  fi

  tar -C "$TRAEFIK_STAGE/rootfs" -cf - . | docker_cmd import \
    --change 'ENTRYPOINT ["/traefik"]' \
    --change 'EXPOSE 80 443' \
    --change 'LABEL org.opencontainers.image.source="https://github.com/traefik/traefik"' \
    --change "LABEL org.opencontainers.image.version=\"$TRAEFIK_RELEASE\"" \
    - traefik:3 >/dev/null || { rm -rf "$TRAEFIK_STAGE"; return 1; }
  rm -rf "$TRAEFIK_STAGE"

  LOCAL_TRAEFIK_VERSION=$(docker_cmd run --rm traefik:3 version 2>/dev/null || true)
  printf '%s\n' "$LOCAL_TRAEFIK_VERSION" | grep -Eq "Version:[[:space:]]+${TRAEFIK_RELEASE#v}$" || {
    warn "The locally constructed Traefik image failed its version probe."
    return 1
  }
  ok "Verified minimal Traefik ${TRAEFIK_RELEASE} image created without the corrupt Alpine snapshot"
}

if traefik_image_usable; then
  ok "Existing Traefik v3 image verified; skipping registry pull"
else
  # The pull output is captured (printed after completion on success), so say
  # what is happening during the silent ~100 MB download window.
  info "Pulling the Traefik v3 image (~100 MB) — output follows when it finishes…"
  if PULL_OUTPUT=$(docker_cmd pull traefik:3 2>&1) && traefik_image_usable; then
    printf '%s\n' "$PULL_OUTPUT"
  else
    printf '%s\n' "$PULL_OUTPUT" >&2
    warn "Docker Hub image is unusable; switching immediately to the verified layer-free Traefik image…"
    build_traefik_fallback_image || fail "Could not provision Traefik from Docker Hub or verified upstream release assets; ingress cannot operate"
  fi
fi

# Ports 80/443 belong to the Traefik ingress. r574: a stock apache2/nginx
# that holds them is stopped and disabled ONLY on a fresh install, and only
# when it actually listens there — every run used to stop and disable both
# whenever they were merely active, including upgrades and the panel's
# self-update, silently killing a web server an operator had deliberately
# re-enabled on other ports.
#   web_ports_holder <process>: 0 = it listens on :80/:443, 1 = it does not,
#   2 = unknown (no `ss` to ask).
web_ports_holder() {
  local _listen
  command -v ss >/dev/null 2>&1 || return 2
  _listen=$(sudo ss -H -ltnp 2>/dev/null || ss -H -ltnp 2>/dev/null || true)
  printf '%s\n' "$_listen" | grep -E '[:.](80|443)[[:space:]]' | grep -q "\"$1\"" && return 0
  return 1
}
if [ "$(uname -s)" = "Linux" ] && command -v systemctl &>/dev/null; then
  _fresh_host=true
  if bare_metal_present || docker_install_present; then _fresh_host=false; fi
  for _web_svc in apache2 nginx; do
    systemctl is-active --quiet "$_web_svc" 2>/dev/null || continue
    _holder_rc=0
    web_ports_holder "$_web_svc" || _holder_rc=$?
    if [ "$_holder_rc" -eq 1 ]; then
      continue # active, but not on :80/:443 — no conflict
    fi
    if [ "$_fresh_host" = true ] && [ "$_holder_rc" -eq 0 ]; then
      warn "$_web_svc is listening on port 80/443, which the Traefik ingress needs — stopping and disabling the $_web_svc service."
      warn "To keep $_web_svc, move it to other ports and re-enable it: sudo systemctl enable --now $_web_svc"
      sudo systemctl stop "$_web_svc" 2>/dev/null || true
      sudo systemctl disable "$_web_svc" 2>/dev/null || true
    elif [ "$_holder_rc" -eq 0 ]; then
      warn "$_web_svc is listening on port 80/443 — Traefik cannot serve domains until it moves (this upgrade leaves it running; to free the ports: sudo systemctl disable --now $_web_svc)."
    else
      warn "$_web_svc is running and its ports could not be checked (no 'ss'). If it holds port 80/443, Traefik cannot serve domains: sudo systemctl disable --now $_web_svc"
    fi
  done
fi

# Host Firewall (UFW on Linux)
if [ "$(uname -s)" = "Linux" ] && command -v ufw &>/dev/null; then
  info "Configuring host firewall (UFW) with safe defaults…"
  sudo ufw allow 22/tcp comment 'SSH' >/dev/null 2>&1 || true
  sudo ufw allow 80/tcp comment 'HTTP (Traefik)' >/dev/null 2>&1 || true
  sudo ufw allow 443/tcp comment 'HTTPS (Traefik)' >/dev/null 2>&1 || true
  if [ -n "${NINEDEPLOY_PORT:-}" ] && [ "${NINEDEPLOY_PORT:-3000}" != "80" ] && [ "${NINEDEPLOY_PORT:-3000}" != "443" ]; then
    sudo ufw allow "${NINEDEPLOY_PORT}/tcp" comment 'NineDeploy Direct Panel' >/dev/null 2>&1 || true
  fi
  ok "Firewall rules updated (UFW: 22, 80, 443 permitted)"
fi
ok "Docker network & ingress ready"

# ── 2. Resolve the version to install ──────────────────────────────────────
#
# Channel:
#   release (default) — latest vX.Y.Z git tag (stable)
#   main              — track the main branch (edge; previous behaviour)
# A specific tag can be pinned with --version vX.Y.Z / NINEDEPLOY_VERSION.

# Source of the tree the installer is about to build. `release` means the
# published tarball for a tag; `git` means a clone/checkout. Set in section 3.
SOURCE_MODE=""
# Records the installed release so a later upgrade knows what is on disk
# without needing a git history.
RELEASE_STAMP_FILE=".ninedeploy-release"

# Download and unpack the source archive of a release tag.
#   $1  tag (vX.Y.Z)
#   $2  directory to unpack into (created; the archive's single top-level
#       directory is stripped)
# Returns non-zero — leaving no partial tree behind — when the release does
# not exist, the download fails, or the archive does not look like NineDeploy.
# r702: EXITS (fail) — never falls back to another source — when the release's
# published checksums or their signature say the bytes are not the release's.
fetch_release_tarball() {
  local _ref="$1" _dest="$2" _stage _archive _url _code _asset _want _got _sig_rc
  _stage=$(mktemp -d) || return 1
  _archive="$_stage/ninedeploy-${_ref}.tar.gz"

  # r702: prefer the release's own archive, checked against its SHA256SUMS.
  # The HTTP status decides the path: 200 = this release publishes checksums,
  # 404 = it does not (a tag older than CHECKSUMS_SINCE), anything else = the
  # asset host could not be asked.
  _code=$(curl -sSL --retry 3 --retry-delay 2 -m 60 -o "$_stage/SHA256SUMS" -w '%{http_code}' \
    "$(release_asset_url "$_ref" SHA256SUMS)" 2>/dev/null || true)
  if [ "$_code" = "200" ]; then
    # The signature first: nothing SHA256SUMS says is used before it is
    # checked (when this host can check it).
    _sig_rc=0
    if [ "${NINEDEPLOY_SKIP_SIGNATURE_VERIFY:-}" = "1" ]; then
      _sig_rc=4
    elif command -v cosign >/dev/null 2>&1; then
      if ! curl -fsSL --retry 3 --retry-delay 2 -m 60 -o "$_stage/SHA256SUMS.sigstore.json" \
          "$(release_asset_url "$_ref" SHA256SUMS.sigstore.json)"; then
        rm -rf "$_stage"
        fail "Could not download SHA256SUMS.sigstore.json for $_ref, so cosign cannot verify the release checksums — refusing. Retry later, or re-run with NINEDEPLOY_SKIP_SIGNATURE_VERIFY=1 to rely on the checksum alone."
      fi
      verify_checksums_signature "$_stage/SHA256SUMS" "$_stage/SHA256SUMS.sigstore.json" || _sig_rc=$?
    else
      _sig_rc=2
    fi
    case "$_sig_rc" in
      0) ok "SHA256SUMS for $_ref is signed by NineDeploy's release workflow (cosign, Sigstore keyless)" ;;
      2) info "cosign is not installed — the release signature is not checked (the archive checksum is)" ;;
      3) warn "This host's cosign is too old to read Sigstore bundles (v2.4+ needed) — the release signature is not checked (the archive checksum is)" ;;
      4) warn "NINEDEPLOY_SKIP_SIGNATURE_VERIFY=1 — the release signature is not checked (the archive checksum is)" ;;
      *)
        rm -rf "$_stage"
        fail "The signature on $_ref's SHA256SUMS did NOT verify against NineDeploy's release workflow — refusing to install it (nothing was replaced). If this host cannot reach Sigstore, re-run with NINEDEPLOY_SKIP_SIGNATURE_VERIFY=1 to rely on the checksum alone."
        ;;
    esac

    _asset=$(release_archive_name "$_ref")
    if ! _want=$(sha256sums_lookup "$_asset" < "$_stage/SHA256SUMS"); then
      rm -rf "$_stage"
      fail "$_ref's SHA256SUMS does not list exactly one checksum for $_asset — refusing to install an archive it cannot verify (nothing was replaced)."
    fi
    _url=$(release_asset_url "$_ref" "$_asset")
    if ! curl -fsSL --retry 3 --retry-delay 2 -m 600 "$_url" -o "$_archive"; then
      warn "Could not download $_asset from the $_ref release"
      rm -rf "$_stage"
      return 1
    fi
    _got=$(sha256sum < "$_archive" | awk '{print $1}')
    if [ "$_got" != "$_want" ]; then
      rm -rf "$_stage"
      fail "Checksum mismatch for $_asset: SHA256SUMS lists $_want, the download is $_got — refusing to install it (nothing was replaced)."
    fi
    ok "$_asset matches the release's SHA256SUMS ($_want)"
  elif release_has_checksums "$_ref"; then
    # A release that publishes checksums, but they could not be fetched: its
    # unverified tag archive is never the substitute.
    warn "Could not fetch SHA256SUMS for $_ref (HTTP ${_code:-000}) — not installing an unverified archive of a release that publishes checksums"
    rm -rf "$_stage"
    return 1
  else
    warn "$_ref predates published release checksums — installing GitHub's tag archive, verified by TLS only (as before)"
    _url="https://github.com/${REPO_SLUG}/archive/refs/tags/${_ref}.tar.gz"
    if ! curl -fsSL --retry 3 --retry-delay 2 -m 600 "$_url" -o "$_archive"; then
      rm -rf "$_stage"
      return 1
    fi
  fi
  # A 404 page or an HTML error would still be a file; make tar prove it.
  if ! tar -tzf "$_archive" >/dev/null 2>&1; then
    rm -rf "$_stage"
    return 1
  fi

  mkdir -p "$_dest" || { rm -rf "$_stage"; return 1; }
  # Every failure past this point leaves a partial tree behind. The caller
  # falls back to `git clone`, which refuses a non-empty target, so unwind it.
  if ! tar -xzf "$_archive" -C "$_dest" --strip-components=1; then
    rm -rf "$_stage" "${_dest:?}"
    return 1
  fi
  rm -rf "$_stage"

  # The archive must be the monorepo root, not some other project's tarball.
  if [ ! -f "$_dest/package.json" ] || [ ! -f "$_dest/pnpm-workspace.yaml" ]; then
    warn "The tarball for $_ref does not look like a NineDeploy source tree"
    rm -rf "${_dest:?}"
    return 1
  fi

  # ── Provenance: what is guaranteed about the extracted tree. ────────────
  # r702, a release that publishes checksums (>= CHECKSUMS_SINCE):
  #   * the archive is byte-for-byte the one SHA256SUMS lists (fail-closed);
  #   * with cosign on the host, SHA256SUMS was signed by THIS repository's
  #     release-publish.yml, run for a vX.Y.Z tag or from main (Fulcio cert
  #     over GitHub's OIDC token, recorded in Rekor) — fail-closed;
  #   * without cosign, SHA256SUMS is trusted as GitHub served it over TLS:
  #     the checksum then proves only that the archive is the one published
  #     beside it, not who published it.
  #   The trust root is GitHub (account, Actions, OIDC) plus Sigstore — there
  #   is no maintainer-held key, so a compromise of the repository's release
  #   workflow still signs.
  # An older tag (or the codeload fallback): TLS to github.com only.
  # Either way the checks below add what they can prove about the CONTENT:
  # its package.json version matches the requested tag (a swapped or stale
  # archive — another release's tarball — fails here), and it carries no
  # symlinks and no setuid/setgid files. r574: those checks alone do NOT prove
  # authenticity — an archive modified at the source with the version string
  # left intact passes them.
  _tar_version=$(sed -n 's/.*"version":[[:space:]]*"\([^"]*\)".*/\1/p' "$_dest/package.json" | head -1)
  if [ "v$_tar_version" != "$_ref" ]; then
    warn "Tarball for $_ref carries version '${_tar_version:-?}' — content does not match the tag, refusing"
    rm -rf "${_dest:?}"
    return 1
  fi
  # A source tree must carry no symlinks (extraction-time escapes) and no
  # setuid/setgid bits (privilege seeds for the root-built service).
  if find "$_dest" -type l -print -quit 2>/dev/null | grep -q .; then
    warn "Tarball for $_ref contains symlinks — refusing"
    rm -rf "${_dest:?}"
    return 1
  fi
  if find "$_dest" -type f -perm /6000 -print -quit 2>/dev/null | grep -q .; then
    warn "Tarball for $_ref contains setuid/setgid files — refusing"
    rm -rf "${_dest:?}"
    return 1
  fi
  info "Tarball checks passed: $_ref, version $_tar_version, no symlinks, no setuid bits"
  return 0
}

REF=""
if [ -n "$PINNED_VERSION" ]; then
  REF="$PINNED_VERSION"
  info "Installing pinned version $REF"
elif [ "$CHANNEL" = "main" ]; then
  REF="main"
  info "Edge channel: tracking the main branch"
elif [ "$CHANNEL" = "stable" ]; then
  REF=$(stable_tag)
  [ -n "$REF" ] || fail "The stable channel has no release yet (no stable tag on GitHub). Use --channel release, or pin one with --version vX.Y.Z."
  info "Stable channel: installing $REF"
else
  REF=$(latest_tag)
  if [ -z "$REF" ]; then
    warn "No release tags found — falling back to the main branch"
    REF="main"
  else
    info "Release channel: installing $REF (latest tag)"
  fi
fi

# ── 2b. Docker mode: deploy the panel as a container and finish ──────────
#
# Everything above is shared with bare-metal (Docker daemon, boot enablement,
# swap, the ninedeploy network, the Traefik image, port and firewall prep).
# From here the modes diverge: Docker mode never clones, builds or installs
# systemd units — it fetches the pinned compose file, maintains a 0600 .env
# next to it, and lets compose own the lifecycle. Re-running the installer
# in this mode IS the upgrade path (same volume, same secrets).
install_docker_mode() {
  # Compose v2 ships with the signed Docker APT repo this installer uses,
  # but a pre-existing Docker install may lack the plugin.
  if ! docker_cmd compose version >/dev/null 2>&1; then
    info "Docker Compose v2 plugin missing — installing it…"
    case "$(host_family)" in
      alpine) pkg_install docker-cli-compose ;;
      suse|arch) pkg_install docker-compose ;;
      *) pkg_install docker-compose-plugin ;;
    esac \
      || fail "Docker Compose v2 is required for the Docker install mode. Install the Compose plugin (docker-compose-plugin, or docker-cli-compose on Alpine) and re-run."
  fi

  # Install dir owned by the invoking user, so .env and compose state stay
  # manageable without sudo (root invocations keep root ownership).
  if [ "$(id -u)" -ne 0 ]; then
    sudo install -d -o "$(id -u)" -g "$(id -g)" -m 0755 "$DOCKER_INSTALL_DIR" \
      || fail "Could not create $DOCKER_INSTALL_DIR"
  else
    install -d -m 0755 "$DOCKER_INSTALL_DIR" || fail "Could not create $DOCKER_INSTALL_DIR"
  fi

  # Compose file for $REF: prefer a checkout the installer was run from,
  # otherwise fetch exactly this file from the same origin as install.sh.
  if [ -f "./docker-compose.prod.yml" ] && [ -f "./package.json" ]; then
    info "Using docker-compose.prod.yml from the current checkout"
    cp ./docker-compose.prod.yml "$DOCKER_INSTALL_DIR/docker-compose.yml.new" \
      || fail "Could not stage the compose file"
  else
    info "Fetching docker-compose.prod.yml for $REF…"
    curl -fsSL --retry 3 --retry-delay 2 "https://raw.githubusercontent.com/NineDeploy/NineDeploy/${REF}/docker-compose.prod.yml" \
      -o "$DOCKER_INSTALL_DIR/docker-compose.yml.new" \
      || fail "Could not fetch docker-compose.prod.yml for $REF"
  fi
  # The compose file drives a root-equivalent container deployment, so the
  # fetched content is validated BEFORE it is trusted: the services key must
  # exist, the project must reference exactly ONE image and it must be this
  # panel's own repository, and the compose engine itself must accept the
  # file (dummy values satisfy the required-variable defaults at parse time).
  # The dummy values travel in an --env-file: a `VAR=x docker_cmd …` prefix
  # dies under `DOCKER=(sudo docker)` — sudo's env_reset strips it before
  # docker runs, so a FRESH install (no .env beside the compose file yet)
  # failed this check on a perfectly good compose file.
  grep -q '^services:' "$DOCKER_INSTALL_DIR/docker-compose.yml.new" \
    || fail "The fetched compose file does not look right (missing 'services:') — refusing to deploy it"
  if [ "$(grep -cE '^[[:space:]]*image:[[:space:]]*' "$DOCKER_INSTALL_DIR/docker-compose.yml.new")" != "1" ] \
    || ! grep -Eq "image:[[:space:]]*ghcr\.io/${IMAGE_REPO//./\\.}:" "$DOCKER_INSTALL_DIR/docker-compose.yml.new"; then
    fail "The fetched compose file references an unexpected image — refusing to deploy it"
  fi
  _prov_env="$(mktemp)"
  printf 'NINEDEPLOY_JWT_SECRET=provenance-check\nDOCKER_GID=1\n' > "$_prov_env"
  if ! docker_cmd compose --env-file "$_prov_env" \
      --file "$DOCKER_INSTALL_DIR/docker-compose.yml.new" config --quiet; then
    rm -f "$_prov_env"
    fail "The fetched compose file is not a valid compose project — refusing to deploy it"
  fi
  rm -f "$_prov_env"

  # r570: keep the RUNNING release's compose file and .env before either is
  # replaced, so a failed pull/recreate below can put them back (see
  # docker_rollback_panel). 0700: the .env copy holds the secrets.
  DOCKER_ROLLBACK_DIR="$DOCKER_INSTALL_DIR/.upgrade-rollback"
  rm -rf "${DOCKER_ROLLBACK_DIR:?}"
  (umask 077 && mkdir -p "$DOCKER_ROLLBACK_DIR") || fail "Could not create $DOCKER_ROLLBACK_DIR"
  if [ -f "$DOCKER_INSTALL_DIR/docker-compose.yml" ]; then
    cp -p "$DOCKER_INSTALL_DIR/docker-compose.yml" "$DOCKER_ROLLBACK_DIR/docker-compose.yml" \
      || fail "Could not keep the previous compose file for rollback"
  fi
  if [ -f "$DOCKER_INSTALL_DIR/.env" ]; then
    cp -p "$DOCKER_INSTALL_DIR/.env" "$DOCKER_ROLLBACK_DIR/.env" \
      || fail "Could not keep the previous .env for rollback"
  fi

  mv "$DOCKER_INSTALL_DIR/docker-compose.yml.new" "$DOCKER_INSTALL_DIR/docker-compose.yml"

  # The image tag the release workflow tagged for this ref (docker_image_tag):
  # the pinned `--version` (r263), `:edge` for the main channel, and — r571 —
  # the RESOLVED release tag for the release channel instead of `:latest`.
  # This release's compose file reads it from .env as
  # ${NINEDEPLOY_IMAGE_TAG:-latest} (written below). Upgrade-safe: an .env
  # without the variable keeps `:latest` until the installer writes it.
  IMAGE_TAG="latest"
  if [ -n "$PINNED_VERSION" ]; then
    IMAGE_TAG="$PINNED_VERSION"
  else
    IMAGE_TAG="$(docker_image_tag "" "$CHANNEL" "$REF")"
  fi
  # Compose lets the shell environment override .env; the value this run
  # resolved must be the one that is pulled and started.
  unset NINEDEPLOY_IMAGE_TAG
  # A compose file from an older release (a pinned downgrade) hardcodes
  # `:latest` — substitute it in place as before.
  if grep -q 'ghcr.io/.*:latest' "$DOCKER_INSTALL_DIR/docker-compose.yml"; then
    sed -i.bak "s|ghcr.io/${IMAGE_REPO}:latest|ghcr.io/${IMAGE_REPO}:${IMAGE_TAG}|" \
      "$DOCKER_INSTALL_DIR/docker-compose.yml" \
      && rm -f "$DOCKER_INSTALL_DIR/docker-compose.yml.bak" \
      || fail "Could not substitute the image tag in docker-compose.yml"
  fi
  info "Compose file pinned to ghcr.io/${IMAGE_REPO}:${IMAGE_TAG}"

  cd "$DOCKER_INSTALL_DIR"

  # .env: created 0600, then upserted per run. The JWT secret survives
  # upgrades (rotating it would invalidate every session); DOCKER_GID is
  # refreshed every run because the host's docker group id can change.
  gen_secret() {
    if command -v openssl &>/dev/null; then openssl rand -hex 32
    else head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n'; fi
  }
  upsert_env() { # <key> <value>
    # r449: the value is operator-supplied env (JWT secret, DNS token…) and
    # must never ride through a sed REPLACEMENT — `&`, `|`, backslashes and
    # newlines corrupted the secret or aborted the run after .env was already
    # rewritten. Filter-and-append with printf instead.
    if [ -f .env ] && grep -q "^$1=" .env; then
      local _tmp
      _tmp="$(mktemp)"
      { grep -v "^$1=" .env; printf '%s=%s\n' "$1" "$2"; } > "$_tmp" \
        && cat "$_tmp" > .env \
        && rm -f "$_tmp"
    else
      printf '%s=%s\n' "$1" "$2" >> .env
    fi
  }
  local old_umask jwt_secret docker_gid
  old_umask=$(umask)
  umask 077
  # 0.16: a first install (no .env yet) gets the 0.0.0.0 bind, a detected
  # panel URL and a setup token; an upgrade keeps whatever it had.
  local fresh_env=false
  [ -f .env ] || fresh_env=true
  [ -f .env ] || touch .env
  jwt_secret="$(sed -n 's/^NINEDEPLOY_JWT_SECRET=//p' .env | tail -1)"
  [ -n "${NINEDEPLOY_JWT_SECRET:-}" ] && jwt_secret="$NINEDEPLOY_JWT_SECRET"
  [ -z "$jwt_secret" ] && jwt_secret="$(gen_secret)"
  docker_gid="$(getent group docker | cut -d: -f3)"
  [ -n "$docker_gid" ] || fail "Could not resolve the host docker group id (getent group docker)"
  upsert_env NINEDEPLOY_JWT_SECRET "$jwt_secret"
  upsert_env DOCKER_GID "$docker_gid"
  # r448: an operator-chosen port is a SETTING, not an ambient default — read
  # it back from .env first so a re-run without the env var does not silently
  # flip the published mapping to 3000 (which also collides with whatever made
  # them change ports, and fails the recreate below).
  local panel_port
  panel_port="$(sed -n 's/^NINEDEPLOY_PORT=//p' .env | tail -1)"
  [ -n "${NINEDEPLOY_PORT:-}" ] && panel_port="$NINEDEPLOY_PORT"
  [ -z "$panel_port" ] && panel_port="3000"
  upsert_env NINEDEPLOY_PORT "$panel_port"
  # r571: the image tag this run resolved (read by the compose file as
  # ${NINEDEPLOY_IMAGE_TAG:-latest}). Rewritten on every run: re-running the
  # installer is choosing the version.
  upsert_env NINEDEPLOY_IMAGE_TAG "$IMAGE_TAG"
  # r451: persist an explicit bind address too — the compose file defaults the
  # panel to loopback, and an operator's NINEDEPLOY_BIND=0.0.0.0 install would
  # silently rebind on the next manual `docker compose up -d` without this.
  if [ -n "${NINEDEPLOY_BIND:-}" ] && ! grep -q '^NINEDEPLOY_BIND=' .env; then
    printf 'NINEDEPLOY_BIND=%s\n' "$NINEDEPLOY_BIND" >> .env
  elif [ "$fresh_env" = true ] && ! grep -q '^NINEDEPLOY_BIND=' .env; then
    # 0.16: publish the panel on every interface unless --private. Existing
    # installs without the line keep the compose file's loopback default.
    if [ "$PANEL_PRIVATE" = "1" ]; then
      printf 'NINEDEPLOY_BIND=127.0.0.1\n' >> .env
    elif ref_has_setup_token "$REF"; then
      printf 'NINEDEPLOY_BIND=0.0.0.0\n' >> .env
    else
      # A pre-0.16 release has no setup-token gate: keep its loopback bind.
      printf 'NINEDEPLOY_BIND=127.0.0.1\n' >> .env
    fi
  fi
  # 0.16: one-time first-admin token (see the helpers near the top).
  if ! grep -q '^NINEDEPLOY_SETUP_TOKEN=' .env; then
    if [ -n "${NINEDEPLOY_SETUP_TOKEN:-}" ]; then
      upsert_env NINEDEPLOY_SETUP_TOKEN "$NINEDEPLOY_SETUP_TOKEN"
    elif [ "$fresh_env" = true ]; then
      upsert_env NINEDEPLOY_SETUP_TOKEN "$(gen_setup_token)"
    fi
  fi
  # Optional integrations: written only when provided by the environment and
  # never overwritten once present, so operator edits survive upgrades.
  for _var in NINEDEPLOY_PUBLIC_URL NINEDEPLOY_ACME_EMAIL NINEDEPLOY_DNS_PROVIDER NINEDEPLOY_DNS_TOKEN; do
    _val="${!_var:-}"
    if [ -n "$_val" ] && ! grep -q "^${_var}=" .env; then
      printf '%s=%s\n' "$_var" "$_val" >> .env
    fi
  done
  # A6: `--channel stable` also makes the panel offer stable releases. A choice
  # saved under Settings → System wins over it; never written for a pinned run.
  if [ "$CHANNEL" = "stable" ] && [ -z "$PINNED_VERSION" ]; then
    upsert_env NINEDEPLOY_UPDATE_CHANNEL stable
  fi
  # A2: the panel domain and its certificate email (helpers near the top).
  if [ "$fresh_env" = true ]; then prompt_panel_domain; fi
  if [ -n "$ACME_EMAIL_ARG" ]; then upsert_env NINEDEPLOY_ACME_EMAIL "$ACME_EMAIL_ARG"; fi
  if [ -n "$PANEL_DOMAIN_ARG" ]; then
    local panel_ip
    # A compose file from an older pinned release does not pass the variable.
    grep -q 'NINEDEPLOY_DOMAIN' docker-compose.yml \
      || warn "This release's compose file does not pass NINEDEPLOY_DOMAIN to the panel: after signing in, set the panel domain under Settings -> Security."
    upsert_env NINEDEPLOY_DOMAIN "$PANEL_DOMAIN_ARG"
    # The public URL becomes the domain; fresh installs still seed sslip.io
    # service URLs from this address.
    panel_ip="$(detect_server_ip)"
    if [ -n "$panel_ip" ]; then upsert_env NINEDEPLOY_PUBLIC_IP "$panel_ip"; fi
    if grep -q '^NINEDEPLOY_ACME_EMAIL=.' .env; then
      upsert_env NINEDEPLOY_PUBLIC_URL "https://$PANEL_DOMAIN_ARG"
    else
      warn "No Let's Encrypt email in .env, so the public URL stays as it is. Unless an email is saved under Settings -> Security, $PANEL_DOMAIN_ARG gets a self-signed certificate: re-run with --email, or save it there and set NINEDEPLOY_PUBLIC_URL=https://$PANEL_DOMAIN_ARG in .env."
    fi
    # Traefik reaches the panel through the Docker gateway, never loopback.
    case "$(sed -n 's/^NINEDEPLOY_BIND=//p' .env | tail -1)" in
      "" | 127.0.0.1 | localhost)
        warn "The panel port is bound to 127.0.0.1, which Traefik cannot reach: $PANEL_DOMAIN_ARG answers only once NINEDEPLOY_BIND=0.0.0.0 is set in .env." ;;
    esac
    # Traefik targets host port 3000 (the port inside the container); a panel
    # published elsewhere is not behind the domain (proven in dind, A2).
    if [ "$panel_port" != "3000" ]; then
      warn "The panel is published on port $panel_port, but Traefik forwards $PANEL_DOMAIN_ARG to port 3000: the domain answers only with NINEDEPLOY_PORT=3000."
    fi
  fi
  # The default panel URL follows the bind: http://<server-ip>:port when the
  # port is published on 0.0.0.0 (0.16 fresh installs), localhost:port for
  # the loopback bind older installs keep. Once the operator attaches a
  # domain and an ACME email, Traefik serves HTTPS on :443 and
  # NINEDEPLOY_PUBLIC_URL should be set to that origin.
  local panel_bind
  panel_bind="$(sed -n 's/^NINEDEPLOY_BIND=//p' .env | tail -1)"
  grep -q '^NINEDEPLOY_PUBLIC_URL=' .env \
    || upsert_env NINEDEPLOY_PUBLIC_URL "$(panel_default_url "${panel_bind:-127.0.0.1}" "$panel_port")"
  chmod 600 .env
  umask "$old_umask"

  info "Pulling the NineDeploy panel image (a few hundred MB on first run)…"
  # Preflight: try an unauthenticated `docker manifest inspect` against
  # the exact tag the compose file will use. A 401/403 here means the
  # GHCR package is still private and the operator must flip its
  # visibility to "Public" at
  # https://github.com/orgs/NineDeploy/packages/container/ninedeploy/settings
  # — docs/QUICKSTART.md walks the first deploy. Failing
  # fast here is friendlier than the generic "image pull failed" the
  # compose call would otherwise surface ten seconds later.
  # r570: put the previous compose file + .env back (kept in
  # $DOCKER_ROLLBACK_DIR above). Used when this run fails BEFORE the panel was
  # replaced, so a later manual `docker compose up -d` (or the operator's
  # re-run) starts from the release that is actually running, not from a
  # half-applied upgrade. Fresh installs have nothing to put back.
  docker_restore_previous_files() {
    [ -f "$DOCKER_ROLLBACK_DIR/docker-compose.yml" ] || return 1
    cp -p "$DOCKER_ROLLBACK_DIR/docker-compose.yml" "$DOCKER_INSTALL_DIR/docker-compose.yml" || return 1
    if [ -f "$DOCKER_ROLLBACK_DIR/.env" ]; then
      # cat >, not cp: keeps the live file's inode and 0600 mode.
      cat "$DOCKER_ROLLBACK_DIR/.env" > "$DOCKER_INSTALL_DIR/.env" || return 1
    fi
    return 0
  }
  if ! docker_cmd manifest inspect "ghcr.io/${IMAGE_REPO}:${IMAGE_TAG}" >/dev/null 2>&1; then
    docker_restore_previous_files || true
    fail "ghcr.io/${IMAGE_REPO}:${IMAGE_TAG} is not publicly pullable. Make the GHCR package 'Public' at https://github.com/orgs/NineDeploy/packages/container/ninedeploy/settings (one-time setup), or use the bare-metal installer (./install.sh without --docker) which builds from source."
  fi
  if ! docker_cmd compose pull; then
    docker_restore_previous_files || true
    fail "Image pull failed — check registry connectivity and re-run the installer. The running panel was not touched."
  fi
  # r447: docker-mode upgrades never had the bare-metal r087 treatment — a
  # failed `compose up -d` (recreate) leaves the panel dark. compose v2 has no
  # auto-rollback.
  # r570: and r447's fallback (`docker start <old id>`) could not work for the
  # common failure: compose v2 recreates by CREATING the new container under a
  # temporary name, then STOPPING and REMOVING the old one, then renaming and
  # starting the new one — a failure at the start step (port conflict, runtime
  # error) finds the old container already deleted. So remember the image the
  # old container RAN (its immutable image id — the pull above may already have
  # re-pointed its tag) and, when the old container is gone, recreate it from
  # the previous compose file + .env with that image re-tagged under the
  # reference the previous compose file names. A failed `up` never ran the new
  # release's code, so no migration touched the data volume: the previous
  # release starts on the database it left.
  local _old_cid _old_image
  _old_cid="$(docker_cmd compose ps -q ninedeploy 2>/dev/null || true)"
  _old_image=""
  [ -n "$_old_cid" ] && _old_image="$(docker_cmd inspect --format '{{.Image}}' "$_old_cid" 2>/dev/null || true)"
  docker_rollback_panel() {
    local _prev_ref
    [ -n "$_old_cid" ] || return 1
    if docker_cmd ps -q --no-trunc | grep -q "$_old_cid"; then
      # Failed before the old container was touched (create step) — it still serves.
      docker_restore_previous_files || true
      info "compose up failed before the previous panel container was replaced — it is still running."
      return 0
    fi
    if docker_cmd start "$_old_cid" >/dev/null 2>&1; then
      docker_restore_previous_files || true
      info "compose up failed — the previous panel container was restarted."
      return 0
    fi
    [ -n "$_old_image" ] || return 1
    docker_restore_previous_files || return 1
    _prev_ref="$(docker_cmd compose config --images 2>/dev/null | head -1)"
    [ -n "$_prev_ref" ] || return 1
    info "compose up failed and the previous panel container is gone — recreating it from ${_prev_ref} (image ${_old_image})…"
    docker_cmd tag "$_old_image" "$_prev_ref" || return 1
    docker_cmd compose up -d || return 1
    return 0
  }
  if ! docker_cmd compose up -d; then
    docker_cmd compose logs --tail 50 2>/dev/null || true
    if docker_rollback_panel; then
      fail "docker compose up failed — the previous panel release is running again. Fix the cause above and re-run the installer."
    fi
    fail "docker compose up failed and the previous panel could not be restored automatically. The previous compose file and .env are in $DOCKER_ROLLBACK_DIR."
  fi

  HEALTH_PORT="$(sed -n 's/^NINEDEPLOY_PORT=//p' .env | tail -1)"
  HEALTH_PORT="${HEALTH_PORT:-3000}"
  # r458: probe the address the panel is actually bound to — a specific
  # non-loopback NINEDEPLOY_BIND made the hardcoded 127.0.0.1 probe (and the
  # whole install) false-fail. 0.0.0.0 and empty both mean loopback works.
  HEALTH_HOST="$(sed -n 's/^NINEDEPLOY_BIND=//p' .env | tail -1)"
  case "$HEALTH_HOST" in ""|0.0.0.0|127.0.0.1|localhost) HEALTH_HOST="127.0.0.1" ;; esac
  # r570: the container runs the database migrations at boot, BEFORE /health
  # answers — on a large database that alone can outlast the old fixed 120 s,
  # and the run then reported a healthy upgrade as failed. Same knob as the
  # bare-metal gate; a longer default here because migrations sit inside it.
  DOCKER_HEALTH_TIMEOUT="${NINEDEPLOY_HEALTH_TIMEOUT:-300}"
  info "Waiting for the panel to become healthy (up to ${DOCKER_HEALTH_TIMEOUT}s; first boot after an upgrade runs migrations)…"
  _healthy=false
  for _i in $(seq 1 "$DOCKER_HEALTH_TIMEOUT"); do
    if curl -fsS -m 2 "http://${HEALTH_HOST}:${HEALTH_PORT}/health" >/dev/null 2>&1; then _healthy=true; break; fi
    sleep 1
  done
  if [ "$_healthy" != "true" ]; then
    docker_cmd compose logs --tail 50 2>/dev/null || true
    # r570: no automatic rollback here — the new container ran, so its boot
    # migrations may already have moved the data volume's schema forward.
    fail "Panel did not become healthy in ${DOCKER_HEALTH_TIMEOUT}s (raise with NINEDEPLOY_HEALTH_TIMEOUT=<seconds>) — inspect: cd $DOCKER_INSTALL_DIR && docker compose logs -f (the previous compose file and .env are kept in $DOCKER_ROLLBACK_DIR)"
  fi
  ok "NineDeploy panel is healthy (docker compose project 'ninedeploy')"
  # r570: the new release is up — the previous compose file/.env copy is done.
  rm -rf "${DOCKER_ROLLBACK_DIR:?}"

  PUBLIC_URL="$(sed -n 's/^NINEDEPLOY_PUBLIC_URL=//p' .env | tail -1)"
  echo ""
  echo -e "${BOLD}╔══════════════════════════════════════════╗${NC}"
  echo -e "${BOLD}║       ✓ Installation Complete            ║${NC}"
  echo -e "${BOLD}╚══════════════════════════════════════════╝${NC}"
  echo ""
  echo -e "  ${CYAN}Dashboard:${NC}  ${PUBLIC_URL:-http://localhost:${HEALTH_PORT}}"
  print_setup_token_hint "http://${HEALTH_HOST}:${HEALTH_PORT}" "${PUBLIC_URL:-http://localhost:${HEALTH_PORT}}" \
    "$(sed -n 's/^NINEDEPLOY_SETUP_TOKEN=//p' .env | tail -1)" \
    "$(panel_default_url "$(sed -n 's/^NINEDEPLOY_BIND=//p' .env | tail -1)" "$HEALTH_PORT")"
  # A2: say whether the panel domain points here yet.
  local panel_domain
  panel_domain="$(sed -n 's/^NINEDEPLOY_DOMAIN=//p' .env | tail -1)"
  if [ -n "$panel_domain" ]; then warn_domain_dns "$panel_domain"; fi
  case "$(sed -n 's/^NINEDEPLOY_BIND=//p' .env | tail -1)" in
    ""|127.0.0.1|localhost)
      echo -e "  ${CYAN}Access:${NC}     the panel port is bound to 127.0.0.1 — use an SSH tunnel (ssh -L ${HEALTH_PORT}:127.0.0.1:${HEALTH_PORT} <server>)"
      echo -e "              or set NINEDEPLOY_BIND=0.0.0.0 in $DOCKER_INSTALL_DIR/.env and run: docker compose up -d" ;;
    *) warn_if_firewalled "$HEALTH_PORT" ;;
  esac
  echo -e "  ${CYAN}Mode:${NC}       Docker container (compose project 'ninedeploy', restart: unless-stopped)"
  echo -e "  ${CYAN}Directory:${NC}  $DOCKER_INSTALL_DIR (secrets in .env, mode 0600)"
  echo ""
  echo -e "  ${YELLOW}Manage:${NC}"
  echo -e "    cd $DOCKER_INSTALL_DIR && docker compose logs -f"
  echo -e "    docker compose restart"
  echo -e "    docker compose down        # data volume survives"
  # r571: the image is pinned to a release tag in .env now — a bare
  # `compose pull` re-pulls the SAME release, so say how to move it.
  echo -e "  ${YELLOW}Upgrade:${NC} re-run this installer (it auto-detects the Docker install), or set"
  echo -e "    NINEDEPLOY_IMAGE_TAG=vX.Y.Z in $DOCKER_INSTALL_DIR/.env, then: docker compose pull && docker compose up -d"
  echo ""
}

if [ "$INSTALL_MODE" = "docker" ]; then
  install_docker_mode
  exit 0
fi

# ── 3. Get the code ────────────────────────────────────────────────────────
#
# Preferred source: the tarball GitHub publishes for a release tag. It is the
# exact tree the tag points at, needs no git on the host, and cannot land on a
# half-fetched object the way a shallow clone can. `git` stays available for
# the edge channel and as the fallback when the archive endpoint is
# unreachable or the tag has no tarball yet.
# r702: for a release that publishes checksums the "tarball" is the release's
# own checksummed archive (see fetch_release_tarball); a checksum or signature
# mismatch stops the installer there — it is never answered with git. The git
# fallback (an unreachable asset host, a clone-based install, --channel main)
# is verified by TLS to github.com only, as before.

# Replace the tracked source tree in $INSTALL_DIR with the release tarball for
# $1, preserving .env and .data. Returns non-zero when the tarball could not
# be fetched, leaving the existing install untouched. (r357: the replaced
# tree, node_modules included, is renamed into .upgrade-rollback-<ts>/ first.)
upgrade_from_release() {
  local _ref="$1" _stage
  _stage=$(mktemp -d) || return 1
  rm -rf "$_stage"

  info "Downloading the $_ref release tarball from GitHub…"
  if ! fetch_release_tarball "$_ref" "$_stage"; then
    rm -rf "$_stage"
    return 1
  fi

  # r357: set the running release aside (same-filesystem renames, no copy)
  # before anything below replaces it, so a failure between here and the
  # section-5 restart can put it back and start the panel on it — see
  # prepare_upgrade_rollback / rollback_failed_upgrade. Not being able to is
  # a warning, not a failure: the upgrade proceeds as it did before r357.
  if prepare_upgrade_rollback "$_stage"; then
    ok "Previous release set aside for rollback in $ROLLBACK_DIR"
    BACKUP_HINT="${BACKUP_HINT:-No pre-update backup was taken}. The previous release's tree is kept in $ROLLBACK_DIR"
  else
    warn "Could not set the previous release aside — continuing WITHOUT a code rollback point: if this upgrade fails, the panel stays down until the installer is re-run."
  fi

  # Drop source directories that the new tree owns wholesale, so a file
  # deleted in this release does not survive as a stale module. Everything
  # holding state — .env, .data, node_modules — is outside this list.
  # (r357: a no-op when prepare_upgrade_rollback already moved them.)
  local _d
  for _d in apps packages scripts systemd patches docs .github; do
    rm -rf "${INSTALL_DIR:?}/$_d"
  done

  # Copy the new tree over the install dir (dotfiles included) — but not
  # install.sh: this script is executing from that exact path, and bash reads
  # it incrementally. Overwriting it in place would make the running shell
  # resume at a byte offset in different content.
  (cd "$_stage" && tar -cf - --exclude=./install.sh .) | (cd "$INSTALL_DIR" && tar -xf -) || {
    rm -rf "$_stage"
    fail "Could not unpack $_ref over $INSTALL_DIR"
  }

  # Swap the installer itself by rename: the running process keeps its open
  # descriptor on the old inode and finishes reading the script it started.
  if [ -f "$_stage/install.sh" ]; then
    if cp "$_stage/install.sh" "$INSTALL_DIR/.install.sh.new"; then
      chmod +x "$INSTALL_DIR/.install.sh.new" 2>/dev/null || true
      mv -f "$INSTALL_DIR/.install.sh.new" "$INSTALL_DIR/install.sh"         || rm -f "$INSTALL_DIR/.install.sh.new"
    fi
  fi

  rm -rf "$_stage"
  ok "Source tree replaced with release $_ref"
  return 0
}

# Move an existing git checkout onto $1 and hard-reset it to the remote.
update_from_git() {
  local _ref="$1"

  # The remote must be the canonical repository. A fork or a renamed remote
  # left behind by an earlier manual clone would silently pin the upgrade to
  # somebody else's history.
  local _origin
  _origin=$(git remote get-url origin 2>/dev/null || true)
  if [ "$_origin" != "$REPO_URL" ]; then
    if [ -z "$_origin" ]; then
      git remote add origin "$REPO_URL"
    else
      warn "origin was $_origin — repointing it at $REPO_URL"
      git remote set-url origin "$REPO_URL"
    fi
  fi

  # Fresh installs used to be cloned with --depth 1, so the checkout holds
  # exactly one commit and none of the objects a newer tag needs. `git fetch
  # --tags` on such a repository succeeds while fetching nothing usable, and
  # the checkout then lands on the old commit — which is how an "upgraded"
  # host keeps serving the previous release. Deepen first.
  if [ "$(git rev-parse --is-shallow-repository 2>/dev/null || echo false)" = "true" ]; then
    info "Shallow checkout detected — fetching full history so tags resolve…"
    # --force: a moved tag (the `stable` channel tag is moved daily) would
    # otherwise be rejected as "would clobber existing tag" and fail the fetch.
    git fetch --force --unshallow --tags origin 2>/dev/null \
      || git fetch --force --depth=2147483647 --tags origin 2>/dev/null \
      || warn "Could not deepen the checkout; tag resolution may be limited"
  fi

  # --force + --prune-tags so a retagged or deleted release is reflected
  # locally instead of leaving a stale tag pointing at old objects.
  info "Fetching the latest refs from GitHub…"
  git fetch --force --tags --prune origin || fail "could not fetch from $REPO_URL"
  git fetch --force --prune --prune-tags --tags origin >/dev/null 2>&1 || true

  if [ "$FORCE_REFRESH" = "1" ]; then
    warn "--force: discarding local modifications to tracked files"
    git reset --hard HEAD >/dev/null 2>&1 || true
  fi

  # Contract (see --help): only --force discards local modifications. A dirty
  # tree must not be silently clobbered by `checkout --force`, so on the
  # default path detect it and abort with guidance (the panel is restarted in
  # the fail branch below).
  if [ "$FORCE_REFRESH" != "1" ] && { ! git diff --quiet || ! git diff --cached --quiet; }; then
    [ "$HAS_SYSTEMD" = true ] && sudo systemctl start ninedeploy
    fail "local modifications to tracked files would be overwritten; commit or stash them, or re-run with --force"
  fi

  if [ "$FORCE_REFRESH" = "1" ]; then
    git checkout --force "$_ref" || { [ "$HAS_SYSTEMD" = true ] && sudo systemctl start ninedeploy; fail "could not check out $_ref"; }
  else
    git checkout "$_ref" || { [ "$HAS_SYSTEMD" = true ] && sudo systemctl start ninedeploy; fail "could not check out $_ref"; }
  fi

  # Land exactly on the remote's idea of $_ref. `checkout --force` on a branch
  # leaves the old local commit in place; on a tag it can silently keep a
  # stale local tag. Reset to the fetched object either way — but ONLY under
  # --force, which is the flag that owns destroying local state.
  if [ "$FORCE_REFRESH" = "1" ]; then
    if [ "$_ref" = "main" ]; then
      git reset --hard origin/main || warn "could not reset to origin/main; continuing with the checked-out tree"
    else
      git reset --hard "refs/tags/$_ref" || warn "could not reset to $_ref; continuing with the checked-out tree"
    fi
  fi

  ok "Source tree at $_ref ($(git rev-parse --short HEAD 2>/dev/null || echo unknown))"
}

# r262: an upgrade stops the running ninedeploy unit below, and the only
# place that started it again was the normal `systemctl restart` in section
# 5. Every `fail`/set -e exit in between — tarball fetch failure, a registry
# outage in `pnpm install`, a broken build, a SIGTERM to the self-update
# unit — left the panel stopped, and with it the UI that reports the failed
# update. This EXIT trap puts the unit back whenever the upgrade stopped it
# and did not reach that restart. Fresh installs (no unit was running) never
# arm it. `start` on a unit that is already active is a no-op, so a path that
# restarted it itself (update_from_git's fail branches) is not restarted
# twice. The exit status of the failed run is preserved.
SERVICE_STOPPED_BY_UPGRADE=false
restore_service_on_exit() {
  local _rc=$?
  # r357: a failed release-tarball upgrade first puts the previous release
  # back (and its database, when migrations already ran), so the start below
  # has a build to start. Every success path disarms the rollback before it
  # can exit, so an armed rollback here IS a failure — $? is not consulted:
  # after a SIGTERM (the self-update unit being stopped) bash reports 0.
  # Called under `|| true`: errexit is off inside, and nothing it does can cut
  # the service start short.
  if [ "${ROLLBACK_ARMED:-false}" = true ]; then
    ROLLBACK_ARMED=false
    rollback_failed_upgrade "$_rc" || true
  fi
  start_service_after_failed_upgrade "$_rc" || true
  # r357: the failed new tree is deleted only once the panel is back up — a
  # big node_modules takes a while, and systemd's stop timeout is finite.
  if [ -n "${ROLLBACK_DISCARD:-}" ]; then
    rm -rf "${ROLLBACK_DISCARD:?}" 2>/dev/null || true
    ROLLBACK_DISCARD=""
  fi
  return 0
}
start_service_after_failed_upgrade() {
  local _rc="$1"
  [ "${SERVICE_STOPPED_BY_UPGRADE:-false}" = true ] || return 0
  SERVICE_STOPPED_BY_UPGRADE=false
  systemctl is-active --quiet ninedeploy 2>/dev/null && return 0
  # The unit's ExecStart is apps/server/dist/server.js. A release-tarball
  # upgrade replaces apps/ wholesale, so past that point there is no build to
  # start — say so instead of kicking off a Restart=always crash loop.
  if [ ! -f "${INSTALL_DIR:-.}/apps/server/dist/server.js" ]; then
    warn "The upgrade did not complete (exit ${_rc}) and no server build is left to restart the panel on."
    warn "The panel is DOWN. Fix the cause above and re-run the installer: cd ${INSTALL_DIR:-.} && ./install.sh"
    return 0
  fi
  warn "The upgrade did not complete (exit ${_rc}) — starting the ninedeploy service again so the panel is not left down."
  if sudo systemctl start ninedeploy; then
    warn "ninedeploy was started again. Re-run the installer once the cause above is fixed."
  else
    warn "ninedeploy could not be restarted — inspect: journalctl -u ninedeploy -n 50"
  fi
  return 0
}

# r357: r262 restarts the unit after a failed upgrade, but the release-tarball
# path (which is what the panel's self-update takes) replaced apps/ wholesale
# and pnpm then rewrites node_modules — so a failure after the unpack (registry
# outage in `pnpm install`, a broken build, a failed migration, a unit that
# does not render) left no build to start, and the panel stayed DOWN until an
# operator re-ran the installer by hand.
#
# Before the unpack, every top-level entry the new tree will replace — plus
# node_modules, the stamp and the running installer — is RENAMED into
# $INSTALL_DIR/.upgrade-rollback-<ts>/old (same filesystem: no copy, no extra
# disk). A manifest records what was moved (M) and what the new release adds
# (A). On a failed exit before the section-5 restart the EXIT trap moves the
# new tree aside, renames the old one back, and then r262 starts the unit on
# it. A successful, healthy upgrade deletes the rollback dir(s).
#
# Database: the DB snapshot above is taken with the service already stopped,
# and it stays stopped until section 5 — nothing writes to the database in
# between except `pnpm db:migrate`. So:
#   * failure BEFORE the migrate step: the DB is untouched; code-only rollback.
#   * failure AT/AFTER the migrate step: the DB may be on the NEW schema, which
#     the old code must not run against. The pre-upgrade snapshot is restored
#     (losing nothing: no writes happened since it was taken) and only then is
#     the old code put back. When no snapshot covers the database the server
#     actually uses, or it cannot be restored, the code is NOT rolled back —
#     new code on a migrated DB beats old code on it — and r262 decides.
ROLLBACK_DIR=""
ROLLBACK_DISCARD=""
ROLLBACK_ARMED=false
DB_MIGRATION_STARTED=false
DB_BACKUP_COVERS_MIGRATION=false

# Device number of a path (not following symlinks); a rename is only possible
# within one device, and `mv` across devices silently degrades to a copy.
path_device() { stat -c %d "$1" 2>/dev/null || stat -f %d "$1" 2>/dev/null || printf '?%s\n' "$1"; }

# Rename $INSTALL_DIR/$1 into the rollback dir and record it. Returns non-zero
# (leaving the entry where it was) when it cannot be renamed.
rollback_take_entry() {
  local _name="$1" _src
  case "$_name" in
    ''|.|..|.env|.data|.git|install.sh|.upgrade-rollback-*|"$RELEASE_STAMP_FILE") return 0 ;;
  esac
  grep -qxF -e "M $_name" -e "A $_name" "$ROLLBACK_DIR/manifest" 2>/dev/null && return 0
  _src="$INSTALL_DIR/$_name"
  if [ -e "$_src" ] || [ -L "$_src" ]; then
    [ "$(path_device "$_src")" = "$(path_device "$ROLLBACK_DIR")" ] || return 1
    mv "$_src" "$ROLLBACK_DIR/old/$_name" || return 1
    if ! printf 'M %s\n' "$_name" >> "$ROLLBACK_DIR/manifest"; then
      mv "$ROLLBACK_DIR/old/$_name" "$_src"
      return 1
    fi
  else
    # New in this release: removed again on rollback.
    printf 'A %s\n' "$_name" >> "$ROLLBACK_DIR/manifest" || return 1
  fi
  return 0
}

# $1 = the verified, unpacked release tree. On success ROLLBACK_DIR is set,
# ROLLBACK_ARMED=true and the old tree is out of the way (the unpack lands in
# an empty slot). On failure every entry already moved is put back and it
# returns non-zero with the install exactly as it was.
prepare_upgrade_rollback() {
  local _stage="$1" _dir _p
  _dir="$INSTALL_DIR/.upgrade-rollback-$(date +%Y%m%d-%H%M%S)"
  [ -e "$_dir" ] && _dir="$_dir-$$"
  if ! mkdir -p "$_dir/old" 2>/dev/null || ! : > "$_dir/manifest" 2>/dev/null; then
    rm -rf "${_dir:?}" 2>/dev/null || true
    return 1
  fi
  ROLLBACK_DIR="$_dir"
  # Armed before the first rename, so no exit path in between loses a moved
  # entry. The trap is (re)installed here too: a host whose unit was not
  # running never armed r262's.
  ROLLBACK_ARMED=true
  trap restore_service_on_exit EXIT

  # The stamp is rewritten after the unpack, and the installer is swapped by
  # rename — keep the old inode (hard link; copy as a fallback) and stamp.
  if [ -f "$INSTALL_DIR/$RELEASE_STAMP_FILE" ]; then
    cp -p "$INSTALL_DIR/$RELEASE_STAMP_FILE" "$_dir/old/$RELEASE_STAMP_FILE" && printf 'S\n' >> "$_dir/manifest"
  else
    printf 'N\n' >> "$_dir/manifest"
  fi
  if [ -f "$INSTALL_DIR/install.sh" ]; then
    { ln "$INSTALL_DIR/install.sh" "$_dir/old/install.sh" 2>/dev/null \
        || cp -p "$INSTALL_DIR/install.sh" "$_dir/old/install.sh"; } && printf 'I\n' >> "$_dir/manifest"
  fi

  # node_modules is rewritten in place by `pnpm install`, and the list below
  # is what the pre-r357 unpack deleted outright; everything else the new
  # tree carries at its top level is overwritten by the unpack.
  for _p in node_modules apps packages scripts systemd patches docs .github \
            "$_stage"/* "$_stage"/.[!.]* "$_stage"/..?*; do
    case "$_p" in
      "$_stage"/*) [ -e "$_p" ] || [ -L "$_p" ] || continue; _p="${_p##*/}" ;;
    esac
    if ! rollback_take_entry "$_p"; then
      warn "Could not move $INSTALL_DIR/$_p aside (different filesystem or mount point?)"
      if rollback_upgrade_tree; then
        ROLLBACK_ARMED=false
        rm -rf "${ROLLBACK_DISCARD:?}"
        ROLLBACK_DISCARD=""
        return 1
      fi
      fail "Could not put the previous release back after a failed rollback preparation — its pieces are in $_dir"
    fi
  done
  return 0
}

# Undo the swap: new tree -> $ROLLBACK_DIR/failed (then deleted), old tree ->
# its place. Returns non-zero, keeping $ROLLBACK_DIR, if anything did not move.
rollback_upgrade_tree() {
  local _dir="${ROLLBACK_DIR:-}" _kind _name _ok=true
  [ -n "$_dir" ] && [ -f "$_dir/manifest" ] || return 1
  mkdir -p "$_dir/failed" || return 1
  while IFS=' ' read -r _kind _name; do
    case "$_kind" in M|A) ;; *) continue ;; esac
    [ -n "$_name" ] || continue
    if [ -e "$INSTALL_DIR/$_name" ] || [ -L "$INSTALL_DIR/$_name" ]; then
      if [ -e "$_dir/failed/$_name" ] || [ -L "$_dir/failed/$_name" ]; then
        rm -rf "${INSTALL_DIR:?}/${_name:?}" || _ok=false
      else
        mv "$INSTALL_DIR/$_name" "$_dir/failed/$_name" || _ok=false
      fi
    fi
  done < "$_dir/manifest"
  while IFS=' ' read -r _kind _name; do
    [ "$_kind" = M ] && [ -n "$_name" ] || continue
    [ -e "$_dir/old/$_name" ] || [ -L "$_dir/old/$_name" ] || continue
    if [ -e "$INSTALL_DIR/$_name" ] || [ -L "$INSTALL_DIR/$_name" ]; then
      _ok=false
      continue
    fi
    mv "$_dir/old/$_name" "$INSTALL_DIR/$_name" || _ok=false
  done < "$_dir/manifest"
  if grep -qx 'I' "$_dir/manifest" && [ -f "$_dir/old/install.sh" ]; then
    # Still the same inode when the installer was never swapped (the hard
    # link above); `mv` refuses to rename a file onto itself.
    if [ "$_dir/old/install.sh" -ef "$INSTALL_DIR/install.sh" ]; then
      rm -f "$_dir/old/install.sh"
    else
      mv -f "$_dir/old/install.sh" "$INSTALL_DIR/install.sh" || _ok=false
    fi
  fi
  if grep -qx 'S' "$_dir/manifest" && [ -f "$_dir/old/$RELEASE_STAMP_FILE" ]; then
    mv -f "$_dir/old/$RELEASE_STAMP_FILE" "$INSTALL_DIR/$RELEASE_STAMP_FILE" || _ok=false
  elif grep -qx 'N' "$_dir/manifest"; then
    rm -f "$INSTALL_DIR/$RELEASE_STAMP_FILE"
  fi
  [ "$_ok" = true ] || return 1
  # Only the failed new tree (and emptied bookkeeping) is left in $_dir; the
  # caller deletes it — the EXIT trap after the service is back up.
  ROLLBACK_DISCARD="$_dir"
  ROLLBACK_DIR=""
  return 0
}

# Put the pre-update snapshot of the database back. The migrated files are
# moved aside first and returned if the restore cannot complete, so a failure
# here never leaves a database that is neither schema.
restore_pre_upgrade_database() {
  local _archive _tmp _data="$INSTALL_DIR/.data" _f _ok=true
  [ -n "${BACKUP_FILE:-}" ] || return 1
  _archive="$INSTALL_DIR/${BACKUP_FILE#./}"
  [ -f "$_archive" ] || return 1
  _tmp="$_data/.r357-db-restore.$$"
  rm -rf "${_tmp:?}"
  (umask 077 && mkdir -p "$_tmp/migrated") || return 1
  if ! tar -xzf "$_archive" -C "$_tmp" || [ ! -f "$_tmp/.data/ninedeploy.db" ]; then
    rm -rf "${_tmp:?}"
    return 1
  fi
  # A -wal/-shm left by the migrated database must never be replayed over the
  # restored file: move the whole set aside, then install the snapshot's set.
  for _f in ninedeploy.db ninedeploy.db-wal ninedeploy.db-shm ninedeploy.db-journal; do
    if [ -e "$_data/$_f" ]; then
      mv -f "$_data/$_f" "$_tmp/migrated/$_f" || _ok=false
    fi
  done
  if [ "$_ok" = true ]; then
    for _f in ninedeploy.db-wal ninedeploy.db-shm ninedeploy.db; do
      if [ -f "$_tmp/.data/$_f" ]; then
        mv -f "$_tmp/.data/$_f" "$_data/$_f" || _ok=false
      fi
    done
  fi
  if [ "$_ok" != true ]; then
    for _f in ninedeploy.db ninedeploy.db-wal ninedeploy.db-shm ninedeploy.db-journal; do
      rm -f "$_data/$_f"
      if [ -e "$_tmp/migrated/$_f" ]; then mv -f "$_tmp/migrated/$_f" "$_data/$_f"; fi
    done
    warn "Could not restore the database snapshot; the database was left as the migration step left it."
    return 1
  fi
  rm -rf "${_tmp:?}"
  return 0
}

# Called by the EXIT trap after a failed run with the rollback armed.
rollback_failed_upgrade() {
  local _rc="$1"
  [ -n "${ROLLBACK_DIR:-}" ] && [ -f "$ROLLBACK_DIR/manifest" ] || return 0
  warn "The upgrade failed (exit ${_rc}) — rolling back to the previous release (${PREVIOUS_REF:-v${PREVIOUS_VERSION:-?}})…"
  if [ "${DB_MIGRATION_STARTED:-false}" = true ]; then
    if [ "${DB_BACKUP_COVERS_MIGRATION:-false}" != true ]; then
      warn "Migrations already ran and no pre-update snapshot covers the database the panel uses — NOT putting the previous code back on a database it may not understand."
      warn "The previous release's tree is kept in $ROLLBACK_DIR"
      return 1
    fi
    if ! restore_pre_upgrade_database; then
      warn "The previous release's tree is kept in $ROLLBACK_DIR; the pre-update snapshot is ${INSTALL_DIR}/${BACKUP_FILE#./}"
      return 1
    fi
    ok "Database restored from the pre-update snapshot ${BACKUP_FILE}"
  fi
  if rollback_upgrade_tree; then
    ok "Previous release restored in $INSTALL_DIR"
    return 0
  fi
  warn "The rollback did not complete — the previous release's remaining pieces are in $ROLLBACK_DIR"
  return 1
}

# A healthy upgrade: nothing to roll back to any more, including rollback dirs
# a previously failed run kept.
prune_upgrade_rollbacks() {
  local _d
  ROLLBACK_ARMED=false
  for _d in "$INSTALL_DIR"/.upgrade-rollback-*; do
    [ -d "$_d" ] || continue
    rm -rf "${_d:?}" || warn "Could not remove the rollback copy $_d"
  done
  ROLLBACK_DIR=""
  # r460: pre-update snapshots carry master.key and a full DB copy each —
  # without pruning every upgrade grows .data by the size of the database.
  # Keep the newest five; filenames are installer-generated (no spaces).
  if [ -d "$INSTALL_DIR/.data/upgrade-backups" ]; then
    ls -1t "$INSTALL_DIR/.data/upgrade-backups"/pre-update-*.tar.gz 2>/dev/null       | tail -n +6       | while IFS= read -r _b; do rm -f -- "$_b" || true; done
  fi
}


if [ -f "$INSTALL_DIR/package.json" ]; then
  # What is on disk right now, so the upgrade reports both ends of the jump.
  PREVIOUS_VERSION=$(node -p "require('${INSTALL_DIR}/package.json').version" 2>/dev/null || echo "")
  PREVIOUS_REF=$(cat "$INSTALL_DIR/$RELEASE_STAMP_FILE" 2>/dev/null || echo "")
  if [ -n "$PREVIOUS_VERSION" ]; then
    info "Existing installation found at $INSTALL_DIR — upgrading ${PREVIOUS_REF:-v$PREVIOUS_VERSION} → $REF"
  else
    info "Existing installation found at $INSTALL_DIR — updating…"
  fi
  cd "$INSTALL_DIR"

  HAS_SYSTEMD=false
  if [ "$(uname -s)" = "Linux" ] && command -v systemctl &>/dev/null && systemctl is-active --quiet ninedeploy 2>/dev/null; then
    HAS_SYSTEMD=true
    # r262: arm the restart BEFORE stopping, so no exit path in between can
    # leave the panel down (see restore_service_on_exit).
    SERVICE_STOPPED_BY_UPGRADE=true
    trap restore_service_on_exit EXIT
    info "Stopping the service for a consistent backup…"
    sudo systemctl stop ninedeploy
  fi

  # Safety net before an upgrade: snapshot the database and the master key.
  #
  # This used to pass both paths to one `tar` invocation with stderr muted.
  # `master.key` is written lazily — the first time something is encrypted —
  # so on an instance that has no secrets yet the file does not exist, tar
  # exits non-zero, and the whole snapshot was skipped: including the
  # database, which did exist. The operator saw one muted warning and
  # upgraded with no rollback point at all.
  #
  # Archive each member that is actually present, and treat a missing
  # master.key as the normal state it is. SQLite's -wal/-shm sidecars are
  # included when present so the snapshot is a consistent set rather than a
  # main file missing its most recent committed transactions.
  BACKUP_OK=false
  BACKUP_FILE=""
  if [ -d .data ]; then
    BACKUP_MEMBERS=""
    for _f in .data/ninedeploy.db .data/ninedeploy.db-wal .data/ninedeploy.db-shm .data/master.key; do
      [ -f "$_f" ] && BACKUP_MEMBERS="$BACKUP_MEMBERS $_f"
    done

    if [ -z "$BACKUP_MEMBERS" ]; then
      # Nothing to lose — a data directory with no database is a fresh state.
      info "No database or master key in .data yet — nothing to snapshot"
    else
      mkdir -p .data/upgrade-backups
      BACKUP_FILE=".data/upgrade-backups/pre-update-$(date +%Y%m%d-%H%M%S).tar.gz"
      # The archive contains master.key — the key that decrypts EVERY stored
      # secret. Tighten the umask for the tar child so the file lands 0600
      # regardless of the caller's ambient umask, and assert it afterwards.
      # shellcheck disable=SC2086 # deliberate word splitting: one arg per member
      if TAR_ERR=$(umask 077 && tar -czf "$BACKUP_FILE" $BACKUP_MEMBERS 2>&1); then
        chmod 600 "$BACKUP_FILE" 2>/dev/null || true
        BACKUP_OK=true
        ok "Pre-update backup saved to $BACKUP_FILE ($(echo "$BACKUP_MEMBERS" | wc -w) file(s))"
      else
        rm -f "$BACKUP_FILE"
        BACKUP_FILE=""
        warn "Could not snapshot .data — continuing WITHOUT a rollback point:"
        printf '%s\n' "$TAR_ERR" | head -5 | sed 's/^/     /'
      fi
    fi
  fi

  # Used by the readiness gate's failure messages, which must not claim a
  # backup exists when the snapshot above did not run.
  if [ "$BACKUP_OK" = true ]; then
    BACKUP_HINT="A pre-update backup is in ${INSTALL_DIR}/${BACKUP_FILE#./}"
  else
    BACKUP_HINT="No pre-update backup was taken"
  fi

  # ── Upgrade: release tarball first, git second ─────────────────────────
  #
  # A tagged release is the artifact operators are told to run, so it is what
  # the installer fetches. `git` remains the path for the edge channel and
  # for hosts that cannot reach the archive endpoint, and it is the only way
  # to move an existing clone-based install forward.
  if [ "$REF" != "main" ] && upgrade_from_release "$REF"; then
    SOURCE_MODE="release"
  elif [ -d "$INSTALL_DIR/.git" ]; then
    [ "$REF" = "main" ] || warn "Release tarball unavailable for $REF — falling back to git"
    # r450: the git path has NO code-rollback point (only the release path
    # prepares one) — say so BEFORE the tree is swapped, not in the failure
    # message after the panel is already down.
    warn "The git upgrade path has no automatic code rollback: if the build fails after this point, the previous tree is NOT restored — only the pre-update backup (if one was taken) survives."
    SOURCE_MODE="git"
    update_from_git "$REF"
  else
    fail "Could not download the $REF release tarball, and $INSTALL_DIR is not a git checkout. Re-run with --channel main, or check network access to github.com."
  fi
else
  # ── Fresh install ──────────────────────────────────────────────────────
  if [ "$REF" != "main" ]; then
    info "Downloading the $REF release tarball from GitHub…"
    # Unpack into a staging directory rather than straight into $INSTALL_DIR:
    # a failed extraction must never remove whatever the operator already had
    # at that path, and `git clone` (the fallback) refuses a non-empty target.
    FRESH_STAGE=$(mktemp -d) && rm -rf "$FRESH_STAGE"
    if fetch_release_tarball "$REF" "$FRESH_STAGE"; then
      mkdir -p "$INSTALL_DIR"
      if (cd "$FRESH_STAGE" && tar -cf - .) | (cd "$INSTALL_DIR" && tar -xf -); then
        SOURCE_MODE="release"
        ok "Unpacked NineDeploy $REF to $INSTALL_DIR"
      else
        warn "Could not unpack $REF into $INSTALL_DIR — falling back to git clone"
      fi
    else
      warn "Release tarball unavailable for $REF — falling back to git clone"
    fi
    rm -rf "$FRESH_STAGE"
  fi
  if [ -z "$SOURCE_MODE" ]; then
    info "Cloning NineDeploy to $INSTALL_DIR …"
    git clone --depth 1 ${REF:+-b "$REF"} "$REPO_URL" "$INSTALL_DIR"
    SOURCE_MODE="git"
  fi
  cd "$INSTALL_DIR"
fi

# Stamp what is on disk. `git describe` is unavailable in a tarball install,
# so without this an upgrade has no way to report the version it replaced.
printf '%s\n' "$REF" > "$INSTALL_DIR/$RELEASE_STAMP_FILE" 2>/dev/null || true

if [ "$FORCE_REFRESH" = "1" ] && [ "$SOURCE_MODE" = "git" ]; then
  # Untracked leftovers can shadow a moved or renamed source file. .data,
  # .env and the upgrade backups are gitignored on purpose and must survive,
  # so clean only what git tracks as ignored *build* output — never -x.
  # A release-tarball install has no git index; upgrade_from_release already
  # replaced the source directories wholesale.
  git clean -fd -e .data -e .env -e '*.local' >/dev/null 2>&1 || true
fi

# ── 3. Install + build ────────────────────────────────────────────────────
#
# pnpm prints nothing without a TTY — on a cold host `pnpm install` alone runs
# 10-20 silent minutes, which every operator to date has read as a hung
# installer (two documented kills mid-build). Both steps get heartbeats and a
# failure log tail.

info "Installing dependencies… (silent — heartbeats every 30s; often 10-20 min on a cold host)"
# Fail-CLOSED on lockfile drift. The previous behaviour fell back to a bare
# `pnpm install`, which re-resolves the whole dependency graph ON A PRODUCTION
# HOST and immediately runs the lifecycle scripts of whatever versions the
# registry hands out — turning a mismatched lockfile into a supply-chain
# event. The loose path is now an explicit operator decision.
if ! run_quiet_step "pnpm install" pnpm install --frozen-lockfile; then
  if [ "${NINEDEPLOY_ALLOW_LOOSE_INSTALL:-0}" = "1" ]; then
    warn "Frozen install failed — retrying with a regenerated lockfile (NINEDEPLOY_ALLOW_LOOSE_INSTALL=1)."
    run_quiet_step "pnpm install (retry)" pnpm install \
      || fail "pnpm install failed — see the log tail above"
  else
    fail "pnpm install --frozen-lockfile failed: the lockfile does not match package.json (or the registry is unreachable). Fix the checkout, or re-run with NINEDEPLOY_ALLOW_LOOSE_INSTALL=1 to explicitly accept a re-resolved dependency graph on this host."
  fi
fi

# A build artifact from the previous release is not overwritten by a checkout
# — dist/ is gitignored, so an upgrade that rebuilds nothing keeps serving the
# old panel bundle, which is exactly the "upgraded but the UI still shows the
# previous version" failure. Drop the artifacts and the turbo cache so the
# build below cannot be skipped.
#
# r262: only AFTER dependencies installed. Clearing dist/ first meant a
# registry outage in `pnpm install` left no build for the service to be
# restarted on; nothing in the install step reads these directories.
info "Clearing build artifacts from the previous release…"
rm -rf .turbo apps/*/.turbo packages/*/.turbo 2>/dev/null || true
rm -rf apps/*/dist packages/*/dist 2>/dev/null || true
rm -rf node_modules/.cache node_modules/.vite 2>/dev/null || true

info "Building the panel, API and CLI… (silent — often 5-10 min)"
run_quiet_step "pnpm build" pnpm build \
  || fail "pnpm build failed — see the log tail above"

# Prove the tree that was just built is the one that was requested. A silent
# mismatch here is the difference between "upgraded" and "still on the old
# release but nobody noticed".
BUILT_VERSION=$(node -p "require('${INSTALL_DIR}/package.json').version" 2>/dev/null || echo "unknown")
if [ "$REF" = "main" ]; then
  ok "Built NineDeploy ${BUILT_VERSION} from main ($(git rev-parse --short HEAD 2>/dev/null || echo unknown))"
elif [ "v${BUILT_VERSION}" = "$REF" ]; then
  ok "Built NineDeploy ${BUILT_VERSION} (matches $REF)"
else
  warn "Checked out $REF but package.json reports ${BUILT_VERSION} — the tag and the manifest disagree."
fi
if [ ! -f "${INSTALL_DIR}/apps/web/dist/index.html" ]; then
  fail "The web panel did not build (apps/web/dist/index.html is missing). Re-run with --force."
fi

# Link global CLI executable
if [ -f "$INSTALL_DIR/apps/cli/dist/index.js" ]; then
  chmod +x "$INSTALL_DIR/apps/cli/dist/index.js" 2>/dev/null || true
  sudo ln -sf "$INSTALL_DIR/apps/cli/dist/index.js" /usr/local/bin/ninedeploy 2>/dev/null || true
  ok "CLI symlinked to /usr/local/bin/ninedeploy"
fi

# ── 4. Configure ──────────────────────────────────────────────────────────

# A2: a first install asks for the panel domain (see prompt_panel_domain).
if [ ! -f ".env" ]; then prompt_panel_domain; fi

if [ ! -f ".env" ]; then
  info "Creating .env from template…"
  # Tighten the umask BEFORE the file exists: `cp` then `chmod 600` at the end
  # leaves the generated JWT secret and master key world-readable (default
  # umask 022) for the duration of the sed passes below. `sed -i` also creates
  # its temp file under the same umask.
  _nd_old_umask=$(umask)
  umask 077
  cp .env.example .env

  # Set production mode
  sed -i.bak "s|^NODE_ENV=.*|NODE_ENV=production|" .env && rm -f .env.bak

  # Generate strong 32-byte hex secrets (64 chars)
  JWT_SECRET=$(openssl rand -hex 32 2>/dev/null || node -e "console.log(require('crypto').randomBytes(32).toString('hex'))")
  sed -i.bak "s|^NINEDEPLOY_JWT_SECRET=.*|NINEDEPLOY_JWT_SECRET=${JWT_SECRET}|" .env && rm -f .env.bak

  MASTER_KEY=$(openssl rand -hex 32 2>/dev/null || node -e "console.log(require('crypto').randomBytes(32).toString('hex'))")
  sed -i.bak "s|^NINEDEPLOY_MASTER_KEY=.*|NINEDEPLOY_MASTER_KEY=${MASTER_KEY}|" .env && rm -f .env.bak

  # 0.16: the panel listens on 0.0.0.0 (the template default) unless
  # --private; the public URL is the detected server address, and the first
  # admin needs the one-time setup token printed at the end of this run.
  _nd_host="${NINEDEPLOY_HOST:-0.0.0.0}"
  [ "$PANEL_PRIVATE" = "1" ] && _nd_host="127.0.0.1"
  sed -i.bak "s|^NINEDEPLOY_HOST=.*|NINEDEPLOY_HOST=${_nd_host}|" .env && rm -f .env.bak
  _nd_port="${NINEDEPLOY_PORT:-3000}"
  sed -i.bak "s|^NINEDEPLOY_PORT=.*|NINEDEPLOY_PORT=${_nd_port}|" .env && rm -f .env.bak
  _nd_url="${NINEDEPLOY_PUBLIC_URL:-$(panel_default_url "$_nd_host" "$_nd_port")}"
  sed -i.bak "s|^NINEDEPLOY_PUBLIC_URL=.*|NINEDEPLOY_PUBLIC_URL=${_nd_url}|" .env && rm -f .env.bak
  printf 'NINEDEPLOY_SETUP_TOKEN=%s\n' "${NINEDEPLOY_SETUP_TOKEN:-$(gen_setup_token)}" >> .env

  chmod 600 .env
  umask "$_nd_old_umask"

  ok ".env created with generated production secrets"
else
  chmod 600 .env
  ok ".env already exists"
fi
# A6: `--channel stable` also makes the panel offer stable releases. A choice
# saved under Settings → System wins over it; never written for a pinned run
# (the panel's self-update pins its target and must not change the channel).
if [ "$CHANNEL" = "stable" ] && [ -z "$PINNED_VERSION" ]; then
  if grep -q '^NINEDEPLOY_UPDATE_CHANNEL=' .env; then
    sed -i.bak 's|^NINEDEPLOY_UPDATE_CHANNEL=.*|NINEDEPLOY_UPDATE_CHANNEL=stable|' .env && rm -f .env.bak
  else
    printf 'NINEDEPLOY_UPDATE_CHANNEL=stable\n' >> .env
  fi
fi
# A2: the panel domain and its certificate email (helpers near the top). The
# public URL follows below, once the email prompt has had its turn.
if [ -n "$ACME_EMAIL_ARG" ]; then env_file_set NINEDEPLOY_ACME_EMAIL "$ACME_EMAIL_ARG"; fi
if [ -n "$PANEL_DOMAIN_ARG" ]; then
  env_file_set NINEDEPLOY_DOMAIN "$PANEL_DOMAIN_ARG"
  # Fresh installs still seed sslip.io service URLs from this address.
  _nd_panel_ip="$(detect_server_ip)"
  if [ -n "$_nd_panel_ip" ]; then env_file_set NINEDEPLOY_PUBLIC_IP "$_nd_panel_ip"; fi
fi

# r451: the data dir holds ninedeploy.db (credential hashes, sessions), the
# repos/ checkouts (committed secrets included) and PM2's dump (managed-app
# env) — the installer is meticulous about .env and backups; .data must not be
# the one world-readable thing. 0750: the service runs as root, so owner rwx
# suffices and no other local account can traverse in. Mode is only applied
# on CREATE (chmod would fight operator tweaks on re-runs).
if [ ! -d .data ]; then
  install -d -m 0750 .data
else
  mkdir -p .data
fi

info "Running database migrations…"
# Export .env first: drizzle.config reads NINEDEPLOY_DB_PATH/NINEDEPLOY_DATA_DIR
# from the environment, so without this the migrate step could target a
# different database file than the one the service actually runs on.
set -a
# shellcheck disable=SC1091
. ./.env
set +a
# A2: not again when the domain prompt just asked for it.
if [ -z "${NINEDEPLOY_ACME_EMAIL:-}" ] && [ "${ACME_PROMPTED:-0}" != "1" ]; then
  ACME_INPUT=""
  # r261: the old `-r`/`-w` test on /dev/tty only read the device node's
  # permission bits (crw-rw-rw-, true for everyone). With NO controlling
  # terminal — the panel's self-update runs this script inside a systemd-run
  # unit — opening /dev/tty fails with ENXIO, the prompt's redirection failed,
  # and `set -e` aborted the upgrade here: after the panel was stopped and
  # before migrate/restart. Prompt only when the terminal can actually be
  # opened, never during a panel-driven self-update, and never let a failing
  # prompt abort the run.
  if [ -z "${ND_SELF_UPDATE_TARGET:-}" ] && { : <>/dev/tty; } 2>/dev/null; then
    { printf "Let's Encrypt account email (required for automatic HTTPS; Enter to configure later): " >/dev/tty; } 2>/dev/null || true
    IFS= read -r ACME_INPUT </dev/tty 2>/dev/null || ACME_INPUT=""
  fi
  if [ -n "$ACME_INPUT" ] && printf '%s' "$ACME_INPUT" | grep -Eq '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'; then
    sed -i.bak "s|^NINEDEPLOY_ACME_EMAIL=.*|NINEDEPLOY_ACME_EMAIL=${ACME_INPUT}|" .env && rm -f .env.bak
    export NINEDEPLOY_ACME_EMAIL="$ACME_INPUT"
    ok "Automatic HTTPS configured for $ACME_INPUT"
  else
    [ -z "$ACME_INPUT" ] || warn "The supplied ACME email is invalid and was not saved."
    warn "Automatic HTTPS is not active yet: set the Let's Encrypt account email in Settings -> Security after signing in. NineDeploy will apply it to Traefik immediately."
  fi
fi
# A2: with a certificate email, the panel domain becomes the public URL (links
# in emails, OAuth callbacks). Traefik reaches the panel through the Docker
# gateway, so a loopback-only panel cannot serve it.
if [ -n "$PANEL_DOMAIN_ARG" ]; then
  if [ -n "${NINEDEPLOY_ACME_EMAIL:-}" ]; then
    env_file_set NINEDEPLOY_PUBLIC_URL "https://$PANEL_DOMAIN_ARG"
    export NINEDEPLOY_PUBLIC_URL="https://$PANEL_DOMAIN_ARG"
  else
    warn "No Let's Encrypt email in .env, so the public URL stays as it is. Unless an email is saved under Settings -> Security, $PANEL_DOMAIN_ARG gets a self-signed certificate: re-run with --email, or save it there and set NINEDEPLOY_PUBLIC_URL=https://$PANEL_DOMAIN_ARG in .env."
  fi
  case "${NINEDEPLOY_HOST:-0.0.0.0}" in
    127.0.0.1 | localhost)
      warn "The panel listens on 127.0.0.1, which Traefik cannot reach: $PANEL_DOMAIN_ARG answers only once NINEDEPLOY_HOST=0.0.0.0 is set in .env." ;;
  esac
fi
# r357: from here on the database may be on the new schema, so a rollback to
# the previous release must restore the pre-update snapshot first — which is
# only possible when that snapshot (.data/ninedeploy.db) IS the database the
# panel runs on (NINEDEPLOY_DB_PATH resolves against the install root, as in
# apps/server/src/config.ts). Anything else: no code rollback past this line.
_r357_db_path="${NINEDEPLOY_DB_PATH:-./.data/ninedeploy.db}"
_r357_db_path="${_r357_db_path#file:}"
case "$_r357_db_path" in
  /*) ;;
  *) _r357_db_path="$INSTALL_DIR/${_r357_db_path#./}" ;;
esac
if [ "${BACKUP_OK:-false}" = true ] && [ "$_r357_db_path" = "$INSTALL_DIR/.data/ninedeploy.db" ]; then
  DB_BACKUP_COVERS_MIGRATION=true
fi
DB_MIGRATION_STARTED=true
pnpm db:migrate

# ── 5. Systemd (Linux) ────────────────────────────────────────────────────

if [ "$(uname -s)" = "Linux" ] && command -v systemctl &>/dev/null; then
  info "Setting up systemd service…"
  # Render the CHECKED-IN hardened unit (systemd/ninedeploy.service) with this
  # install's paths so the installer and the repo unit never drift apart.
  # Placeholders: @NODE@, @INSTALL_DIR@, @DATA_DIR@.
  UNIT_TEMPLATE="$INSTALL_DIR/systemd/ninedeploy.service"
  if [ ! -f "$UNIT_TEMPLATE" ]; then
    fail "systemd/ninedeploy.service not found in the repo — re-run ./install.sh"
  fi
  # Absolute data dir (resolve_data_dir, near the top — CI runs it under the
  # script's own set -u; r087 was an unbound DATA_DIR right here).
  DATA_DIR=$(resolve_data_dir "$INSTALL_DIR")
  # Writable homes for the toolchains the panel spawns: DOCKER_CONFIG (buildx
  # builder-activity writes) and PM2_HOME (dump.pm2 persistence). Both default
  # to /root/... which the hardened unit's ProtectHome=read-only makes
  # unwritable — the unit exports these env vars pointing here.
  mkdir -p "$DATA_DIR/docker-config" "$DATA_DIR/pm2"
  SERVICE_FILE="/etc/systemd/system/ninedeploy.service"
  UNIT_STAGE_DIR=$(mktemp -d)
  UNIT_STAGE_FILE="$UNIT_STAGE_DIR/ninedeploy.service"
  sed -e "s|@NODE@|$(which node)|g" \
      -e "s|@INSTALL_DIR@|${INSTALL_DIR}|g" \
      -e "s|@DATA_DIR@|${DATA_DIR}|g" \
      -e "s|@USER@|root|g" \
      -e "s|@GROUP@|root|g" \
      "$UNIT_TEMPLATE" > "$UNIT_STAGE_FILE"

  # Validate before replacing the live unit. This catches malformed rendered
  # paths without leaving an existing installation unbootable.
  if command -v systemd-analyze &>/dev/null; then
    systemd-analyze verify "$UNIT_STAGE_FILE" >/dev/null || {
      rm -f "$UNIT_STAGE_FILE"
      rmdir "$UNIT_STAGE_DIR"
      fail "Rendered systemd unit is invalid"
    }
  fi
  sudo install -m 0644 "$UNIT_STAGE_FILE" "$SERVICE_FILE"
  rm -f "$UNIT_STAGE_FILE"
  rmdir "$UNIT_STAGE_DIR"

  # Older NineDeploy releases installed Type=notify + WatchdogSec=90. A
  # lingering drop-in can override the new main unit and SIGTERM the server
  # (and its docker pull child) during a long image extraction. Keep this
  # installer-owned, last-applied safety override for both fresh installs and
  # in-place upgrades. User overrides that sort later are detected below.
  RUNTIME_DROPIN_DIR="/etc/systemd/system/ninedeploy.service.d"
  RUNTIME_DROPIN="$RUNTIME_DROPIN_DIR/zzzz-ninedeploy-runtime-safety.conf"
  sudo mkdir -p "$RUNTIME_DROPIN_DIR"
  # v0.2.4 briefly used a numeric prefix which sorts before the conventional
  # override.conf name. Remove only that installer-owned obsolete file; keep
  # every administrator-owned drop-in intact.
  sudo rm -f "$RUNTIME_DROPIN_DIR/99-ninedeploy-runtime-safety.conf"
  printf '%s\n' \
    '# Managed by NineDeploy install.sh — do not enable a service watchdog.' \
    '[Service]' \
    'Type=simple' \
    'WatchdogSec=0' | sudo tee "$RUNTIME_DROPIN" >/dev/null

  sudo systemctl daemon-reload

  EFFECTIVE_TYPE=$(systemctl show ninedeploy --property=Type --value)
  EFFECTIVE_WATCHDOG=$(systemctl show ninedeploy --property=WatchdogUSec --value)
  if [ "$EFFECTIVE_TYPE" != "simple" ]; then
    fail "Unsafe effective systemd Type=$EFFECTIVE_TYPE (expected simple); inspect: systemctl cat ninedeploy"
  fi
  case "$EFFECTIVE_WATCHDOG" in
    # Modern systemd (v240+) reports a DISABLED watchdog as "infinity", not
    # "0" — most visibly on never-started units during fresh installs. Both
    # spellings mean "will never fire": accept them, reject real intervals.
    0|0us|0ms|0s|infinity|infinityus) ;;
    *) fail "Unsafe effective systemd watchdog $EFFECTIVE_WATCHDOG (expected disabled); inspect: systemctl cat ninedeploy" ;;
  esac
  ok "systemd runtime policy verified (Type=simple, watchdog disabled)"

  # The service runs as root. When the code tree lives inside the invoking
  # user's HOME, everything root executes — and the .env it reads — is
  # writable by that user, which is a quiet path from "panel user" to "root
  # code execution" on the next restart. Locking ownership is a deliberate,
  # opt-in step because it also requires future updates to run under sudo.
  case "$INSTALL_DIR" in
    "$HOME"/*)
      if [ "${NINEDEPLOY_HARDEN_OWNERSHIP:-0}" = "1" ]; then
        sudo chown -R root:root "$INSTALL_DIR"
        ok "Install tree handed to root (NINEDEPLOY_HARDEN_OWNERSHIP=1)."
        warn "Future updates must run the installer under sudo: sudo -E ./install.sh"
      else
        warn "The root service runs code from a user-writable tree: $INSTALL_DIR"
        warn "Harden it by re-running with NINEDEPLOY_HARDEN_OWNERSHIP=1 (future updates then need sudo)."
      fi
      ;;
  esac

  sudo systemctl enable ninedeploy
  sudo systemctl restart ninedeploy
  # r262: the unit is running the new build now; a later failure (health
  # gate, Traefik check) is reported as such, not papered over by the trap.
  SERVICE_STOPPED_BY_UPGRADE=false
  # r357: likewise the code rollback — a failure from here on is the new
  # release's to report; the rollback dir is kept for a manual rollback until
  # the health gate below passes.
  ROLLBACK_ARMED=false

  # Everything the operator would otherwise have to go and collect by hand.
  # A readiness failure is the single most common place an install ends, and
  # "inspect journalctl" costs a round trip that the installer can spend for
  # them — it already knows the unit name, the port and the data directory.
  dump_service_diagnostics() {
    echo ""
    warn "── systemd unit ────────────────────────────────────────────────"
    systemctl --no-pager --full status ninedeploy 2>&1 | head -20 || true
    echo ""
    warn "── last 60 log lines (journalctl -u ninedeploy) ─────────────────"
    sudo journalctl -u ninedeploy -n 60 --no-pager 2>&1 || true
    echo ""
    warn "── port ${HEALTH_PORT} ─────────────────────────────────────────────────"
    (ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null || true) | grep -E "[:.]${HEALTH_PORT}[[:space:]]" || echo "  nothing is listening on ${HEALTH_PORT}"
    echo ""
  }

  # Readiness gate: give the API time to answer /health before declaring
  # success — a failed migration or boot crash surfaces here, not later.
  #
  # The window is generous because the first boot after an upgrade also runs
  # migrations, but a process that is *not running* is never going to answer,
  # so a crash-loop is called immediately rather than after the full timeout.
  HEALTH_PORT="${NINEDEPLOY_PORT:-3000}"
  # r570: probe the address the panel actually binds (the bare-metal twin of
  # r458). An operator who bound NINEDEPLOY_HOST to one specific interface
  # (a private IP) had every upgrade fail this gate on 127.0.0.1 — AFTER the
  # new release was already running, so the self-update reported a healthy
  # upgrade as failed. Wildcard and loopback binds still probe 127.0.0.1.
  PANEL_PROBE_HOST="${NINEDEPLOY_HOST:-0.0.0.0}"
  case "$PANEL_PROBE_HOST" in
    ""|0.0.0.0|::|"[::]"|127.0.0.1|localhost) PANEL_PROBE_HOST="127.0.0.1" ;;
    *:*) PANEL_PROBE_HOST="[${PANEL_PROBE_HOST#[}"; PANEL_PROBE_HOST="${PANEL_PROBE_HOST%]}]" ;;
  esac
  HEALTH_TIMEOUT="${NINEDEPLOY_HEALTH_TIMEOUT:-120}"
  info "Waiting for the API to come up (up to ${HEALTH_TIMEOUT}s)…"
  if command -v curl &>/dev/null; then
    _health_ok=false
    _crash_strikes=0
    for i in $(seq 1 "$HEALTH_TIMEOUT"); do
      if curl -fsS -m 2 "http://${PANEL_PROBE_HOST}:${HEALTH_PORT}/health" >/dev/null 2>&1; then
        _health_ok=true
        break
      fi

      # `Restart=always` means a unit that dies is restarted, so a crashing
      # server oscillates between active and activating instead of settling
      # into `failed`. Count consecutive non-active samples: three in a row
      # is a boot that is not going to succeed, not a slow start.
      case "$(systemctl is-active ninedeploy 2>/dev/null || echo unknown)" in
        active)
          _crash_strikes=0
          ;;
        failed)
          warn "ninedeploy entered the failed state while starting"
          dump_service_diagnostics
          fail "Service failed to start. ${BACKUP_HINT:-No pre-update backup was taken}"
          ;;
        *)
          _crash_strikes=$((_crash_strikes + 1))
          if [ "$_crash_strikes" -ge 3 ]; then
            warn "ninedeploy is restarting repeatedly — it is crashing on boot"
            dump_service_diagnostics
            fail "Service is crash-looping. ${BACKUP_HINT:-No pre-update backup was taken}"
          fi
          ;;
      esac

      sleep 1
    done

    if [ "$_health_ok" = true ]; then
      ok "NineDeploy service is healthy (systemd, hardened unit)"
    else
      warn "The process is running but /health did not answer within ${HEALTH_TIMEOUT}s"
      dump_service_diagnostics
      fail "Service did not become healthy in ${HEALTH_TIMEOUT}s. Raise the window with NINEDEPLOY_HEALTH_TIMEOUT=<seconds> if this host is just slow. ${BACKUP_HINT:-No pre-update backup was taken}"
    fi
  else
    ok "NineDeploy service started (systemd, hardened unit)"
  fi
  # r357: the new release is up — drop the previous one's rollback copy.
  prune_upgrade_rollbacks

  # /health is only a valid installation gate when the mandatory ingress
  # container is also live on the shared network. Never print a successful
  # installation while domain routing is unavailable.
  TRAEFIK_RUNNING=$(docker_cmd inspect ninedeploy-traefik --format '{{.State.Running}}' 2>/dev/null || true)
  TRAEFIK_NETWORKS=$(docker_cmd inspect ninedeploy-traefik --format '{{json .NetworkSettings.Networks}}' 2>/dev/null || true)
  if [ "$TRAEFIK_RUNNING" != "true" ] || ! printf '%s' "$TRAEFIK_NETWORKS" | grep -q '"ninedeploy"'; then
    docker_cmd logs --tail 50 ninedeploy-traefik 2>&1 || true
    fail "Traefik failed its post-install runtime check; NineDeploy installation is not healthy"
  fi
  TRAEFIK_HTTP_STATUS=$(curl -sS -o /dev/null -m 5 -w '%{http_code}' -H 'Host: ninedeploy-install-check.invalid' http://127.0.0.1/ 2>/dev/null || true)
  case "$TRAEFIK_HTTP_STATUS" in
    2??|3??|4??) ;;
    *) docker_cmd logs --tail 50 ninedeploy-traefik 2>&1 || true; fail "Traefik is running but its HTTP entrypoint on :80 is not responding" ;;
  esac
  ok "Traefik ingress verified (network attached, :80 responding)"

  # ── Unit environment self-check ───────────────────────────────────────────
  # The hardened unit MUST export the toolchain homes: DOCKER_CONFIG (buildx
  # builder-activity writes) and PM2_HOME (dump.pm2 persistence). Both default
  # to /root/... and ProtectHome=read-only makes that unwritable — a unit
  # missing either variable re-creates the "read-only file system" build
  # failures and the silent PM2 persistence loss. A stale rendered unit or an
  # admin drop-in that omits them is caught HERE, on every install/upgrade,
  # instead of by the operator's first deploy.
  UNIT_ENV=$(systemctl show ninedeploy --property=Environment --value)
  for _req_var in DOCKER_CONFIG PM2_HOME; do
    if ! printf '%s' "$UNIT_ENV" | grep -q "$_req_var="; then
      warn "ninedeploy unit does not export $_req_var — docker builds (and PM2 persistence) will fail with 'read-only file system'."
      warn "Re-run this installer to re-render the unit, or add it via: sudo systemctl edit ninedeploy"
    fi
  done

  # ── Deploy preflight: prove a build actually works in this environment ──
  # The panel spawns `docker build` under the hardened unit (ProtectHome makes
  # /root read-only; buildx writes builder activity under $DOCKER_CONFIG), and
  # the operator's FIRST deploy is a miserable place to discover a broken
  # toolchain. A five-second scratch build — run with the same DOCKER_CONFIG
  # the unit exports — catches that class of failure here, while the installer
  # can still explain it.
  PROBE_DIR=$(mktemp -d)
  printf 'FROM scratch\nCOPY .keep /\n' > "$PROBE_DIR/Dockerfile"
  : > "$PROBE_DIR/.keep"
  if DOCKER_CONFIG="$DATA_DIR/docker-config" docker_cmd build -q -t ninedeploy-install-probe "$PROBE_DIR" >/dev/null 2>&1; then
    ok "Deploy preflight passed: docker build works (DOCKER_CONFIG=$DATA_DIR/docker-config)"
  else
    warn "Deploy preflight FAILED — \`docker build\` is broken in this environment, so the first deploy will fail too. Output:"
    PROBE_OUT=$(DOCKER_CONFIG="$DATA_DIR/docker-config" docker_cmd build -t ninedeploy-install-probe "$PROBE_DIR" 2>&1 || true)
    printf '%s\n' "$PROBE_OUT" | tail -8
    case "$PROBE_OUT" in
      *"read-only file system"*)
        warn "Known cause: buildx writing under a read-only /root. This release's unit exports DOCKER_CONFIG=$DATA_DIR/docker-config — verify with: systemctl show ninedeploy --property=Environment"
        ;;
    esac
  fi
  docker_cmd rmi ninedeploy-install-probe >/dev/null 2>&1 || true
  rm -rf "$PROBE_DIR"

  # PM2 boot resurrection: bare-metal deployments live in a PM2 daemon that
  # dies with every reboot — and with the panel restart above. The server
  # keeps /root/.pm2/dump.pm2 fresh after each lifecycle change; this unit
  # restores that dump at boot. It is a clean no-op until the first PM2
  # deployment writes a dump (ConditionPathExists).
  PM2_UNIT_TEMPLATE="$INSTALL_DIR/systemd/ninedeploy-pm2.service"
  PM2_CLI="$INSTALL_DIR/apps/server/node_modules/pm2/bin/pm2"
  if [ -f "$PM2_UNIT_TEMPLATE" ] && [ -f "$PM2_CLI" ]; then
    PM2_STAGE_DIR=$(mktemp -d)
    sed -e "s|@NODE@|$(which node)|g" \
        -e "s|@INSTALL_DIR@|${INSTALL_DIR}|g" \
        -e "s|@PM2_HOME@|${DATA_DIR}/pm2|g" \
        "$PM2_UNIT_TEMPLATE" > "$PM2_STAGE_DIR/ninedeploy-pm2.service"
    if ! command -v systemd-analyze &>/dev/null \
      || systemd-analyze verify "$PM2_STAGE_DIR/ninedeploy-pm2.service" >/dev/null 2>&1; then
      sudo install -m 0644 "$PM2_STAGE_DIR/ninedeploy-pm2.service" /etc/systemd/system/ninedeploy-pm2.service
      sudo systemctl daemon-reload
      sudo systemctl enable ninedeploy-pm2 >/dev/null 2>&1 || true
      # Also runs right now: on upgrades the panel restart above killed the
      # PM2 daemon's processes, and resurrect brings them straight back.
      sudo systemctl start ninedeploy-pm2 >/dev/null 2>&1 || true
      if systemctl is-failed --quiet ninedeploy-pm2 2>/dev/null; then
        sudo systemctl reset-failed ninedeploy-pm2 >/dev/null 2>&1 || true
        warn "PM2 resurrect unit failed to run; inspect: journalctl -u ninedeploy-pm2 -n 30"
      else
        ok "PM2 deployments are restored at boot (ninedeploy-pm2.service)"
      fi
    else
      warn "Rendered ninedeploy-pm2 unit failed verification; PM2 deployments will not auto-restore at boot"
    fi
    rm -rf "$PM2_STAGE_DIR"
  else
    warn "PM2 CLI or unit template not found; bare-metal deployments will not auto-restore at boot"
  fi
else
  # r357: nothing is restarted here, so the build is the success criterion.
  prune_upgrade_rollbacks
  if [ -n "${PREVIOUS_VERSION:-}" ]; then
    # r459: this was an UPGRADE — any running foreground panel still executes
    # the OLD code over the replaced tree (node's lazy requires can even crash
    # it). Do not let the banner below read as "restarted".
    warn "systemd not available and this was an upgrade: a running foreground panel still executes the OLD code — restart it manually now."
  else
    warn "systemd not available — start the panel in the foreground when ready."
  fi
  info "For production, set up a process manager (systemd/pm2/launchd)."
fi

# ── 6. Print summary ──────────────────────────────────────────────────────

HOST="${NINEDEPLOY_HOST:-0.0.0.0}"
PORT="${NINEDEPLOY_PORT:-3000}"
PUBLIC_URL="${NINEDEPLOY_PUBLIC_URL:-$(panel_default_url "$HOST" "$PORT")}"
# 0.16: an upgraded install that predates the setup token and was never set
# up uses the token the server generated in its data directory.
SETUP_TOKEN="${NINEDEPLOY_SETUP_TOKEN:-}"
if [ -z "$SETUP_TOKEN" ]; then
  _nd_token_file="$(resolve_data_dir "$INSTALL_DIR" 2>/dev/null)/setup-token"
  [ -f "$_nd_token_file" ] && SETUP_TOKEN="$(tr -d '[:space:]' < "$_nd_token_file")"
fi
case "$HOST" in
  ""|0.0.0.0|::|"[::]"|127.0.0.1|localhost) PROBE_HOST="127.0.0.1" ;;
  *:*) PROBE_HOST="[${HOST#[}"; PROBE_HOST="${PROBE_HOST%]}]" ;;
  *) PROBE_HOST="$HOST" ;;
esac

echo ""
echo -e "${BOLD}╔══════════════════════════════════════════╗${NC}"
echo -e "${BOLD}║       ✓ Installation Complete            ║${NC}"
echo -e "${BOLD}╚══════════════════════════════════════════╝${NC}"
echo ""
echo -e "  ${CYAN}Dashboard:${NC}  ${PUBLIC_URL}"
echo -e "  ${CYAN}API:${NC}       ${PUBLIC_URL}/health"
print_setup_token_hint "http://${PROBE_HOST}:${PORT}" "$PUBLIC_URL" "$SETUP_TOKEN" "$(panel_default_url "$HOST" "$PORT")"
# A2: say whether the panel domain points here yet.
if [ -n "${NINEDEPLOY_DOMAIN:-}" ]; then warn_domain_dns "$NINEDEPLOY_DOMAIN"; fi
case "$HOST" in
  127.0.0.1|localhost) echo -e "  ${CYAN}Access:${NC}     the panel listens on 127.0.0.1 only — use an SSH tunnel (ssh -L ${PORT}:127.0.0.1:${PORT} <server>)" ;;
  *) [ "$(uname -s)" = "Linux" ] && warn_if_firewalled "$PORT" ;;
esac
echo ""
echo -e "  ${YELLOW}Next steps:${NC}"
echo -e "  1. Open ${PUBLIC_URL} in your browser"
echo -e "  2. Create your admin account (first-run setup)"
echo -e "  3. Deploy your first service from the Hub or a Git repo"
echo ""
if [ "$(uname -s)" = "Linux" ]; then
  echo -e "  ${CYAN}Manage:${NC}"
  echo -e "  sudo systemctl status ninedeploy"
  echo -e "  sudo systemctl restart ninedeploy"
  echo -e "  journalctl -u ninedeploy -f"
fi
echo ""
echo -e "  ${CYAN}CLI:${NC}"
echo -e "  npx ninedeploy init      # complete initial admin setup"
echo -e "  npx ninedeploy doctor    # run diagnostic check"
echo -e "  npx ninedeploy services list"
echo ""