scripts/restore-backup-vm.sh
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bash
# Restore a Coolify instance backup into an offline KVM VM to test database migrations.
#
# Usage:
# scripts/restore-backup-vm.sh asks for the inputs below
# scripts/restore-backup-vm.sh destroy deletes the VM, its disk, and the Coolify base images
#
# Inputs (asked interactively, so they do not go into the shell history):
# Backup file pg_dump custom-format backup (.dmp), optionally gzipped (.gz)
# APP_KEY APP_KEY of the instance that created the backup (base64:...), not shown
# Version Coolify version or image tag, e.g. 4.3.23 or sha-99c8409 (empty: latest stable)
# Cloud y for a Coolify Cloud backup: starts Coolify with SELF_HOSTED=false, as Cloud runs.
# Self-hosted mode on a Cloud backup adds the localhost server and makes app:init
# try SSH to every server, so Reverb does not start and the container stays unhealthy.
#
# Flow:
# 1. Base image (once per version, until destroy): a separate builder VM with internet access installs
# Coolify. It never receives the backup. Its disk is kept as the base image.
# 2. The restore VM boots from a copy of the base image on a host-only network without
# internet access and without DNS. It never has internet access, so the restored
# instance cannot reach real servers, notification channels, or Stripe.
# 3. The script confirms that the VM is offline, copies the backup, replaces the database,
# sets APP_KEY, turns off Horizon and the scheduler, and starts Coolify.
# Coolify runs its migrations and seeders on start; no queued or scheduled job runs.
#
# Environment overrides:
# VM_NAME (coolify-restore-test), VM_MEMORY (4096), VM_VCPUS (4), VM_DISK_SIZE (60G),
# IMAGE_DIR (/var/lib/libvirt/images/coolify-restore-test),
# NAT_NETWORK (default), OFFLINE_NETWORK (coolify-restore-offline),
# OFFLINE_SUBNET (192.168.241)
set -euo pipefail
VM_NAME=${VM_NAME:-coolify-restore-test}
BUILDER_NAME=$VM_NAME-builder
VM_MEMORY=${VM_MEMORY:-4096}
VM_VCPUS=${VM_VCPUS:-4}
VM_DISK_SIZE=${VM_DISK_SIZE:-60G}
IMAGE_DIR=${IMAGE_DIR:-/var/lib/libvirt/images/coolify-restore-test}
NAT_NETWORK=${NAT_NETWORK:-default}
OFFLINE_NETWORK=${OFFLINE_NETWORK:-coolify-restore-offline}
OFFLINE_SUBNET=${OFFLINE_SUBNET:-192.168.241}
UBUNTU_IMAGE_URL=https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img
UBUNTU_IMAGE="$IMAGE_DIR/ubuntu-noble-amd64.qcow2"
SSH_KEY="$IMAGE_DIR/ssh_key"
VIRSH=(virsh --connect qemu:///system)
SSH_OPTS=(-i "$SSH_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -o ServerAliveInterval=15 -o ServerAliveCountMax=8)
log() { printf '\n\033[1;34m==> %s\033[0m\n' "$*"; }
die() {
printf '\033[1;31mERROR: %s\033[0m\n' "$*" >&2
exit 1
}
usage() {
sed -n '2,14p' "$0" | sed 's/^# \{0,1\}//'
exit 1
}
# delete_vm <name>: stops and deletes a VM with its disk and seed ISO.
delete_vm() {
if "${VIRSH[@]}" dominfo "$1" >/dev/null 2>&1; then
"${VIRSH[@]}" destroy "$1" >/dev/null 2>&1 || true
"${VIRSH[@]}" undefine "$1" --nvram >/dev/null 2>&1 || "${VIRSH[@]}" undefine "$1" >/dev/null
fi
rm -f "$IMAGE_DIR/$1.qcow2" "$IMAGE_DIR/$1-seed.iso"
}
# vm_ip <name>: the DHCP lease of the VM. Each VM has a new random MAC address,
# so a lease of a deleted VM cannot match.
vm_ip() {
"${VIRSH[@]}" domifaddr "$1" --source lease 2>/dev/null |
awk '$3 == "ipv4" { sub(/\/.*/, "", $4); print $4; exit }'
}
# wait_for_ip <name>: prints the IP address on stdout and the progress on stderr.
wait_for_ip() {
local ip=""
printf 'Waiting for the IP address of %s (up to 3 minutes)' "$1" >&2
for _ in $(seq 1 90); do
ip=$(vm_ip "$1")
[ -n "$ip" ] && break
printf '.' >&2
sleep 2
done
echo >&2
[ -n "$ip" ] || die "VM $1 did not get an IP address from DHCP. Boot log: virsh --connect qemu:///system console $1"
echo "$ip"
}
# wait_for_ssh <ip> <name>: waits for SSH and for cloud-init to finish.
wait_for_ssh() {
printf 'Waiting for SSH on %s (up to 5 minutes)' "$1"
for _ in $(seq 1 60); do
if ssh "${SSH_OPTS[@]}" "root@$1" true 2>/dev/null; then
echo
echo "Waiting for cloud-init in the VM (up to 10 minutes)..."
ssh "${SSH_OPTS[@]}" "root@$1" 'timeout 600 cloud-init status --wait >/dev/null' ||
echo "cloud-init did not finish cleanly; continuing. Log: /var/log/cloud-init-output.log in the VM"
return 0
fi
printf '.'
sleep 3
done
echo
die "SSH to root@$1 does not respond. Boot log: virsh --connect qemu:///system console $2"
}
vm_ssh() { ssh "${SSH_OPTS[@]}" "root@$VM_IP" "$@"; }
ensure_offline_network() {
if ! "${VIRSH[@]}" net-info "$OFFLINE_NETWORK" >/dev/null 2>&1; then
log "Creating offline libvirt network $OFFLINE_NETWORK ($OFFLINE_SUBNET.0/24)"
local xml
xml=$(mktemp)
# No <forward> element: libvirt blocks all traffic between this network and other networks.
# <dns enable='no'/>: the VM cannot resolve host names through the host either.
cat >"$xml" <<EOF
<network>
<name>$OFFLINE_NETWORK</name>
<bridge stp='on' delay='0'/>
<dns enable='no'/>
<ip address='$OFFLINE_SUBNET.1' netmask='255.255.255.0'>
<dhcp><range start='$OFFLINE_SUBNET.10' end='$OFFLINE_SUBNET.200'/></dhcp>
</ip>
</network>
EOF
"${VIRSH[@]}" net-define "$xml" >/dev/null
rm -f "$xml"
"${VIRSH[@]}" net-autostart "$OFFLINE_NETWORK" >/dev/null
fi
if [[ " $("${VIRSH[@]}" net-list --name | xargs) " != *" $OFFLINE_NETWORK "* ]]; then
"${VIRSH[@]}" net-start "$OFFLINE_NETWORK" >/dev/null
fi
}
# create_vm <name> <backing-image> <network>: creates a VM with a copy-on-write disk on the image.
create_vm() {
local name=$1 backing=$2 network=$3 seed_dir
log "Creating VM $name on network $network (${VM_VCPUS} vCPU, ${VM_MEMORY} MiB RAM)"
qemu-img create -q -f qcow2 -F qcow2 -b "$backing" "$IMAGE_DIR/$name.qcow2" "$VM_DISK_SIZE"
seed_dir=$(mktemp -d)
cat >"$seed_dir/user-data" <<EOF
#cloud-config
hostname: $name
disable_root: false
ssh_pwauth: false
users:
- name: root
ssh_authorized_keys:
- $(cat "$SSH_KEY.pub")
runcmd:
- sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
- systemctl restart ssh
EOF
printf 'instance-id: %s-%s\nlocal-hostname: %s\n' "$name" "$(date +%s)" "$name" >"$seed_dir/meta-data"
# Match the NIC by name, not by MAC: the restore VM boots from the disk of the builder VM
# with a different MAC address. A new instance-id makes cloud-init apply this on each VM.
cat >"$seed_dir/network-config" <<EOF
version: 2
ethernets:
nic:
match:
name: "en*"
dhcp4: true
EOF
if command -v xorriso >/dev/null; then
xorriso -as mkisofs -quiet -V cidata -J -r -o "$IMAGE_DIR/$name-seed.iso" "$seed_dir/user-data" "$seed_dir/meta-data" "$seed_dir/network-config" 2>/dev/null
elif command -v cloud-localds >/dev/null; then
cloud-localds --network-config="$seed_dir/network-config" "$IMAGE_DIR/$name-seed.iso" "$seed_dir/user-data" "$seed_dir/meta-data"
else
genisoimage -quiet -V cidata -J -r -o "$IMAGE_DIR/$name-seed.iso" "$seed_dir/user-data" "$seed_dir/meta-data" "$seed_dir/network-config"
fi
rm -rf "$seed_dir"
virt-install --connect qemu:///system --name "$name" --memory "$VM_MEMORY" --vcpus "$VM_VCPUS" \
--import --os-variant ubuntu24.04 \
--disk "path=$IMAGE_DIR/$name.qcow2,format=qcow2,bus=virtio" \
--disk "path=$IMAGE_DIR/$name-seed.iso,device=cdrom" \
--network "network=$network,model=virtio" \
--graphics none --noautoconsole >/dev/null
}
# build_base_image <path>: installs Coolify in a builder VM with internet access and keeps its disk.
build_base_image() {
local base=$1
if [ ! -f "$UBUNTU_IMAGE" ]; then
log "Downloading Ubuntu 24.04 cloud image"
curl -fL --progress-bar -o "$UBUNTU_IMAGE.part" "$UBUNTU_IMAGE_URL"
mv "$UBUNTU_IMAGE.part" "$UBUNTU_IMAGE"
fi
delete_vm "$BUILDER_NAME"
create_vm "$BUILDER_NAME" "$UBUNTU_IMAGE" "$NAT_NETWORK"
VM_IP=$(wait_for_ip "$BUILDER_NAME")
wait_for_ssh "$VM_IP" "$BUILDER_NAME"
log "Installing Coolify $VERSION in the builder VM (no backup data in this VM)"
vm_ssh "set -o pipefail; curl -fsSL https://cdn.coollabs.io/coolify/install.sh | bash -s -- $VERSION" ||
die "The Coolify install failed in the builder VM. No base image was saved."
# A broken base image would be reused by every later run, so check it before it is saved.
vm_ssh 'docker inspect coolify coolify-db coolify-redis >/dev/null && [ -f /data/coolify/source/.env ]' ||
die "Coolify is not installed in the builder VM. No base image was saved."
echo "Shutting down the builder VM (up to 2 minutes)..."
vm_ssh 'shutdown -h now' || true
for _ in $(seq 1 60); do
[ "$("${VIRSH[@]}" domstate "$BUILDER_NAME")" = "shut off" ] && break
sleep 2
done
[ "$("${VIRSH[@]}" domstate "$BUILDER_NAME")" = "shut off" ] || die "The builder VM did not shut down."
"${VIRSH[@]}" undefine "$BUILDER_NAME" --nvram >/dev/null 2>&1 || "${VIRSH[@]}" undefine "$BUILDER_NAME" >/dev/null
mv "$IMAGE_DIR/$BUILDER_NAME.qcow2" "$base"
rm -f "$IMAGE_DIR/$BUILDER_NAME-seed.iso"
}
# --- main ---------------------------------------------------------------------
[ "${1:-}" = "destroy" ] && {
delete_vm "$VM_NAME"
delete_vm "$BUILDER_NAME"
rm -f "$IMAGE_DIR"/coolify-base-*.qcow2
echo "Deleted VM $VM_NAME, its disk, and the Coolify base images."
exit 0
}
[ $# -eq 0 ] || usage
[ -t 0 ] || die "Run this script in an interactive terminal: it asks for its inputs."
read -r -e -p "Backup file (.dmp or .gz): " BACKUP_FILE
BACKUP_FILE=${BACKUP_FILE/#\~/$HOME}
read -r -s -p "APP_KEY of the backed-up instance (input is hidden): " APP_KEY
echo
read -r -p "Coolify version or image tag (empty for latest stable): " VERSION
read -r -p "Is this a Coolify Cloud backup? [y/N]: " CLOUD_BACKUP
[ -s "$BACKUP_FILE" ] || die "Backup file '$BACKUP_FILE' does not exist or is empty."
[[ "$APP_KEY" == base64:* ]] || die "APP_KEY must start with 'base64:'."
for cmd in virsh virt-install qemu-img ssh scp ssh-keygen curl jq; do
command -v "$cmd" >/dev/null || die "Missing command: $cmd"
done
[ -e /dev/kvm ] || die "/dev/kvm is not available."
if [ -z "$VERSION" ]; then
VERSION=$(curl -fsSL https://cdn.coollabs.io/coolify/versions.json | jq -r '.coolify.v4.version')
echo "Latest stable version: $VERSION"
fi
[[ "${CLOUD_BACKUP,,}" =~ ^(y|yes)$ ]] && SELF_HOSTED=false || SELF_HOSTED=true
[[ "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]] || die "Version '$VERSION' contains characters that are not allowed."
BASE_IMAGE="$IMAGE_DIR/coolify-base-$VERSION.qcow2"
# Other VMs and containers can use most of the host memory. A VM that does not fit
# makes the host swap until it stops responding, so stop before it starts.
# The previous VM is deleted below, so its memory counts as available.
OLD_VM_KIB=$("${VIRSH[@]}" dominfo "$VM_NAME" 2>/dev/null | awk '/^State:/ { running = ($2 == "running") } /^Used memory:/ { kib = $3 } END { print running ? kib : 0 }' || true)
AVAILABLE_MIB=$(($(awk '/^MemAvailable:/ { print $2 }' /proc/meminfo) / 1024 + ${OLD_VM_KIB:-0} / 1024))
REQUIRED_MIB=$((VM_MEMORY + 2048))
if [ "$AVAILABLE_MIB" -lt "$REQUIRED_MIB" ]; then
die "Not enough free memory: ${AVAILABLE_MIB} MiB available, ${REQUIRED_MIB} MiB required (VM_MEMORY + 2048 MiB for the host).
Stop other VMs (virsh --connect qemu:///system list) or set a lower VM_MEMORY."
fi
if "${VIRSH[@]}" dominfo "$VM_NAME" >/dev/null 2>&1; then
log "Deleting the previous VM $VM_NAME"
fi
delete_vm "$VM_NAME"
mkdir -p "$IMAGE_DIR"
# The VM disks grow on the host disk. When the host disk is full, QEMU pauses the VM without a
# message. The restore VM holds the backup and the restored database (about 3 times the backup),
# a new base image needs about 6 GiB. A gzipped backup is counted as 5 times its size.
BACKUP_MIB=$(($(stat -c %s "$BACKUP_FILE") / 1024 / 1024))
gzip -t "$BACKUP_FILE" 2>/dev/null && BACKUP_MIB=$((BACKUP_MIB * 5))
FREE_MIB=$(($(df --output=avail -k "$IMAGE_DIR" | tail -n 1) / 1024))
NEEDED_MIB=$((BACKUP_MIB * 3 + 2048))
[ -f "$BASE_IMAGE" ] || NEEDED_MIB=$((NEEDED_MIB + 6144))
if [ "$FREE_MIB" -lt "$NEEDED_MIB" ]; then
die "Not enough free disk space in $IMAGE_DIR: ${FREE_MIB} MiB free, about ${NEEDED_MIB} MiB needed.
Free disk space, or set IMAGE_DIR to a larger disk."
fi
if [ ! -f "$SSH_KEY" ]; then
# Do not use the comment "coolify": install.sh removes authorized_keys lines with exactly that comment.
ssh-keygen -q -t ed25519 -N '' -C restore-test-vm -f "$SSH_KEY"
fi
if [ -f "$BASE_IMAGE" ]; then
log "Using the cached base image for Coolify $VERSION"
else
build_base_image "$BASE_IMAGE"
fi
ensure_offline_network
create_vm "$VM_NAME" "$BASE_IMAGE" "$OFFLINE_NETWORK"
VM_IP=$(wait_for_ip "$VM_NAME")
log "VM IP address on the offline network: $VM_IP"
wait_for_ssh "$VM_IP" "$VM_NAME"
log "Confirming that the VM has no internet access"
if vm_ssh 'timeout 5 bash -c "echo >/dev/tcp/1.1.1.1/443" 2>/dev/null || timeout 5 bash -c "echo >/dev/tcp/8.8.8.8/53" 2>/dev/null'; then
die "The VM can reach the internet. Stopped before the backup was copied."
fi
if vm_ssh 'getent hosts cdn.coollabs.io >/dev/null'; then
die "The VM can resolve internet host names. Stopped before the backup was copied."
fi
log "Copying the backup into the VM ($(du -h "$BACKUP_FILE" | cut -f1))"
if gzip -t "$BACKUP_FILE" 2>/dev/null; then
gzip -dc "$BACKUP_FILE" | vm_ssh 'cat >/root/coolify-backup.dmp'
else
scp "${SSH_OPTS[@]}" "$BACKUP_FILE" "root@$VM_IP:/root/coolify-backup.dmp"
fi
log "Restoring the backup and starting Coolify (migrations run on start)"
# Send APP_KEY through stdin, so it does not show in the process list of this host or the VM.
{
printf 'APP_KEY=%q\n' "$APP_KEY"
printf 'SELF_HOSTED=%q\n' "$SELF_HOSTED"
cat <<'REMOTE'
set -euo pipefail
cd /data/coolify/source
echo "Waiting for coolify-db (up to 2 minutes)..."
for _ in $(seq 1 60); do
[ "$(docker inspect -f '{{.State.Health.Status}}' coolify-db 2>/dev/null)" = healthy ] && break
sleep 2
done
[ "$(docker inspect -f '{{.State.Health.Status}}' coolify-db 2>/dev/null)" = healthy ] || {
echo "ERROR: coolify-db is not healthy. Logs: docker logs coolify-db" >&2
exit 1
}
# Start the same image tag again after the restore.
LATEST_IMAGE=$(docker inspect -f '{{.Config.Image}}' coolify | sed 's/.*://')
docker stop coolify >/dev/null
# coolify-realtime only exists before 4.4; compose recreates it when the restored version still uses it.
docker rm -f coolify-realtime >/dev/null 2>&1 || true
sed -i "s|^APP_KEY=.*|APP_KEY=${APP_KEY}|" .env
# Turn off Horizon (queue workers) and the scheduler: no job from the backup runs.
# Jobs that the UI dispatches stay in Redis and are never processed.
for flag in HORIZON_ENABLED SCHEDULER_ENABLED; do
sed -i "/^${flag}=/d" .env
echo "${flag}=false" >>.env
done
sed -i '/^SELF_HOSTED=/d' .env
echo "SELF_HOSTED=${SELF_HOSTED}" >>.env
# Recreate the database: tables from a newer fresh install must not remain.
docker exec coolify-db psql -q -U coolify -d postgres -v ON_ERROR_STOP=1 \
-c 'DROP DATABASE coolify WITH (FORCE)' -c 'CREATE DATABASE coolify OWNER coolify'
echo "Running pg_restore of $(du -h /root/coolify-backup.dmp | cut -f1) (a large backup takes several minutes)..."
restore_section() {
docker exec -i coolify-db pg_restore --exit-on-error --no-acl --no-owner --section="$1" -U coolify -d coolify </root/coolify-backup.dmp
}
restore_section pre-data
restore_section data
# Coolify Cloud backups leave out the rows of log and execution tables, such as
# scheduled_database_backup_executions. Rows that point to them must go before
# post-data adds the foreign keys.
docker exec coolify-db psql -q -U coolify -d coolify -v ON_ERROR_STOP=1 -c "
DO \$\$
DECLARE deleted bigint;
BEGIN
IF to_regclass('public.database_backup_s3_replicas') IS NOT NULL THEN
DELETE FROM database_backup_s3_replicas r
WHERE NOT EXISTS (SELECT 1 FROM scheduled_database_backup_executions e WHERE e.id = r.execution_id);
GET DIAGNOSTICS deleted = ROW_COUNT;
RAISE NOTICE 'Deleted % database_backup_s3_replicas rows without an execution', deleted;
END IF;
END
\$\$;"
restore_section post-data
# Drop cached data of the fresh installation.
REDIS_PASSWORD=$(grep '^REDIS_PASSWORD=' .env | cut -d= -f2-)
docker exec coolify-redis redis-cli -a "$REDIS_PASSWORD" --no-auth-warning FLUSHALL >/dev/null
COMPOSE_FILES="-f docker-compose.yml -f docker-compose.prod.yml"
[ -f docker-compose.custom.yml ] && COMPOSE_FILES="$COMPOSE_FILES -f docker-compose.custom.yml"
[ -f docker-compose.postgres-upgrade.yml ] && COMPOSE_FILES="$COMPOSE_FILES -f docker-compose.postgres-upgrade.yml"
echo "Starting Coolify $LATEST_IMAGE..."
# --pull never: the VM is offline, all images come from the base image.
# shellcheck disable=SC2086
LATEST_IMAGE=$LATEST_IMAGE docker compose --env-file .env $COMPOSE_FILES up -d --pull never --wait --wait-timeout 600
# The container reports healthy while migrations still run on a large database.
# Seeders continue in the background; with many servers they can run for a long time.
echo "Waiting for migrations to finish (progress: docker logs -f coolify in the VM)..."
started=$SECONDS
next_report=$((SECONDS + 60))
while docker exec coolify sh -c "ps aux | grep -q '[a]rtisan start:migration'"; do
sleep 5
if [ "$SECONDS" -ge "$next_report" ]; then
echo " migrations still running ($(((SECONDS - started) / 60)) min)"
next_report=$((next_report + 60))
fi
done
echo
echo "--- Migration output (docker logs coolify) ---"
docker logs coolify 2>&1 | sed -n '/Migration is enabled/,/Seeder is enabled/p'
echo
echo "--- Errors after start ---"
docker logs coolify 2>&1 | grep -iE 'error|exception|fail' | tail -n 30 || true
echo
echo "--- Pending migrations ---"
docker exec coolify php artisan migrate:status --pending || true
echo
echo "--- Queue workers and scheduler ---"
if docker exec coolify sh -c "ps aux | grep -qE '[a]rtisan (horizon|schedule:work|queue:work)'"; then
echo "ERROR: A queue worker or the scheduler runs."
exit 1
fi
echo "Horizon and the scheduler are off."
REMOTE
} | vm_ssh 'bash -s'
HOST_ADDR=$(hostname -f 2>/dev/null || hostname)
cat <<EOF
$(printf '\033[1;32m')Restore finished.$(printf '\033[0m')
Coolify UI (from this host): http://$VM_IP:8000
From your workstation: ssh -N -L 8000:$VM_IP:8000 -L 6001:$VM_IP:6001 -L 6002:$VM_IP:6002 root@$HOST_ADDR
then open http://localhost:8000
Log in with a user account from the backup.
SSH into the VM: ssh -i $SSH_KEY root@$VM_IP
Coolify logs: ssh -i $SSH_KEY root@$VM_IP docker logs -f coolify
Delete the VM: $0 destroy
The VM never had internet access. Horizon and the scheduler are off, so no job runs.
EOF