deploy/vektor-media-selfheal.sh

Rama main · réplica de lectura del repositorio original.

#!/bin/bash
# Vektor selfheal media — fixed handlers called only by vektor-actions.
# State is persistent on the PVE host so API restarts cannot reset limits.
set -u
ACTION="${1:-}"
STATE_DIR=/var/lib/vektor/selfheal-media
COOLDOWN=1800
MAX_ATTEMPTS=2
mkdir -p "$STATE_DIR"
chmod 700 "$STATE_DIR"

case "$ACTION" in
  selfheal_media_reset_jellyfin|selfheal_media_reset_immich|selfheal_media_reset_authelia)
    SERVICE="${ACTION#selfheal_media_reset_}"
    case "$SERVICE" in
      jellyfin|immich|authelia) ;;
      *) echo "ERROR: service inconnue"; exit 2 ;;
    esac
    STATE="$STATE_DIR/$SERVICE.state"
    exec 9>"$STATE_DIR/$SERVICE.lock"
    flock -x 9
    rm -f "$STATE"
    echo "RESET: budget de tentatives $SERVICE réinitialisé après résolution Grafana."
    exit 0
    ;;
  selfheal_media_jellyfin)
    SERVICE=jellyfin
    CT=102
    STATE="$STATE_DIR/jellyfin.state"
    probe() {
      pct exec 102 -- sh -c 'systemctl is-active --quiet jellyfin && curl -fsS --max-time 5 http://127.0.0.1:8096/health >/dev/null'
    }
    restart() {
      pct exec 102 -- systemctl restart jellyfin
    }
    ;;
  selfheal_media_immich)
    SERVICE=immich
    CT=109
    STATE="$STATE_DIR/immich.state"
    # Check server/API only. Never restart DB/Redis/ML and never touch data.
    probe() {
      pct exec 109 -- docker exec immich_server node -e 'fetch("http://127.0.0.1:2283/api/server/ping").then(async r=>{const b=await r.json();process.exit(r.status===200&&b.res==="pong"?0:1)}).catch(()=>process.exit(1))'
    }
    restart() {
      pct exec 109 -- docker restart immich_server
    }
    ;;
  selfheal_media_authelia)
    SERVICE=authelia
    CT=104
    STATE="$STATE_DIR/authelia.state"
    # SSO Authelia (CT 104, conteneur Docker, image pinnee 4.39.28).
    # Probe = endpoint local /api/health ; restart = conteneur uniquement,
    # jamais la DB (CT 106) ni la config. Complementaire du gardien sso-switch
    # (VPS) qui bascule les sites sur la replique pendant la panne.
    probe() {
      pct exec 104 -- sh -c 'curl -fsS --max-time 5 http://127.0.0.1:9091/api/health >/dev/null'
    }
    restart() {
      pct exec 104 -- docker restart authelia
    }
    ;;
  *)
    echo "ERROR: action inconnue"
    exit 2
    ;;
esac

# Serialize requests; a fixed per-service state file counts across alert IDs.
exec 9>"$STATE_DIR/$SERVICE.lock"
flock -x 9
NOW=$(date +%s)
ATTEMPTS=0
LAST=0
if [ -f "$STATE" ]; then
  read -r ATTEMPTS LAST < "$STATE" || true
fi
if ! [[ "$ATTEMPTS" =~ ^[0-9]+$ && "$LAST" =~ ^[0-9]+$ ]]; then
  ATTEMPTS=0
  LAST=0
fi

if probe >/dev/null 2>&1; then
  rm -f "$STATE"
  echo "HEALTHY: $SERVICE répond sainement avant action ; aucun restart nécessaire. Budget de l’épisode réinitialisé."
  exit 0
fi
if [ "$ATTEMPTS" -ge "$MAX_ATTEMPTS" ]; then
  echo "MAX_ATTEMPTS: $SERVICE est toujours en panne après $ATTEMPTS/$MAX_ATTEMPTS tentatives ; escalade opérateur requise, aucun rollback de données effectué."
  exit 0
fi
if [ "$LAST" -gt 0 ] && [ $((NOW - LAST)) -lt "$COOLDOWN" ]; then
  REMAIN=$((COOLDOWN - (NOW - LAST)))
  echo "COOLDOWN: $SERVICE, prochaine tentative possible dans ${REMAIN}s (cooldown 30 min)."
  exit 0
fi

ATTEMPTS=$((ATTEMPTS + 1))
# Save before the disruptive action: interruption/crash cannot grant extra retries.
printf '%s %s\n' "$ATTEMPTS" "$NOW" > "$STATE.tmp"
chmod 600 "$STATE.tmp"
mv -f "$STATE.tmp" "$STATE"
if ! restart >/dev/null 2>&1; then
  echo "ERROR: restart $SERVICE échoué (tentative $ATTEMPTS/$MAX_ATTEMPTS)."
  exit 1
fi

# Health probes with bounded wait, so command success alone is not declared healthy.
for delay in 2 3 5 8 13 21; do
  sleep "$delay"
  if probe >/dev/null 2>&1; then
    echo "ATTEMPT $ATTEMPTS/$MAX_ATTEMPTS HEALTHY: $SERVICE redémarré et son contrôle local répond OK."
    exit 0
  fi
done
echo "UNHEALTHY: $SERVICE ne répond toujours pas après le restart (tentative $ATTEMPTS/$MAX_ATTEMPTS) ; aucune restauration DB/photos/config n'a été tentée."
exit 1