install.sh
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env bash
# ==============================================================================
# fusionAIze Grid - Advanced Universal Orchestrator
# Version 1.10.1
# ==============================================================================
set -euo pipefail
# --- Colors & Styling ---
RED='\033[0;31m'
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
CYAN='\033[0;36m'
MAGENTA='\033[0;35m'
NC='\033[0m'
BOLD='\033[1m'
DIM='\033[2m'
echo -e "\033[38;2;0;82;204m▐▘ ▘ \033[38;2;196;217;0m▄▖▄▖\033[38;2;0;82;204m \033[0m \033[38;2;38;67;123m▄▖ ▘ ▌\033[0m"
echo -e "\033[38;2;0;82;204m▜▘▌▌▛▘▌▛▌▛▌\033[38;2;196;217;0m▌▌▐ \033[38;2;0;82;204m▀▌█▌\033[0m \033[38;2;38;67;123m▌ ▛▘▌▛▌\033[0m"
echo -e "\033[38;2;0;82;204m▐ ▙▌▄▌▌▙▌▌▌\033[38;2;196;217;0m▛▌▟▖\033[38;2;0;82;204m▙▖▙▖\033[0m \033[38;2;38;67;123m▙▌▌ ▌▙▌\033[0m ${DIM}v1.10.1${NC}"
echo ""
echo -e "${DIM} Sovereign AI Infrastructure — Advanced Universal Orchestrator${NC}"
echo ""
# --- Helpers ---
info() { echo -e " ${BLUE}ℹ${NC} $1"; }
success() { echo -e " ${GREEN}✔${NC} $1"; }
warning() { echo -e " ${YELLOW}⚠${NC} $1"; }
error() { echo -e " ${RED}✘${NC} $1"; exit 1; }
divider() { echo -e " ${DIM}──────────────────────────────────────────────────────${NC}"; }
prompt() {
read -r -p "$(echo -e " ${CYAN}▸${NC} $1")" "$2" || printf -v "$2" "%s" ""
}
prompt_hidden() {
read -r -s -p "$(echo -e " ${CYAN}▸${NC} $1")" "$2"
echo ""
}
_quit() {
echo ""
divider
success "fusionAIze Grid Orchestration Complete! 🚀"
divider
echo ""
exit 0
}
# Single canonical state directory (legacy ~/.grid-state is deprecated)
STATE_DIR="${HOME}/.config/faigrid/registry"
STATE_FILE="${STATE_DIR}/state.env"
# Repository root for relative path resolution
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
TOPOLOGY_FILE="${HOME}/.config/faigrid/.env.topology"
# User-scoped registry — stable regardless of install method (git clone vs Homebrew)
LOCAL_REGISTRY="${STATE_DIR}"
# --- 1.3.0 Legacy Migration Hook ---
if [[ -f "${REPO_ROOT}/core/heart/scripts/migrate_1.3.sh" ]]; then
bash "${REPO_ROOT}/core/heart/scripts/migrate_1.3.sh"
fi
# --- Registry location migration (repo-relative → user-scoped) ---
_old_registry="${REPO_ROOT}/.faigrid/state"
if [[ -d "$_old_registry" ]]; then
mkdir -p "$LOCAL_REGISTRY"
for _f in "$_old_registry"/*.state; do
[[ -f "$_f" ]] || continue
_base="$(basename "$_f")"
if [[ ! -f "$LOCAL_REGISTRY/$_base" ]]; then
cp "$_f" "$LOCAL_REGISTRY/$_base"
fi
done
fi
unset _old_registry _f _base
CURRENT_ROLE="none"
CURRENT_VERSION="none"
# Ensure user config dirs exist
mkdir -p "$LOCAL_REGISTRY"
mkdir -p "$(dirname "$TOPOLOGY_FILE")"
# --- Agent-Native CLI Commands ---
_cmd_help() {
echo ""
echo -e " ${BOLD}Usage:${NC}"
echo -e " faigrid [OPTIONS]"
echo -e " faigrid --status [--json]"
echo ""
echo -e " ${BOLD}Options:${NC}"
echo -e " --mode local|remote Execution mode (default: interactive)"
echo -e " --target SSH_TARGET Remote SSH target for remote mode"
echo -e " --role ROLE Node role: core|edge|worker|backup|external|runner"
echo -e " --action ACTION Action: install|verify|update|control|workbench|uninstall"
echo -e " --component NAME Target a specific component"
echo -e " --vnc Enable VNC for core role"
echo -e " --yes Auto-confirm non-destructive prompts"
echo -e " --force Bypass destructive prompts (except delete-all-data)"
echo -e " --status [--json] Print node health status and exit"
echo -e " --help Show this help and exit"
echo ""
echo -e " ${BOLD}Agent interface:${NC}"
echo -e " faigrid --status --json Machine-readable JSON health output"
echo -e " faigrid-workbench status Workbench plugin registry"
echo -e " fgm health grid-messenger HTTP status"
echo -e " fgm notify \"text\" Send Telegram notification"
echo ""
exit 0
}
_port_reachable() {
# -s silent, no -f so any HTTP response (incl. 404) counts as "port open"
curl --connect-timeout 1 -so /dev/null "http://127.0.0.1:${1}/" 2>/dev/null
}
_cmd_status() {
local output_json="${1:-false}"
local version
version="$(cat "${REPO_ROOT}/VERSION" 2>/dev/null || echo "unknown")"
local node
node="$(hostname -s 2>/dev/null || hostname 2>/dev/null || echo "unknown")"
local os_name
os_name="$(uname -s | tr '[:upper:]' '[:lower:]')"
local ts
ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
local svc_names=("messenger" "n8n" "faigate" "openclaw")
local svc_ports=("9119" "5678" "8090" "18789")
local svc_stati=()
local i=0
while [[ $i -lt 4 ]]; do
if _port_reachable "${svc_ports[$i]}" 2>/dev/null; then
svc_stati+=("running")
else
svc_stati+=("stopped")
fi
i=$((i + 1))
done
if [[ "$output_json" == "true" ]]; then
printf '{\n'
printf ' "version": "%s",\n' "$version"
printf ' "node": "%s",\n' "$node"
printf ' "os": "%s",\n' "$os_name"
printf ' "services": {\n'
local j=0
while [[ $j -lt 4 ]]; do
local comma=","
[[ $j -eq 3 ]] && comma=""
printf ' "%s": {"port": %s, "status": "%s"}%s\n' \
"${svc_names[$j]}" "${svc_ports[$j]}" "${svc_stati[$j]}" "$comma"
j=$((j + 1))
done
printf ' },\n'
printf ' "at": "%s"\n' "$ts"
printf '}\n'
else
divider
echo -e " ${BOLD}Node Status${NC} ${DIM}v${version} · ${node} · ${os_name}${NC}"
divider
printf " %-14s %-6s %s\n" "SERVICE" "PORT" "STATUS"
echo -e " ${DIM}──────────────────────────────────────${NC}"
local k=0
while [[ $k -lt 4 ]]; do
local st="${svc_stati[$k]}"
printf " %-14s %-6s " "${svc_names[$k]}" "${svc_ports[$k]}"
if [[ "$st" == "running" ]]; then
echo -e "${GREEN}✔ running${NC}"
else
echo -e "${DIM}✘ stopped${NC}"
fi
k=$((k + 1))
done
divider
echo ""
fi
exit 0
}
# --- CLI Arguments ---
AUTO_YES="false"
FORCE="false"
VNC_CHOICE=""
ROLE_CHOICE=""
ACTION_NAME=""
MODE_CHOICE=""
EXEC_MODE=""
SSH_TARGET=""
ROLE_NAME=""
BOOTSTRAP_MODE="false"
COMPONENT_NAME=""
OUTPUT_JSON="false"
RUN_STATUS="false"
while [[ $# -gt 0 ]]; do
case $1 in
--mode)
MODE_CHOICE="$2"
case "$MODE_CHOICE" in
local|remote) EXEC_MODE="$MODE_CHOICE" ;;
*) error "Invalid --mode value: '${MODE_CHOICE}' (expected local|remote)" ;;
esac
shift 2 ;;
--target) SSH_TARGET="$2"; shift 2 ;;
--role) ROLE_NAME="$2"; shift 2 ;;
--action) ACTION_NAME="$2"; shift 2 ;;
--component) COMPONENT_NAME="$2"; shift 2 ;;
--vnc) VNC_CHOICE="y"; shift ;;
--yes) AUTO_YES="true"; shift ;;
--force) FORCE="true"; shift ;;
--bootstrap) BOOTSTRAP_MODE="true"; shift ;;
--json) OUTPUT_JSON="true"; shift ;;
--status) RUN_STATUS="true"; shift ;;
--help|-h) _cmd_help ;;
*) error "Unknown parameter passed: $1" ;;
esac
done
[[ "$RUN_STATUS" == "true" ]] && _cmd_status "$OUTPUT_JSON"
# --- Hardware Check Functions ---
check_hardware() {
local target_role=$1
local mode=$2
local ssh_target=$3
info "Performing hardware pre-flight checks for role: ${target_role}..."
local ncpu
local mem_kb
local mem_gb
if [[ "$mode" == "local" ]]; then
if [[ "$(uname -s)" == "Darwin" ]]; then
ncpu=$(sysctl -n hw.ncpu)
mem_kb=$(( $(sysctl -n hw.memsize) / 1024 ))
else
ncpu=$(nproc)
mem_kb=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
fi
else
if ssh -q "$ssh_target" "uname -s" | grep -q 'Darwin'; then
ncpu=$(ssh -q "$ssh_target" "sysctl -n hw.ncpu")
mem_kb=$(ssh -q "$ssh_target" "sysctl -n hw.memsize")
mem_kb=$(( mem_kb / 1024 ))
else
ncpu=$(ssh -q "$ssh_target" "nproc")
mem_kb=$(ssh -q "$ssh_target" "awk '/MemTotal/ {print \$2}' /proc/meminfo || echo 2000000")
fi
fi
mem_gb=$(( mem_kb / 1024 / 1024 ))
info "Detected CPU Cores: ${ncpu}"
info "Detected RAM: ${mem_gb} GB"
if [[ "$target_role" == "worker" ]]; then
if (( mem_gb < 16 )); then
warning "Role 'worker' usually requires >16GB RAM for local LLMs. You have ${mem_gb}GB."
fi
if (( ncpu < 4 )); then
warning "Role 'worker' needs >=4 CPU Cores for efficient prompt processing."
fi
elif [[ "$target_role" == "core" ]]; then
if (( mem_gb < 4 )); then
warning "Role 'core' runs n8n, OpenClaw, and PostgreSQL. Recommended >=4GB RAM. You have ${mem_gb}GB."
fi
fi
success "Hardware check completed."
}
# --- State Inspection ---
inspect_state() {
local mode=$1
local ssh_target=$2
if [ "$mode" = "local" ]; then
if [ -f "$STATE_FILE" ]; then
# shellcheck disable=SC1090
source "$STATE_FILE" || true
CURRENT_ROLE=${GRID_ROLE:-none}
CURRENT_VERSION=${GRID_VERSION:-none}
fi
elif [ "$mode" = "remote" ]; then
# Check remote state safely: canonical registry first, legacy read-only fallback
local remote_state=""
local canonical="\$HOME/.config/faigrid/registry/state.env"
local legacy="\$HOME/.grid-state"
if ssh -q -o ConnectTimeout=3 "$ssh_target" "[ -f \"$canonical\" ]" 2>/dev/null; then
remote_state=$(ssh -q -o ConnectTimeout=3 "$ssh_target" "cat \"$canonical\"" || echo "")
elif ssh -q -o ConnectTimeout=3 "$ssh_target" "[ -f \"$legacy\" ]" 2>/dev/null; then
remote_state=$(ssh -q -o ConnectTimeout=3 "$ssh_target" "cat \"$legacy\"" || echo "")
warning "Legacy state file ~/.grid-state found on ${ssh_target}; canonical registry is preferred."
fi
if [ -n "$remote_state" ]; then
CURRENT_ROLE=$(echo "$remote_state" | grep "GRID_ROLE" | cut -d'=' -f2 || echo "none")
CURRENT_VERSION=$(echo "$remote_state" | grep "GRID_VERSION" | cut -d'=' -f2 || echo "none")
fi
fi
}
# --- Write State ---
write_state() {
local mode=$1
local ssh_target=$2
local role=$3
# Save to local registry first
mkdir -p "$LOCAL_REGISTRY"
{
echo "GRID_ROLE=$role"
echo "GRID_VERSION=latest"
echo "INSTALL_DATE=\"$(date)\""
echo "EXEC_MODE=$mode"
if [ "$mode" = "remote" ]; then
echo "SSH_TARGET=$ssh_target"
fi
} > "$LOCAL_REGISTRY/${role}.state"
if [ "$mode" = "local" ]; then
{
echo "GRID_ROLE=$role"
echo "GRID_VERSION=latest"
echo "INSTALL_DATE=\"$(date)\""
} > "$STATE_FILE"
success "Saved state to ${STATE_FILE} (and local registry)"
else
ssh -q "$ssh_target" "mkdir -p \"\$HOME/.config/faigrid/registry\"; \
echo 'GRID_ROLE=$role' > \"\$HOME/.config/faigrid/registry/state.env\"; \
echo 'GRID_VERSION=latest' >> \"\$HOME/.config/faigrid/registry/state.env\"; \
echo 'INSTALL_DATE=\"$(date)\"' >> \"\$HOME/.config/faigrid/registry/state.env\"" || true
success "Saved remote state to ${ssh_target}:~/.config/faigrid/registry/state.env (and local registry)"
fi
}
# --- Role Resolver ---
resolve_role_dir() {
local role=$1
case "$role" in
"edge") echo "edge/pi" ;;
"core") echo "core/heart" ;;
"worker") echo "worker" ;;
"backup") echo "backup" ;;
"external") echo "external" ;;
"runner") echo "core/runners" ;;
"supervisor") echo "core/supervisor" ;;
*) error "Unknown role: $role" ;;
esac
}
# --- Grid Status Probe ---
# Uses individual variables (Bash 3.2 compatible — no associative arrays)
GRID_STATUS_core="○"
GRID_STATUS_edge="○"
GRID_STATUS_worker="○"
GRID_STATUS_backup="○"
GRID_STATUS_external="○"
GRID_STATUS_runner="○"
GRID_STATUS_supervisor="○"
probe_grid_status() {
# Check local registry only (instant)
local roles=("core" "edge" "worker" "backup" "external" "runner" "supervisor")
for role in "${roles[@]}"; do
if [[ -f "$LOCAL_REGISTRY/${role}.state" ]]; then
# Bash 3.2 lacks associative arrays; role comes from the hardcoded array
# above, never from operator/network input, so this eval is safe (V7).
eval "GRID_STATUS_${role}=\"✔\""
fi
done
}
# --- Load Local State ---
# Retrieves target info (EXEC_MODE, SSH_TARGET) from registry
load_local_state() {
local role=$1
local state_file="$LOCAL_REGISTRY/${role}.state"
if [ -f "$state_file" ]; then
# Safer parsing for Bash 3.2 + set -e + pipefail
while IFS='=' read -r key val; do
if [ -z "$key" ] || echo "$key" | grep -q "^#"; then
continue
fi
case "$key" in
EXEC_MODE) EXEC_MODE="$val" ;;
SSH_TARGET) SSH_TARGET="$val" ;;
esac
done < "$state_file"
if [ -n "${EXEC_MODE:-}" ]; then
MODE_CHOICE="$EXEC_MODE"
fi
fi
return 0
}
# --- Service Discovery ---
DISCOVERED_SERVICES=()
discover_services() {
local mode=$1
local ssh_target=$2
local run_cmd
if [[ "$mode" == "local" ]]; then
run_cmd="bash -c"
else
run_cmd="ssh -q $ssh_target"
fi
info "Scanning target for existing services..."
echo ""
# Service probe list: name, check command
local -a probes=(
"Caddy:command -v caddy || pgrep -x caddy"
"Pi-hole:command -v pihole || [ -d /etc/pihole ]"
"Docker:command -v docker"
"Ollama:command -v ollama || pgrep -x ollama"
"LM Studio:command -v lms"
"n8n:docker ps --format '{{.Names}}' 2>/dev/null | grep -q n8n"
"PostgreSQL:command -v psql || docker ps --format '{{.Names}}' 2>/dev/null | grep -q postgres"
"Redis:command -v redis-cli || docker ps --format '{{.Names}}' 2>/dev/null | grep -q redis"
"Restic:command -v restic"
"Tailscale:command -v tailscale"
"Nginx:command -v nginx || pgrep -x nginx"
)
for probe in "${probes[@]}"; do
local svc_name="${probe%%:*}"
local svc_cmd="${probe#*:}"
if $run_cmd "$svc_cmd" &>/dev/null; then
DISCOVERED_SERVICES+=("$svc_name")
echo -e " ${GREEN}●${NC} ${svc_name} detected"
fi
done
if [[ ${#DISCOVERED_SERVICES[@]} -eq 0 ]]; then
echo -e " ${DIM}No known services detected (bare system).${NC}"
fi
echo ""
}
# ==============================================================================
# STEP 1: GRID OVERVIEW
# ==============================================================================
divider
echo -e " ${BOLD}Step 1 │ Your fusionAIze Grid${NC}"
divider
echo ""
echo -e " The Grid ${BOLD}4+1 Architecture${NC} consists of these roles:"
echo ""
# Start of interactive wizard
set +e
# Probe known nodes from .faigrid/state for registered icons
probe_grid_status
_grid_icon() {
local role=$1
local status
case "$role" in
core) status="$GRID_STATUS_core" ;;
edge) status="$GRID_STATUS_edge" ;;
worker) status="$GRID_STATUS_worker" ;;
backup) status="$GRID_STATUS_backup" ;;
external) status="$GRID_STATUS_external" ;;
runner) status="$GRID_STATUS_runner" ;;
supervisor) status="$GRID_STATUS_supervisor" ;;
*) status="○" ;;
esac
if [[ "$status" == "✔" ]]; then
echo -e "${GREEN}✔${NC}"
else
echo -e "${DIM}○${NC}"
fi
}
echo -e " $(_grid_icon core) ${BOLD}grid-core${NC} ${DIM}AI Workbench — n8n, OpenClaw, Postgres, Redis${NC} ${YELLOW}[required]${NC}"
echo -e " $(_grid_icon edge) ${BOLD}grid-edge${NC} ${DIM}TLS ingress, reverse proxy, firewall, DNS${NC} ${DIM}[optional]${NC}"
echo -e " $(_grid_icon worker) ${BOLD}grid-worker${NC} ${DIM}Local LLM inference — Ollama, LM Studio${NC} ${DIM}[optional]${NC}"
echo -e " $(_grid_icon backup) ${BOLD}grid-backup${NC} ${DIM}Offsite storage vault — Synology, S3, USB${NC} ${DIM}[optional]${NC}"
echo -e " $(_grid_icon external) ${BOLD}grid-external${NC} ${DIM}Cloud extension — ext. n8n, Plane PM${NC} ${DIM}[optional]${NC}"
echo -e " $(_grid_icon runner) ${BOLD}grid-runner${NC} ${DIM}Isolated shell/browser execution environments${NC} ${DIM}[optional]${NC}"
echo -e " $(_grid_icon supervisor) ${BOLD}grid-supervisor${NC} ${DIM}Control API — Scheduler, Runner Pool, Recovery${NC} ${DIM}[optional]${NC}"
echo ""
echo -e " ${DIM}Tip: Start with grid-core. Add other nodes as your grid grows.${NC}"
# ==============================================================================
# STEP 2: WHICH NODE TO PROVISION NOW
# ==============================================================================
echo ""
divider
echo -e " ${BOLD}Step 2 │ Select Node to Provision${NC}"
divider
if [[ -z "$ROLE_NAME" ]]; then
echo ""
echo " Which node do you want to work on right now?"
echo ""
echo -e " ${BOLD}1)${NC} grid-core ${DIM}The brain — always provision this first${NC}"
echo -e " ${BOLD}2)${NC} grid-edge ${DIM}Secure ingress gateway (Caddy, DNS)${NC}"
echo -e " ${BOLD}3)${NC} grid-worker ${DIM}Local LLM execution (MacBook, GPU server)${NC}"
echo -e " ${BOLD}4)${NC} grid-backup ${DIM}Restic vault (NAS, USB disk, S3 bucket)${NC}"
echo -e " ${BOLD}5)${NC} grid-external ${DIM}Public cloud node (VPS, agency server)${NC}"
echo -e " ${BOLD}6)${NC} grid-runner ${DIM}Isolated secure execution runners${NC}"
echo -e " ${BOLD}7)${NC} grid-supervisor ${DIM}Control API for Scheduler, Runner Pool, Recovery${NC}"
echo ""
echo -e " ${BOLD}q)${NC} Quit"
echo ""
prompt "Enter role (1-6 / q): " ROLE_CHOICE
case "$ROLE_CHOICE" in
1) ROLE_NAME="core" ;;
2) ROLE_NAME="edge" ;;
3) ROLE_NAME="worker" ;;
4) ROLE_NAME="backup" ;;
5) ROLE_NAME="external" ;;
6) ROLE_NAME="runner" ;;
7) ROLE_NAME="supervisor" ;;
[Qq]|"") _quit ;;
*) warning "Invalid choice. Please enter 1-6 or q." ;;
esac
# Smart suggestion for backup co-location
if [[ "$ROLE_NAME" == "backup" ]]; then
echo ""
echo -e " ${CYAN}💡 Tip:${NC} You don't need a dedicated backup node."
echo -e " Restic can run directly on ${BOLD}grid-core${NC} or ${BOLD}grid-external${NC},"
echo -e " backing up to a Synology NAS, S3 bucket, or USB drive."
echo ""
if [[ "$AUTO_YES" != "true" ]]; then
prompt "Continue with a dedicated backup node anyway? (Y/n): " BACKUP_CONFIRM
if [[ "${BACKUP_CONFIRM:-Y}" =~ ^[Nn]$ ]]; then
info "Returning to node selection..."
exec bash "$0"
fi
fi
fi
fi
ROLE_DIR="$(resolve_role_dir "$ROLE_NAME")"
success "Selected: ${BOLD}${ROLE_NAME}${NC}"
# Check for cached target info
load_local_state "$ROLE_NAME"
# ==============================================================================
# STEP 3: DEPLOY MODE
# ==============================================================================
echo ""
divider
echo -e " ${BOLD}Step 3 │ Deploy Mode${NC}"
divider
if [[ -z "$MODE_CHOICE" ]]; then
if [[ -n "${EXEC_MODE:-}" ]]; then
# Known target found in registry
target_desc="Local Node"
if [ "$EXEC_MODE" = "remote" ]; then
target_desc="SSH: ${SSH_TARGET:-unknown}"
fi
echo ""
echo -e " ${CYAN}💡${NC} Known target found for ${BOLD}${ROLE_NAME}${NC}:"
echo ""
echo -e " ${BOLD}1)${NC} Use existing ${DIM}(${target_desc})${NC}"
echo -e " ${BOLD}2)${NC} Change target ${DIM}(Switch to local/remote/new IP)${NC}"
echo ""
prompt "Choice (1/2): " USE_EXISTING
if [ "$USE_EXISTING" = "2" ]; then
EXEC_MODE=""
MODE_CHOICE=""
SSH_TARGET=""
fi
fi
fi
# If no cached mode (or user chose to change), ask
if [[ -z "$MODE_CHOICE" ]]; then
echo ""
echo -e " How are you deploying ${BOLD}${ROLE_NAME}${NC}?"
echo ""
echo -e " ${BOLD}1)${NC} On-Node ${DIM}— I am logged into the target node right now${NC}"
echo -e " ${BOLD}2)${NC} Remote Push ${DIM}— Push to a remote node via SSH from this workstation${NC}"
echo ""
prompt "Select mode (1/2): " MODE_CHOICE
# Numeric mapping is the interactive form; the words are accepted too.
case "$MODE_CHOICE" in
1|local) EXEC_MODE="local" ;;
2|remote) EXEC_MODE="remote" ;;
esac
fi
if [[ "${EXEC_MODE:-}" == "remote" ]]; then
if [[ -z "$SSH_TARGET" ]]; then
default_ip="192.168.178.10"
case "$ROLE_NAME" in
core) default_ip="192.168.178.20" ;;
worker) default_ip="192.168.178.30" ;;
backup) default_ip="192.168.178.40" ;;
supervisor) default_ip="192.168.178.20" ;;
esac
echo ""
prompt "SSH target (e.g. grid@${default_ip}): " SSH_TARGET
fi
info "Testing SSH connectivity to ${SSH_TARGET}..."
if ! ssh -q "$SSH_TARGET" exit; then
error "SSH connectivity to ${SSH_TARGET} failed. Ensure keys or passwords are correct."
fi
# Pre-flight: Check for rsync on remote target
if ! ssh -q "$SSH_TARGET" "command -v rsync" > /dev/null; then
warning "rsync is missing on ${SSH_TARGET}."
prompt "Would you like to try installing rsync automatically via sudo? (y/n): " INSTALL_RSYNC
if [[ "$INSTALL_RSYNC" == "y" ]]; then
info "Attempting to install rsync on ${SSH_TARGET} (may prompt for password)..."
# Use ssh -t to allow interactive sudo password prompt
ssh -t "$SSH_TARGET" "sudo apt-get update && sudo apt-get install -y rsync"
if ssh -q "$SSH_TARGET" "command -v rsync" > /dev/null; then
success "rsync installed successfully."
else
error "rsync installation failed. Please install it manually (e.g., 'sudo apt install rsync') and try again."
fi
else
error "rsync is required for remote deployment. Please install it manually and try again."
fi
fi
success "Connected to ${SSH_TARGET} (rsync verified)."
else
EXEC_MODE="local"
success "On-Node mode: this machine is the target."
fi
# Detect existing state
inspect_state "$EXEC_MODE" "$SSH_TARGET"
echo ""
if [[ "$CURRENT_ROLE" != "none" ]]; then
success "Detected existing node: Role [${BOLD}${CURRENT_ROLE}${NC}], Version [${CURRENT_VERSION}]"
else
info "No state registry found. Scanning target for existing services..."
echo ""
discover_services "$EXEC_MODE" "$SSH_TARGET"
# If services were discovered, suggest bootstrapping
if [[ ${#DISCOVERED_SERVICES[@]} -gt 0 ]]; then
echo -e " ${CYAN}💡${NC} Services are already running, but this node is ${BOLD}not registered${NC}"
echo -e " with fusionAIze Grid. You can:"
echo ""
echo -e " ${BOLD}a)${NC} ${GREEN}Adopt${NC} ${DIM}— Register this node as ${BOLD}${ROLE_NAME}${NC}${DIM} (keeps everything as-is)${NC}"
echo -e " ${BOLD}b)${NC} Continue ${DIM}— Proceed to action selection without registering${NC}"
echo ""
prompt "Adopt or continue? (a/b): " ADOPT_CHOICE
if [[ "${ADOPT_CHOICE:-a}" =~ ^[Aa]$ ]]; then
write_state "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME"
CURRENT_ROLE="$ROLE_NAME"
echo ""
success "Node adopted as ${BOLD}${ROLE_NAME}${NC}. Future runs will recognize this node."
fi
fi
fi
# ==============================================================================
# BOOTSTRAP MODE: Register an existing node without running install scripts
# ==============================================================================
if [[ "$BOOTSTRAP_MODE" == "true" ]]; then
echo ""
write_state "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME"
echo ""
success "Node bootstrapped as [${BOLD}${ROLE_NAME}${NC}]. Future runs will detect this role automatically."
exit 0
fi
# If the target already has a different role, warn
if [[ "$CURRENT_ROLE" != "none" && "$CURRENT_ROLE" != "$ROLE_NAME" ]]; then
echo ""
warning "This target is already registered as ${BOLD}${CURRENT_ROLE}${NC}, but you selected ${BOLD}${ROLE_NAME}${NC}."
if [[ "$AUTO_YES" == "true" ]]; then
ROLE_NAME="$CURRENT_ROLE"
ROLE_DIR="$(resolve_role_dir "$ROLE_NAME")"
info "Keeping existing role: ${ROLE_NAME} (--yes: non-destructive auto-confirm)"
else
prompt "Change this node's role? (y/N): " CHANGE_ROLE
if [[ ! "${CHANGE_ROLE:-N}" =~ ^[Yy]$ ]]; then
ROLE_NAME="$CURRENT_ROLE"
ROLE_DIR="$(resolve_role_dir "$ROLE_NAME")"
info "Keeping existing role: ${ROLE_NAME}"
fi
fi
fi
# ==============================================================================
# STEP 4: ACTION SELECTION
# ==============================================================================
echo ""
divider
echo -e " ${BOLD}Step 4 │ Action${NC}"
divider
if [[ -z "$ACTION_NAME" ]]; then
echo ""
echo -e " What do you want to do with this ${BOLD}${ROLE_NAME}${NC} node?"
echo ""
if [[ "$CURRENT_ROLE" != "none" ]]; then
# Existing registered node — safe actions first, destructive at bottom
echo -e " ${BOLD}1)${NC} ${GREEN}Verify${NC} ${DIM}Read-only: check services, ports, disk (changes nothing)${NC}"
echo -e " ${BOLD}2)${NC} Update ${DIM}System packages only (apt upgrade). Your configs stay untouched${NC}"
echo -e " ${BOLD}3)${NC} Control ${DIM}Start / Stop / Restart managed services${NC}"
if [[ "$ROLE_NAME" == "core" ]]; then
echo -e " ${BOLD}4)${NC} ${MAGENTA}Workbench${NC} ${DIM}AI Tooling — CLIs, Agents, Routers${NC}"
fi
echo ""
if [[ "$ROLE_NAME" == "core" ]]; then
echo -e " ${BOLD}5)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}"
echo -e " ${BOLD}6)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}"
echo -e " ${BOLD}7)${NC} ${CYAN}Backup${NC} ${DIM}Dump Postgres + n8n volume to /var/backups/grid-core-heart${NC}"
else
echo -e " ${BOLD}4)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}"
echo -e " ${BOLD}5)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}"
fi
echo ""
echo -e " ${BOLD}h)${NC} ${BOLD}Help / Connect${NC} ${DIM}How to access services from your workstation${NC}"
echo -e " ${BOLD}s)${NC} Switch node ${DIM}Go back to node selection (Step 2)${NC}"
echo -e " ${BOLD}q)${NC} Quit"
echo ""
echo -e " ${DIM}Tip: Start with Verify to see the current state of this node.${NC}"
echo ""
if [[ "$ROLE_NAME" == "core" ]]; then
prompt "Select action (1-7 / h / s / q): " ACTION_CHOICE
else
prompt "Select action (1-5 / h / s / q): " ACTION_CHOICE
fi
case "$ACTION_CHOICE" in
1) ACTION_NAME="verify" ;;
2) ACTION_NAME="update" ;;
3) ACTION_NAME="control" ;;
4) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="workbench"; else ACTION_NAME="install"; fi ;;
5) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="install"; else ACTION_NAME="uninstall"; fi ;;
6) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="uninstall"; else warning "Invalid choice. Please enter 1-5, h, s, or q."; fi ;;
7) if [[ "$ROLE_NAME" == "core" ]]; then ACTION_NAME="backup"; else warning "Invalid choice. Please enter 1-5, h, s, or q."; fi ;;
[Hh]) _show_help "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" ; exec bash "$0" ;;
[Ss]) exec bash "$0" ;;
[Qq]|"") _quit ;;
*) warning "Invalid choice." ;;
esac
else
# Fresh target — install is the natural first action
echo -e " ${BOLD}1)${NC} Install ${DIM}Set up this node with the ${ROLE_NAME} role${NC}"
echo -e " ${BOLD}2)${NC} Verify ${DIM}Check connectivity and environment (read-only)${NC}"
echo ""
echo -e " ${BOLD}s)${NC} Switch node ${DIM}Go back to node selection (Step 2)${NC}"
echo -e " ${BOLD}q)${NC} Quit"
echo ""
echo -e " ${DIM}Tip: Start with Verify to check connectivity first.${NC}"
echo ""
prompt "Select action (1-2 / s / q): " ACTION_CHOICE
case "$ACTION_CHOICE" in
1) ACTION_NAME="install" ;;
2) ACTION_NAME="verify" ;;
[Ss]) exec bash "$0" ;;
[Qq]|"") _quit ;;
*) warning "Invalid choice. Please enter 1-2, s, or q." ;;
esac
fi
fi
# ==============================================================================
# STEP 5: SAFETY & CONFIRMATION
# ==============================================================================
if [[ "$ACTION_NAME" == "install" && "$CURRENT_ROLE" != "none" ]]; then
echo ""
divider
echo -e " ${BOLD}Step 5 │ Safety Check${NC}"
divider
echo ""
warning "This target already has a ${BOLD}${CURRENT_ROLE}${NC} installation."
warning "A fresh install may overwrite critical services."
if [[ "$FORCE" == "false" ]]; then
prompt "Type 'overwrite' to confirm destructive reinstall: " CONFIRM_OVERWRITE
if [[ "${CONFIRM_OVERWRITE:-}" != "overwrite" ]]; then
error "Overwrite not confirmed. Aborting."
fi
else
info "--force flag detected. Bypassing overwrite confirmation."
fi
fi
# Uninstall safety gate — warn before any data is touched
if [[ "$ACTION_NAME" == "uninstall" ]]; then
echo ""
divider
echo -e " ${BOLD}Step 5 │ Uninstall Safety Check${NC}"
divider
echo ""
warning "You are about to UNINSTALL the ${BOLD}${ROLE_NAME}${NC} node."
if [[ "$ROLE_NAME" == "core" ]]; then
echo ""
echo -e " ${RED}✖${NC} ${BOLD}ALL Docker volumes will be permanently deleted:${NC}"
echo -e " ${DIM}postgres_data${NC} — n8n workflows, credentials, execution history"
echo -e " ${DIM}n8n_data${NC} — n8n settings, installed nodes, encryption key"
echo -e " ${DIM}redis_data${NC} — queue state"
echo ""
echo -e " ${YELLOW}Tip: Run${NC} ${BOLD}Backup${NC} ${YELLOW}first to preserve your data.${NC}"
fi
echo ""
prompt "Type 'delete-all-data' to confirm permanent data loss: " CONFIRM_UNINSTALL
if [[ "${CONFIRM_UNINSTALL:-}" != "delete-all-data" ]]; then
error "Uninstall not confirmed. Aborting."
exit 1
fi
fi
# Hardware Checks (only for fresh installs)
if [[ "$ACTION_NAME" == "install" ]]; then
check_hardware "$ROLE_NAME" "$EXEC_MODE" "$SSH_TARGET"
fi
# --- Role-Specific Options ---
if [[ "$ROLE_NAME" == "core" && "$ACTION_NAME" == "install" ]]; then
if [[ -z "$VNC_CHOICE" ]]; then
echo ""
if [[ "$AUTO_YES" == "true" ]]; then
VNC_CHOICE="N"
info "Disabling VNC (--yes: non-destructive auto-confirm)"
else
prompt "Enable VNC GUI for the AI Workbench? (y/N): " VNC_CHOICE
fi
fi
fi
# ==============================================================================
# TOPOLOGY WRITING
# ==============================================================================
echo "# Generated by Universal Orchestrator v2.0" > "$TOPOLOGY_FILE"
echo "SERVER_NAME=${ROLE_NAME}-node" >> "$TOPOLOGY_FILE"
echo "ROLE=$ROLE_NAME" >> "$TOPOLOGY_FILE"
if [[ "$ROLE_NAME" == "core" ]]; then
if [[ "${VNC_CHOICE:-}" =~ ^[Yy]$ ]]; then
echo "ENABLE_VNC=true" >> "$TOPOLOGY_FILE"
else
echo "ENABLE_VNC=false" >> "$TOPOLOGY_FILE"
fi
fi
# ==============================================================================
# DEPLOYMENT PHASE — with post-action loop
# ==============================================================================
_run_action() {
local mode=$1
local ssh_target=$2
local role=$3
local action=$4
local role_dir=$5
local cmd_arg="${COMPONENT_NAME:-all}"
# Workbench — map to 'control workbench', skip the interactive control submenu
if [ "$action" = "workbench" ]; then
action="control"
cmd_arg="workbench"
fi
# Handle control sub-commands interactively (skipped for workbench above)
if [ "$action" = "control" ] && [ "$cmd_arg" != "workbench" ]; then
echo ""
echo -e " ${BOLD}Control Interface${NC} — Select command for ${BOLD}${role}${NC}:"
echo ""
echo -e " ${BOLD}1)${NC} ${GREEN}Start${NC}"
echo -e " ${BOLD}2)${NC} ${RED}Stop${NC}"
echo -e " ${BOLD}3)${NC} ${YELLOW}Restart${NC}"
echo -e " ${BOLD}4)${NC} Status"
echo -e " ${BOLD}5)${NC} Reload"
echo -e " ${BOLD}6)${NC} Logs ${DIM}last 60 lines — all services${NC}"
echo ""
echo -e " ${BOLD}a)${NC} Action selection ${DIM}(Back to Step 4)${NC}"
echo ""
prompt "Select command (1-6 / a / s / q): " CONTROL_CHOICE
case "$CONTROL_CHOICE" in
1) cmd_arg="start" ;;
2) cmd_arg="stop" ;;
3) cmd_arg="restart" ;;
4) cmd_arg="status" ;;
5) cmd_arg="reload" ;;
6) cmd_arg="logs" ;;
[Aa]) return 0 ;;
[Ss]) exec bash "$0" ;;
[Qq]|"") _quit ;;
*) warning "Invalid choice. Defaulting to 'status'." ; cmd_arg="status" ;;
esac
fi
echo ""
divider
echo -e " ${BOLD}Deploying${NC} ${role} → ${action} (${cmd_arg})"
divider
echo ""
if [ "$mode" = "local" ]; then
info "Initiating Local Pipeline [${action}] for ${role}..."
if [ ! -d "$role_dir" ]; then
warning "Directory $role_dir not populated in repo. Skipping module execution."
else
TARGET_SCRIPT="${role_dir}/scripts/${action}.sh"
if [ -f "$TARGET_SCRIPT" ]; then
echo ""
bash "$TARGET_SCRIPT" "$cmd_arg"
else
warning "Script not found: ${TARGET_SCRIPT}. Action skipped."
fi
fi
if [ "$action" = "install" ]; then
write_state "$mode" "$ssh_target" "$role"
fi
elif [ "$mode" = "remote" ]; then
info "Initiating Remote Pipeline [${action}] to ${ssh_target}..."
ssh -q "$ssh_target" "mkdir -p /tmp/grid-install"
info "Transferring payload to target..."
rsync -az --exclude='.git' --exclude='node_modules' "${REPO_ROOT}/" "$ssh_target:/tmp/grid-install/" > /dev/null
scp "${TOPOLOGY_FILE}" "${ssh_target}:/tmp/grid-install/" > /dev/null
info "Executing remote [${action}] payload..."
echo ""
# Calculate relative path from REPO_ROOT to target script
REL_ROLE_DIR="${role_dir#$REPO_ROOT/}"
TARGET_SCRIPT="${REL_ROLE_DIR}/scripts/${action}.sh"
ssh -t "$ssh_target" "cd /tmp/grid-install || exit 1; bash \"$TARGET_SCRIPT\" \"$cmd_arg\""
if [ "$action" = "install" ]; then
write_state "$mode" "$ssh_target" "$role"
fi
fi
}
# Show tailored connection help based on role and target
_show_help() {
local mode=$1
local ssh_target=$2
local role=$3
local ip="localhost"
if [[ "$mode" == "remote" ]]; then
# Extract IP from user@ip
[[ "$ssh_target" == *"@"* ]] && ip=$(echo "$ssh_target" | cut -d'@' -f2) || ip="$ssh_target"
fi
echo ""
divider
echo -e " ${BOLD}Access & Connection Guide │ ${role}${NC}"
divider
echo ""
case "$role" in
core)
echo -e " ${BOLD}n8n Automation Workbench:${NC}"
# Check edge status
local edge_status="○"
[[ -f "$LOCAL_REGISTRY/edge.state" ]] && edge_status="✔"
if [[ "$edge_status" == "✔" ]]; then
echo -e " Public URL : https://n8n.your-domain.com ${DIM}(via grid-edge Caddy Proxy)${NC}"
echo -e " ${DIM}(Make sure your edge DNS is configured to point here)${NC}"
else
echo -e " Local URL : http://${ip}:5678"
if [[ "$mode" == "remote" ]]; then
echo -e " SSH Tunnel : ${CYAN}ssh -L 5678:localhost:5678 ${ssh_target}${NC}"
echo -e " ${DIM}(Run this on your workstation to access n8n in your local browser)${NC}"
fi
fi
echo ""
echo -e " ${BOLD}OpenClaw Gateway:${NC}"
echo -e " Port : 18789 (Internal)"
echo -e " Status : Check via 'Verify' or 'Control'"
;;
edge)
echo -e " ${BOLD}Pi-hole Admin:${NC}"
echo -e " URL : http://${ip}/admin"
echo ""
echo -e " ${BOLD}Caddy Ingress:${NC}"
echo -e " Ports : 80 / 443"
echo -e " Config : /etc/caddy/Caddyfile"
;;
worker)
echo -e " ${BOLD}Ollama API:${NC}"
echo -e " Local URL : http://${ip}:11434"
if [[ "$mode" == "remote" ]]; then
echo -e " SSH Tunnel : ${CYAN}ssh -L 11434:localhost:11434 ${ssh_target}${NC}"
fi
;;
supervisor)
echo -e " ${BOLD}Supervisor Control API:${NC}"
echo -e " URL : http://${ip}:5000"
echo -e " Endpoints : /v1/supervisor/register, /v1/supervisor/poll,"
echo -e " /v1/supervisor/recovery_history/{job_id}, /v1/supervisor/max_workers"
if [[ "$mode" == "remote" ]]; then
echo -e " SSH Tunnel : ${CYAN}ssh -L 5000:localhost:5000 ${ssh_target}${NC}"
fi
;;
*)
echo -e " No specific connection help for this role yet."
;;
esac
echo ""
echo -e " ${BOLD}Documentation:${NC}"
echo -e " Runbooks : docs/runbooks/"
echo -e " Architecture: docs/architecture.md"
echo ""
divider
}
# Destructive actions exit immediately; safe actions loop back
_is_loopable() {
[[ "$1" == "verify" || "$1" == "update" || "$1" == "control" || "$1" == "workbench" || "$1" == "backup" ]]
}
# Execute the requested action
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "$ACTION_NAME" "$ROLE_DIR"
if _is_loopable "$ACTION_NAME"; then
while true; do
echo ""
divider
inspect_state "$EXEC_MODE" "$SSH_TARGET"
if [[ "$CURRENT_ROLE" != "none" ]]; then
echo -e " ${GREEN}✔${NC} ${BOLD}${ROLE_NAME}${NC} is registered ${DIM}(version: ${CURRENT_VERSION})${NC}"
else
echo -e " ${DIM}○${NC} ${BOLD}${ROLE_NAME}${NC} is ${YELLOW}not registered${NC} — run Install or Adopt to register"
fi
divider
echo ""
echo -e " ${BOLD}What next?${NC}"
echo ""
echo -e " ${BOLD}1)${NC} ${GREEN}Verify${NC} ${DIM}Read-only: check services, ports, disk${NC}"
echo -e " ${BOLD}2)${NC} Update ${DIM}System packages only (apt upgrade). Configs stay untouched${NC}"
echo -e " ${BOLD}3)${NC} Control ${DIM}Start / Stop / Restart managed services${NC}"
if [[ "$ROLE_NAME" == "core" ]]; then
echo -e " ${BOLD}4)${NC} ${MAGENTA}Workbench${NC} ${DIM}AI Tooling — CLIs, Agents, Routers${NC}"
fi
echo ""
if [[ "$ROLE_NAME" == "core" ]]; then
echo -e " ${BOLD}5)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}"
echo -e " ${BOLD}6)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}"
echo -e " ${BOLD}7)${NC} ${CYAN}Backup${NC} ${DIM}Dump Postgres + n8n volume to /var/backups/grid-core-heart${NC}"
else
echo -e " ${BOLD}4)${NC} ${YELLOW}Reinstall${NC} ${DIM}⚠ Wipe and re-provision from scratch${NC}"
echo -e " ${BOLD}5)${NC} ${YELLOW}Uninstall${NC} ${DIM}⚠ Remove this node role entirely${NC}"
fi
echo ""
echo -e " ${BOLD}h)${NC} ${BOLD}Help / Connect${NC}"
echo -e " ${BOLD}s)${NC} Switch node ${DIM}Go back to node selection (Step 2)${NC}"
echo -e " ${BOLD}q)${NC} Quit"
echo ""
if [[ "$ROLE_NAME" == "core" ]]; then
prompt "Select action (1-7 / h / s / q): " NEXT_CHOICE
else
prompt "Select action (1-5 / h / s / q): " NEXT_CHOICE
fi
case "$NEXT_CHOICE" in
1) _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "verify" "$ROLE_DIR" || true ;;
2) _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "update" "$ROLE_DIR" || true ;;
3) _run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "control" "$ROLE_DIR" || true ;;
4)
if [[ "$ROLE_NAME" == "core" ]]; then
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "workbench" "$ROLE_DIR" || true
else
warning "This will wipe and re-provision ${BOLD}${ROLE_NAME}${NC}. Are you sure?"
prompt "Type YES to confirm: " CONFIRM
if [[ "$CONFIRM" == "YES" ]]; then
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "install" "$ROLE_DIR"
break
else
info "Reinstall cancelled."
fi
fi
;;
5)
if [[ "$ROLE_NAME" == "core" ]]; then
warning "This will wipe and re-provision ${BOLD}${ROLE_NAME}${NC}. Are you sure?"
prompt "Type YES to confirm: " CONFIRM
if [[ "$CONFIRM" == "YES" ]]; then
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "install" "$ROLE_DIR"
break
else
info "Reinstall cancelled."
fi
else
warning "This will remove the ${BOLD}${ROLE_NAME}${NC} role. Are you sure?"
prompt "Type YES to confirm: " CONFIRM
if [[ "$CONFIRM" == "YES" ]]; then
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "uninstall" "$ROLE_DIR"
break
else
info "Uninstall cancelled."
fi
fi
;;
6)
if [[ "$ROLE_NAME" == "core" ]]; then
echo ""
warning "DESTRUCTIVE: This will permanently delete ALL Docker volumes!"
echo -e " ${RED}✖${NC} ${DIM}postgres_data${NC} — n8n workflows, credentials, execution history"
echo -e " ${RED}✖${NC} ${DIM}n8n_data${NC} — settings, installed nodes, encryption key"
echo -e " ${RED}✖${NC} ${DIM}redis_data${NC} — queue state"
echo ""
echo -e " ${YELLOW}Tip: Run${NC} ${BOLD}Backup (7)${NC} ${YELLOW}first to preserve your data.${NC}"
echo ""
prompt "Type 'delete-all-data' to confirm permanent data loss: " CONFIRM
if [[ "$CONFIRM" == "delete-all-data" ]]; then
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "uninstall" "$ROLE_DIR"
break
else
info "Uninstall cancelled."
fi
else
warning "Invalid choice."
fi
;;
7)
if [[ "$ROLE_NAME" == "core" ]]; then
_run_action "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" "backup" "$ROLE_DIR" || true
else
warning "Invalid choice."
fi
;;
[Hh]) _show_help "$EXEC_MODE" "$SSH_TARGET" "$ROLE_NAME" ;;
[Ss]) exec bash "$0" ;;
[Qq]|"")
break
;;
*) warning "Invalid choice." ;;
esac
done
fi
echo ""
divider
success "fusionAIze Grid Orchestration Complete! 🚀"
divider
echo ""