update_runner.py
Rama main · réplica de lectura del repositorio original.
#!/usr/bin/env python3
"""Download, verify, and atomically install an AgentsServer release."""
from __future__ import annotations
import argparse
from contextlib import contextmanager
import fcntl
import hashlib
import json
import os
import re
import signal
import shutil
import stat
import subprocess
import tarfile
import tempfile
import time
import urllib.request
from urllib.error import HTTPError, URLError
from pathlib import Path
from typing import Any, Callable
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
RELEASE_REPOSITORY = "ZhengyiLuo/AgentsServer"
RELEASE_BASE = f"https://github.com/{RELEASE_REPOSITORY}/releases"
RELEASES_API_URL = f"https://api.github.com/repos/{RELEASE_REPOSITORY}/releases?per_page=100"
RELEASES_PAGE_URL = f"https://github.com/{RELEASE_REPOSITORY}/releases"
MAX_METADATA_BYTES = 1_000_000
MAX_ARCHIVE_BYTES = 200 * 1024 * 1024
SERVER_IDLE_CHECK_TIMEOUT_SECONDS = 10.0
SERVER_STARTUP_READINESS_TIMEOUT_SECONDS = 45.0
SERVER_STARTUP_READINESS_POLL_SECONDS = 1.0
VERSION_PATTERN = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][A-Za-z0-9.-]+)?$")
# The standalone installer allows dependency synchronization to run for up to
# 1,200 seconds. Keep this enclosing budget comfortably above that so the
# updater cannot terminate a healthy installer first.
INSTALLER_TIMEOUT_SECONDS = 1_800
INSTALLER_HEARTBEAT_SECONDS = 10.0
# After install.sh starts there is no trustworthy out-of-process proof that it
# has not activated a candidate. Its TERM handler may therefore be stopping the
# candidate, restoring an arbitrarily large verified Team Hub snapshot, and
# restarting the old release. Never put a finite SIGKILL deadline around that
# recovery transaction. Polling keeps the owning update observable while the
# installer's signal-masked rollback reaches a safe terminal state.
INSTALLER_TERMINATION_POLL_SECONDS = 10.0
INSTALLER_LOG_TAIL_BYTES = 64 * 1024
INSTALLER_LOG_TAIL_LINES = 12
INSTALLER_ERROR_MAX_CHARS = 4_000
INSTALLER_LOG_MAX_BYTES = 1024 * 1024
INSTALLER_ENVIRONMENT_SELECTORS = (
"AGENTSDOCK_AGENT_TOKEN",
"AGENTSDOCK_EXPECTED_SERVICE_CGROUP",
"AGENTSDOCK_MANAGED_UPDATE_ID",
"AGENTSDOCK_PROVIDER_AUTHORITY_FILE",
"AGENTSDOCK_PUBLISH_TOKEN",
"CONDA_PREFIX",
"PYTHONHOME",
"PYTHONPATH",
"UV_CONFIG_FILE",
"UV_NO_PROJECT",
"UV_PROJECT",
"UV_PROJECT_ENVIRONMENT",
"UV_PYTHON",
"UV_WORKING_DIR",
"VIRTUAL_ENV",
"ZENITHBOT_AGENT_TOKEN",
"ZENITHDOCK_AGENT_TOKEN",
)
RELEASE_TRACKS = {"stable", "beta"}
RUNNER_OWNED_ACTIVE_PHASES = {
"starting",
"checking",
"downloading",
"verifying",
"installing",
"restarting",
}
SECURE_PEER_HEALTH_REQUIREMENTS = {
"available": True,
"state_available": True,
"state_error_code": None,
"required": False,
"version": 1,
"control_path": "/api/admin/secure-peers/v1/status",
"proxy_prefix": "/api/team-hub-secure",
}
class ReleaseUnavailableError(RuntimeError):
"""Raised when the repository has not published a signed release yet."""
class UpdateOwnershipLostError(RuntimeError):
"""Raised when a detached updater no longer owns the durable status row."""
def utc_now() -> str:
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def atomic_json(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
temporary.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n")
os.chmod(temporary, 0o600)
os.replace(temporary, path)
@contextmanager
def server_update_status_lock(path: Path):
"""Cross-process lock shared by the server and detached updater."""
path.parent.mkdir(parents=True, exist_ok=True)
lock_path = path.with_name(f".{path.name}.lock")
flags = os.O_RDWR | os.O_CREAT | getattr(os, "O_CLOEXEC", 0)
flags |= getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(lock_path, flags, 0o600)
try:
info = os.fstat(descriptor)
if (
not stat.S_ISREG(info.st_mode)
or info.st_nlink != 1
or info.st_uid != os.getuid()
):
raise PermissionError("server update status lock is unsafe")
os.fchmod(descriptor, 0o600)
fcntl.flock(descriptor, fcntl.LOCK_EX)
yield
finally:
try:
fcntl.flock(descriptor, fcntl.LOCK_UN)
finally:
os.close(descriptor)
def _read_status_unlocked(path: Path) -> dict[str, Any]:
try:
current = json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError, OSError):
return {}
return current if isinstance(current, dict) else {}
def _update_status_unlocked(path: Path, current: dict[str, Any], **changes: Any) -> dict[str, Any]:
current = dict(current)
current.update(changes)
current["updated_at"] = utc_now()
atomic_json(path, current)
return current
def update_status(
path: Path,
*,
expected_update_id: str | None = None,
**changes: Any,
) -> dict[str, Any]:
with server_update_status_lock(path):
current = _read_status_unlocked(path)
if (
expected_update_id is not None
and (
str(current.get("update_id") or "") != expected_update_id
or str(current.get("phase") or "")
not in RUNNER_OWNED_ACTIVE_PHASES
)
):
raise UpdateOwnershipLostError(
"detached updater no longer owns the server update status"
)
return _update_status_unlocked(path, current, **changes)
def trim_installer_log(log_path: Path, limit: int = INSTALLER_LOG_MAX_BYTES) -> None:
"""Retain bounded history between attempts, while no installer is writing."""
try:
with log_path.open("r+b") as log:
log.seek(0, os.SEEK_END)
if log.tell() <= limit:
return
log.seek(-limit, os.SEEK_END)
tail = log.read(limit)
# Do not retain an unclassifiable partial secret line.
tail = tail.partition(b"\n")[2]
log.seek(0)
log.write(tail)
log.truncate()
except FileNotFoundError:
pass
def installer_log_tail(log_path: Path, *, start_offset: int = 0) -> str:
"""Read a bounded diagnostic tail without loading a large install log."""
try:
with log_path.open("rb") as stream:
stream.seek(0, os.SEEK_END)
size = stream.tell()
start = max(start_offset, size - INSTALLER_LOG_TAIL_BYTES)
truncated = start > start_offset
stream.seek(start)
content = stream.read(INSTALLER_LOG_TAIL_BYTES)
except OSError:
return ""
lines = content.decode("utf-8", "replace").splitlines()
if truncated and lines:
# The bounded read may begin inside a secret whose identifying key was
# before the cutoff. Never surface that unclassifiable partial line.
lines = lines[1:]
safe_lines: list[str] = []
for line in lines:
if "AGENTSDOCK_SETUP_RESULT=" in line:
continue
line = re.sub(
r"(?i)(Authorization\s*:\s*Bearer\s+)\S+",
r"\1[REDACTED]",
line,
)
line = re.sub(
r"(?i)(\bBearer\s+)\S+",
r"\1[REDACTED]",
line,
)
line = re.sub(
r"(?i)([\"']?(?:AGENTSDOCK_AGENT_TOKEN|ZENITHDOCK_AGENT_TOKEN|ZENITHBOT_AGENT_TOKEN|AGENTSDOCK_PUBLISH_TOKEN|AGENTSDOCK_PROVIDER_AUTHORITY_FILE|access_token)[\"']?\s*[=:]\s*).*$",
r"\1[REDACTED]",
line,
)
safe_lines.append(line)
tail = "\n".join(safe_lines[-INSTALLER_LOG_TAIL_LINES:])
return tail[-INSTALLER_ERROR_MAX_CHARS:].strip()
def terminate_installer(
process: subprocess.Popen[Any],
*,
on_wait: Callable[[], None] | None = None,
) -> None:
"""Request installer termination and join its protected recovery.
install.sh cannot currently prove to this process whether activation has
begun. Once it has begun, SIGKILL can strand the service stopped or sever a
Team Hub restore before the old release is restarted. TERM is cooperative:
the installer kills ordinary stage workers, then masks further termination
while it completes rollback. Wait without a force-kill deadline.
"""
if process.poll() is not None:
return
if os.name != "nt":
try:
os.killpg(process.pid, signal.SIGTERM)
except OSError:
process.terminate()
else:
process.terminate()
while True:
if on_wait is not None:
try:
on_wait()
except Exception:
# Losing status ownership must not turn a safe join back into
# a detached installer or a force-kill boundary.
pass
try:
process.wait(timeout=INSTALLER_TERMINATION_POLL_SECONDS)
return
except subprocess.TimeoutExpired:
continue
def installer_environment() -> dict[str, str]:
"""Return an installer environment detached from the caller's workspace."""
environment = os.environ.copy()
for name in INSTALLER_ENVIRONMENT_SELECTORS:
environment.pop(name, None)
return environment
def instance_installer_arguments(source: Path) -> list[str]:
"""Never let a named server run an old default-only installer."""
from server_instances import instance_name
name = instance_name(os.environ.get("AGENTS_SERVER_INSTANCE", "default"))
if name == "default":
return [] # Preserve older signed default installers.
helper = source / "server_instances.py"
installer = source / "install.sh"
if (not helper.is_file() or helper.is_symlink()
or "INSTANCE_PROTOCOL = 1" not in helper.read_text()
or '--instance)' not in installer.read_text()):
raise RuntimeError("This release does not support named instances; the existing server was not replaced.")
return ["--instance", name]
def run_installer(
command: list[str],
*,
cwd: Path,
status_path: Path,
log_path: Path,
version: str,
expected_update_id: str | None = None,
managed_update_id: str | None = None,
expected_service_cgroup: str | None = None,
timeout_seconds: float = INSTALLER_TIMEOUT_SECONDS,
heartbeat_seconds: float = INSTALLER_HEARTBEAT_SECONDS,
on_started: Callable[[], None] | None = None,
) -> None:
"""Run the installer with live logging and a durable status heartbeat."""
if expected_update_id is not None:
update_status(
status_path,
expected_update_id=expected_update_id,
heartbeat_at=utc_now(),
)
log_path.parent.mkdir(parents=True, exist_ok=True)
header = (
f"\n--- AgentsServer update {expected_update_id or 'manual'} "
f"to {version} at {utc_now()} ---\n"
).encode()
trim_installer_log(log_path, INSTALLER_LOG_MAX_BYTES - len(header))
started = time.monotonic()
deadline = started + timeout_seconds
environment = installer_environment()
if managed_update_id is not None:
environment["AGENTSDOCK_MANAGED_UPDATE_ID"] = managed_update_id
if expected_service_cgroup is not None:
environment["AGENTSDOCK_EXPECTED_SERVICE_CGROUP"] = expected_service_cgroup
with log_path.open("ab") as log:
os.chmod(log_path, 0o600)
log.write(header)
log.flush()
attempt_start = log.tell()
process = subprocess.Popen(
command,
cwd=cwd,
stdout=log,
stderr=subprocess.STDOUT,
start_new_session=True,
env=environment,
)
if on_started is not None:
on_started()
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
def report_protected_recovery() -> None:
elapsed = max(1, int(time.monotonic() - started))
update_status(
status_path,
expected_update_id=expected_update_id,
phase="installing",
message=(
"Installer timeout reached; waiting for its protected "
f"rollback to finish ({elapsed}s elapsed)."
),
heartbeat_at=utc_now(),
elapsed_seconds=elapsed,
)
terminate_installer(process, on_wait=report_protected_recovery)
log.flush()
tail = installer_log_tail(log_path, start_offset=attempt_start)
trim_installer_log(log_path)
detail = f": {tail}" if tail else ""
raise RuntimeError(
f"installer timed out after {timeout_seconds:g} seconds{detail}"
)
try:
returncode = process.wait(timeout=min(heartbeat_seconds, remaining))
break
except subprocess.TimeoutExpired:
elapsed = max(1, int(time.monotonic() - started))
try:
update_status(
status_path,
expected_update_id=expected_update_id,
phase="installing",
message=f"Installing AgentsServer {version} ({elapsed}s elapsed).",
heartbeat_at=utc_now(),
elapsed_seconds=elapsed,
)
except UpdateOwnershipLostError:
terminate_installer(process)
log.flush()
trim_installer_log(log_path)
raise
log.flush()
tail = installer_log_tail(log_path, start_offset=attempt_start) if returncode else ""
trim_installer_log(log_path)
if returncode != 0:
raise RuntimeError(
f"installer failed ({returncode}): {tail or 'no output; inspect server-update.log'}"
)
def download_bytes(url: str, limit: int, timeout: float = 30.0) -> bytes:
request = urllib.request.Request(url, headers={"User-Agent": "AgentsServer-Updater/1"})
with urllib.request.urlopen(request, timeout=timeout) as response:
declared = int(response.headers.get("Content-Length") or 0)
if declared > limit:
raise RuntimeError(f"download exceeds the {limit}-byte safety limit")
content = response.read(limit + 1)
if len(content) > limit:
raise RuntimeError(f"download exceeds the {limit}-byte safety limit")
return content
def server_health_snapshot(
port: int,
*,
token: str | None = None,
timeout: float = SERVER_IDLE_CHECK_TIMEOUT_SECONDS,
) -> dict[str, Any]:
"""Read one bounded authenticated AgentsServer health document."""
headers = {"User-Agent": "AgentsServer-Updater/1"}
clean_token = str(token or "").strip()
if clean_token:
headers["Authorization"] = f"Bearer {clean_token}"
request = urllib.request.Request(
f"http://127.0.0.1:{int(port)}/api/health",
headers=headers,
)
with urllib.request.urlopen(request, timeout=timeout) as response:
content = response.read(MAX_METADATA_BYTES + 1)
if len(content) > MAX_METADATA_BYTES:
raise RuntimeError("server health response exceeds its safety limit")
try:
health = json.loads(content)
except json.JSONDecodeError as exc:
raise RuntimeError("server health response is invalid JSON") from exc
if not isinstance(health, dict) or health.get("ok") is not True:
raise RuntimeError("server health response is not healthy")
return health
def server_work_snapshot(
port: int,
*,
token: str | None = None,
timeout: float = SERVER_IDLE_CHECK_TIMEOUT_SECONDS,
require_cgroup_safe: bool = False,
require_verified_service_cgroup: bool = False,
expected_server_identity: str | None = None,
) -> tuple[int, int]:
"""Read the live workload immediately before invoking the installer."""
health = server_health_snapshot(port, token=token, timeout=timeout)
if (
expected_server_identity is not None
and health.get("server_identity") != expected_server_identity
):
raise RuntimeError("AgentsServer stable identity changed before restart")
active = health.get("active")
raw_active_count = health.get("active_count")
if isinstance(raw_active_count, bool):
raise RuntimeError("server health response has an invalid active count")
if isinstance(raw_active_count, int):
active_count = max(0, raw_active_count)
elif isinstance(active, list):
active_count = len(active)
else:
raise RuntimeError("server health response is missing its active count")
raw_blocking_queued_count = health.get("update_blocking_queued_count")
if raw_blocking_queued_count is not None:
if (
isinstance(raw_blocking_queued_count, bool)
or not isinstance(raw_blocking_queued_count, int)
or raw_blocking_queued_count < 0
):
raise RuntimeError(
"server health response has an invalid update-blocking queued count"
)
queued_count = raw_blocking_queued_count
else:
# Older servers did not distinguish durable preserved messages from
# volatile queue state. Retain their fail-closed behavior.
queued = health.get("queued")
if not isinstance(queued, dict):
raise RuntimeError("server health response is missing its queued turns")
queued_count = 0
for value in queued.values():
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise RuntimeError("server health response has an invalid queued count")
queued_count += value
if require_cgroup_safe and not active_count and not queued_count:
cgroup = health.get("update_service_cgroup")
if not isinstance(cgroup, dict):
raise RuntimeError(
"server health response is missing its service-cgroup admission proof"
)
raw_count = cgroup.get("unknown_descendant_count")
if (
cgroup.get("safe") is not True
or isinstance(raw_count, bool)
or not isinstance(raw_count, int)
or raw_count != 0
):
raise RuntimeError(
"AgentsServer has an unverified or nonempty service cgroup"
)
if (
require_verified_service_cgroup
and cgroup.get("inspection") != "verified"
):
raise RuntimeError(
"AgentsServer did not return a verified systemd service-cgroup proof"
)
return active_count, queued_count
def assert_post_update_identity(
port: int,
*,
token: str | None,
expected_server_identity: str,
expected_team_hub_id: str | None = None,
expected_team_hub_transport: str | None = None,
expected_team_hub_url: str | None = None,
expected_team_hub_direct_ip_url: str | None = None,
) -> None:
"""Fence a replacement by stable server and managed Hub identities."""
health = server_health_snapshot(port, token=token)
if str(health.get("server_identity") or "") != expected_server_identity:
raise RuntimeError("updated AgentsServer stable identity does not match")
capabilities = health.get("capabilities")
secure_peer_capability = (
capabilities.get("secure_peer_v1")
if isinstance(capabilities, dict)
else None
)
if not isinstance(secure_peer_capability, dict) or any(
secure_peer_capability.get(name) != value
for name, value in SECURE_PEER_HEALTH_REQUIREMENTS.items()
):
raise RuntimeError("updated AgentsServer secure-peer state is unavailable")
if expected_team_hub_id is None:
if (
expected_team_hub_transport is not None
or expected_team_hub_url is not None
or expected_team_hub_direct_ip_url is not None
):
raise RuntimeError("managed Team Hub transport has no bound Hub identity")
return
capability = (
capabilities.get("team_hub_v1")
if isinstance(capabilities, dict)
else None
)
expected_capability = {
"available": True,
"designated_host": True,
"version": 1,
"base_path": "/api/team-hub",
"hub_id": expected_team_hub_id,
"host_server_identity": expected_server_identity,
}
if not isinstance(capability, dict) or any(
capability.get(name) != value
for name, value in expected_capability.items()
):
raise RuntimeError("updated AgentsServer lost or changed its Team Hub identity")
actual_transport = capability.get("transport")
actual_hub_url = capability.get("hub_url")
if expected_team_hub_transport is None:
if actual_transport not in {None, "loopback"} or actual_hub_url is not None:
raise RuntimeError("updated AgentsServer changed its legacy Team Hub transport")
elif (
actual_transport != expected_team_hub_transport
or actual_hub_url != expected_team_hub_url
):
raise RuntimeError("updated AgentsServer changed its Team Hub transport")
if expected_team_hub_direct_ip_url is not None:
expected_routes = [
{
"transport": expected_team_hub_transport or "loopback",
"hub_url": expected_team_hub_url,
}
]
if (
expected_team_hub_direct_ip_url
and expected_team_hub_transport != "direct_ip"
):
expected_routes.append(
{
"transport": "direct_ip",
"hub_url": expected_team_hub_direct_ip_url,
}
)
if capability.get("routes") != expected_routes:
raise RuntimeError("updated AgentsServer changed its Team Hub routes")
def assert_repaired_team_hub_identity(
port: int,
*,
token: str | None,
expected_server_identity: str,
expected_team_hub_transport: str,
expected_team_hub_url: str | None,
expected_team_hub_direct_ip_url: str,
) -> str:
"""Verify that an unavailable managed host was repaired in place."""
health = server_health_snapshot(port, token=token)
capabilities = health.get("capabilities")
capability = (
capabilities.get("team_hub_v1")
if isinstance(capabilities, dict)
else None
)
required = {
"available": True,
"designated_host": True,
"version": 1,
"base_path": "/api/team-hub",
"host_server_identity": expected_server_identity,
"transport": expected_team_hub_transport,
"hub_url": expected_team_hub_url,
}
if not isinstance(capability, dict) or any(
capability.get(name) != value for name, value in required.items()
):
raise RuntimeError("updated AgentsServer did not repair its Team Hub host")
hub_id = capability.get("hub_id")
if not isinstance(hub_id, str) or re.fullmatch(
r"[A-Za-z0-9_.:-]{8,240}", hub_id
) is None:
raise RuntimeError("repaired Team Hub identity is invalid")
expected_routes = [
{
"transport": expected_team_hub_transport,
"hub_url": expected_team_hub_url,
}
]
if (
expected_team_hub_direct_ip_url
and expected_team_hub_transport != "direct_ip"
):
expected_routes.append(
{
"transport": "direct_ip",
"hub_url": expected_team_hub_direct_ip_url,
}
)
if capability.get("routes") != expected_routes:
raise RuntimeError("repaired Team Hub routes changed")
return hub_id
def assert_server_idle(
port: int,
*,
token: str | None = None,
require_verified_service_cgroup: bool = False,
expected_server_identity: str | None = None,
timeout: float = SERVER_IDLE_CHECK_TIMEOUT_SECONDS,
) -> None:
"""Fail closed if work appeared after the update was accepted."""
try:
active_count, queued_count = server_work_snapshot(
port,
token=token,
require_cgroup_safe=True,
require_verified_service_cgroup=require_verified_service_cgroup,
expected_server_identity=expected_server_identity,
timeout=timeout,
)
except Exception as exc:
raise RuntimeError(
f"could not verify that AgentsServer is idle before restart: {exc}"
) from exc
if active_count or queued_count:
parts: list[str] = []
if active_count:
parts.append(
f"{active_count} active agent run{'s' if active_count != 1 else ''}"
)
if queued_count:
parts.append(
f"{queued_count} queued turn{'s' if queued_count != 1 else ''}"
)
raise RuntimeError(
"server became busy before restart: "
+ " and ".join(parts)
+ "; retry the update after work finishes"
)
def transient_server_readiness_error(error: BaseException | None) -> bool:
"""Recognize unavailable transport, never failed authentication or proof."""
if isinstance(error, HTTPError):
return error.code == 503
if isinstance(error, URLError):
error = error.reason
return isinstance(error, (
ConnectionRefusedError,
ConnectionResetError,
ConnectionAbortedError,
TimeoutError,
))
def wait_for_server_idle(
port: int,
*,
status_path: Path,
expected_update_id: str,
expected_server_identity: str,
token: str | None = None,
require_verified_service_cgroup: bool = False,
timeout_seconds: float = SERVER_STARTUP_READINESS_TIMEOUT_SECONDS,
) -> None:
"""Allow a forced-update startup to finish opening its native listener.
The replacement process advances its reserved update during lifespan,
before HTTP startup completes. Keep the exact update alive while transport
becomes ready, but never retry a response that fails the idle/safety proof.
"""
deadline = time.monotonic() + timeout_seconds
last_error: RuntimeError | None = None
while True:
# This CAS also stops a cancelled/finalized or superseded updater.
update_status(
status_path,
expected_update_id=expected_update_id,
heartbeat_at=utc_now(),
message="Waiting for AgentsServer startup before checking installation safety.",
)
remaining = deadline - time.monotonic()
if remaining <= 0:
raise RuntimeError(
f"AgentsServer startup readiness timed out after {timeout_seconds:g} seconds"
) from last_error
try:
assert_server_idle(
port,
token=token,
require_verified_service_cgroup=require_verified_service_cgroup,
expected_server_identity=expected_server_identity,
timeout=min(SERVER_IDLE_CHECK_TIMEOUT_SECONDS, remaining),
)
except RuntimeError as exc:
if not transient_server_readiness_error(exc.__cause__):
raise
last_error = exc
remaining = deadline - time.monotonic()
if remaining > 0:
time.sleep(min(SERVER_STARTUP_READINESS_POLL_SECONDS, remaining))
else:
# Ownership can change while the health request is in flight.
update_status(
status_path,
expected_update_id=expected_update_id,
heartbeat_at=utc_now(),
message="AgentsServer startup, identity, and idle checks passed.",
)
return
def consume_auth_token_file(path: str | None) -> str:
"""Read and immediately remove the updater's one-time health credential."""
clean_path = str(path or "").strip()
if not clean_path:
return ""
token_path = Path(clean_path).expanduser().resolve()
try:
value = json.loads(token_path.read_text())
token = str(value.get("token") or "") if isinstance(value, dict) else ""
finally:
try:
token_path.unlink()
except FileNotFoundError:
pass
if not token:
raise RuntimeError("server update health credential is empty")
return token
def clear_team_hub_maintenance(args: argparse.Namespace) -> bool:
"""Clear the exact durable update fence without opening the Hub DB."""
hub_id = str(getattr(args, "expected_team_hub_id", "") or "").strip()
snapshot = str(getattr(args, "team_hub_snapshot", "") or "").strip()
data_dir = str(getattr(args, "team_hub_data_dir", "") or "").strip()
host_identity = str(
getattr(args, "expected_server_identity", "") or ""
).strip()
operation_id = str(getattr(args, "update_id", "") or "").strip()
if not any((hub_id, snapshot, data_dir)):
return False
if not all((hub_id, snapshot, data_dir, host_identity, operation_id)):
raise RuntimeError("managed Team Hub maintenance arguments are incomplete")
from agentsdock_team_hub.store import HubStore
return HubStore.clear_maintenance_fence_control(
Path(data_dir),
expected_hub_id=hub_id,
expected_host_identity=host_identity,
expected_reason="server-update",
expected_operation_id=operation_id,
expected_snapshot=Path(snapshot),
)
def team_hub_maintenance_fence_present(args: argparse.Namespace) -> bool:
"""Return whether this exact update still has an on-disk Hub fence."""
hub_id = str(getattr(args, "expected_team_hub_id", "") or "").strip()
snapshot = str(getattr(args, "team_hub_snapshot", "") or "").strip()
data_dir = str(getattr(args, "team_hub_data_dir", "") or "").strip()
host_identity = str(
getattr(args, "expected_server_identity", "") or ""
).strip()
operation_id = str(getattr(args, "update_id", "") or "").strip()
if not any((hub_id, snapshot, data_dir)):
return False
if not all((hub_id, snapshot, data_dir, host_identity, operation_id)):
raise RuntimeError("managed Team Hub maintenance arguments are incomplete")
from agentsdock_team_hub.store import HubStore
return HubStore.maintenance_fence_matches_control(
Path(data_dir),
expected_hub_id=hub_id,
expected_host_identity=host_identity,
expected_reason="server-update",
expected_operation_id=operation_id,
expected_snapshot=Path(snapshot),
)
def version_is_prerelease(version: str) -> bool:
return "-" in version.split("+", 1)[0]
def release_track(version: str) -> str:
return "beta" if version_is_prerelease(version) else "stable"
def normalized_release_track(track: str | None) -> str:
value = str(track or "stable").strip().lower()
if value not in RELEASE_TRACKS:
raise ValueError(f"invalid release track: {track}")
return value
def version_key(version: str) -> tuple[Any, ...]:
"""Return a SemVer-compatible key for a trusted release version."""
if not VERSION_PATTERN.fullmatch(version):
raise ValueError(f"invalid release version: {version}")
without_build = version.split("+", 1)[0]
core, separator, prerelease = without_build.partition("-")
major, minor, patch = (int(part) for part in core.split("."))
identifiers: tuple[tuple[int, int | str], ...] = ()
if separator:
identifiers = tuple(
(0, int(part)) if part.isdigit() else (1, part)
for part in prerelease.split(".")
)
return major, minor, patch, 0 if separator else 1, identifiers
def release_manifest_url(version: str) -> str:
return f"{RELEASE_BASE}/download/v{version}/agents-server-manifest.json"
def release_signature_url(version: str) -> str:
return f"{RELEASE_BASE}/download/v{version}/agents-server-manifest.sig"
def release_candidates(releases: Any, track: str = "stable") -> list[str]:
track = normalized_release_track(track)
if not isinstance(releases, list):
raise RuntimeError("GitHub releases response must be a JSON array")
candidates: set[str] = set()
for release in releases:
if not isinstance(release, dict) or release.get("draft") is True:
continue
tag = str(release.get("tag_name") or "")
if not tag.startswith("v"):
continue
version = tag[1:]
if not VERSION_PATTERN.fullmatch(version) or release_track(version) != track:
continue
declared_prerelease = release.get("prerelease")
if declared_prerelease is not None and bool(declared_prerelease) != version_is_prerelease(version):
continue
candidates.add(version)
return sorted(candidates, key=version_key, reverse=True)
def stable_release_candidates(releases: Any) -> list[str]:
"""Backward-compatible stable release discovery."""
return release_candidates(releases, "stable")
def release_versions_from_html(content: bytes, track: str = "stable") -> set[str]:
track = normalized_release_track(track)
text = content.decode("utf-8", "replace")
prefix = f"/{RELEASE_REPOSITORY}/releases/tag/v"
return {
match.group(1)
for match in re.finditer(re.escape(prefix) + r"([^\"'<>/?#]+)", text)
if VERSION_PATTERN.fullmatch(match.group(1)) and release_track(match.group(1)) == track
}
def release_candidates_from_public_pages(track: str = "stable", max_pages: int = 20) -> list[str]:
track = normalized_release_track(track)
versions: set[str] = set()
for page in range(1, max_pages + 1):
url = RELEASES_PAGE_URL if page == 1 else f"{RELEASES_PAGE_URL}?page={page}"
content = download_bytes(url, MAX_METADATA_BYTES)
versions.update(release_versions_from_html(content, track))
next_page = f"{RELEASES_PAGE_URL.removeprefix('https://github.com')}?page={page + 1}"
if next_page not in content.decode("utf-8", "replace"):
break
return sorted(versions, key=version_key, reverse=True)
def stable_release_candidates_from_public_pages(max_pages: int = 20) -> list[str]:
"""Backward-compatible stable release discovery."""
return release_candidates_from_public_pages("stable", max_pages)
def verify_manifest(
manifest_bytes: bytes,
signature: bytes,
public_key_path: Path,
*,
expected_version: str | None = None,
track: str = "stable",
) -> dict[str, Any]:
track = normalized_release_track(track)
key = serialization.load_pem_public_key(public_key_path.read_bytes())
if not isinstance(key, Ed25519PublicKey):
raise RuntimeError("release public key is not an Ed25519 key")
key.verify(signature, manifest_bytes)
manifest = json.loads(manifest_bytes)
if not isinstance(manifest, dict):
raise RuntimeError("release manifest must be a JSON object")
version = str(manifest.get("version") or "")
if not VERSION_PATTERN.fullmatch(version):
raise RuntimeError("release manifest contains an invalid version")
actual_track = release_track(version)
if actual_track != track:
raise RuntimeError(f"release manifest is not on the requested {track} track")
if expected_version is not None and version != expected_version:
raise RuntimeError("release manifest version does not match its immutable release tag")
expected_prerelease = actual_track == "beta"
if manifest.get("prerelease") not in {None, expected_prerelease}:
raise RuntimeError("release manifest prerelease metadata is inconsistent")
if manifest.get("track") not in {None, actual_track}:
raise RuntimeError("release manifest track metadata is inconsistent")
archive = manifest.get("archive")
if not isinstance(archive, dict):
raise RuntimeError("release manifest is missing archive metadata")
expected_name = f"agents-server-{version}.tar.gz"
archive_name = str(archive.get("name") or "")
archive_url = str(archive.get("url") or "")
archive_sha = str(archive.get("sha256") or "").lower()
expected_prefix = f"{RELEASE_BASE}/download/v{version}/"
if archive_name != expected_name or archive_url != expected_prefix + expected_name:
raise RuntimeError("release archive location is not trusted")
if not re.fullmatch(r"[0-9a-f]{64}", archive_sha):
raise RuntimeError("release archive checksum is invalid")
return manifest
def check_release(
public_key_path: Path,
track: str = "stable",
*,
expected_version: str | None = None,
require_latest: bool = False,
) -> dict[str, Any]:
track = normalized_release_track(track)
if expected_version is not None:
version = str(expected_version).strip()
if not VERSION_PATTERN.fullmatch(version):
raise RuntimeError("expected release version is invalid")
if release_track(version) != track:
raise RuntimeError(
f"expected release {version} is not on the requested {track} track"
)
if require_latest:
latest_manifest = check_release(public_key_path, track)
latest_version = str(latest_manifest.get("version") or "")
if latest_version != version:
raise RuntimeError(
f"requested {track} release {version} is no longer the latest "
f"signed {track} release {latest_version}"
)
# The discovery path verified this manifest against the immutable
# versioned tag URL. Return that exact pinned document instead of
# fetching a mutable 'latest' alias or selecting a newer release.
return latest_manifest
try:
manifest_bytes = download_bytes(
release_manifest_url(version),
MAX_METADATA_BYTES,
)
signature = download_bytes(
release_signature_url(version),
MAX_METADATA_BYTES,
)
except HTTPError as exc:
if exc.code == 404:
raise ReleaseUnavailableError(
f"Signed {track} AgentsServer release {version} is unavailable."
) from exc
raise
return verify_manifest(
manifest_bytes,
signature,
public_key_path,
expected_version=version,
track=track,
)
try:
releases_bytes = download_bytes(RELEASES_API_URL, MAX_METADATA_BYTES)
except HTTPError as exc:
if exc.code == 404:
raise ReleaseUnavailableError("No signed AgentsServer release has been published yet.") from exc
if exc.code not in {403, 429}:
raise
candidates = release_candidates_from_public_pages(track)
else:
try:
releases = json.loads(releases_bytes)
except json.JSONDecodeError as exc:
raise RuntimeError("GitHub releases response is invalid JSON") from exc
candidates = release_candidates(releases, track)
if not candidates:
raise ReleaseUnavailableError(f"No signed {track} AgentsServer release is available.")
for version in candidates:
try:
manifest_bytes = download_bytes(release_manifest_url(version), MAX_METADATA_BYTES)
signature = download_bytes(release_signature_url(version), MAX_METADATA_BYTES)
except HTTPError as exc:
if exc.code == 404:
continue
raise
return verify_manifest(
manifest_bytes,
signature,
public_key_path,
expected_version=version,
track=track,
)
raise ReleaseUnavailableError(f"No signed {track} AgentsServer release is available.")
def release_transition_allowed(current: str, target: str, track: str = "stable") -> bool:
"""Allow forward updates and an explicit prerelease-to-stable channel exit."""
track = normalized_release_track(track)
if release_track(target) != track:
return False
if version_key(target) > version_key(current):
return True
return (
version_key(target) < version_key(current)
and track == "stable"
and version_is_prerelease(current)
and not version_is_prerelease(target)
)
def safe_extract(archive_path: Path, destination: Path) -> Path:
destination = destination.resolve()
with tarfile.open(archive_path, "r:gz") as archive:
members = archive.getmembers()
for member in members:
target = (destination / member.name).resolve()
if destination != target and destination not in target.parents:
raise RuntimeError("release archive contains an unsafe path")
if member.issym() or member.islnk():
raise RuntimeError("release archive must not contain links")
archive.extractall(destination, members=members, filter="data")
roots = [entry for entry in destination.iterdir() if entry.is_dir()]
if len(roots) != 1 or not (roots[0] / "install.sh").is_file():
raise RuntimeError("release archive has an invalid layout")
return roots[0]
def run_update(args: argparse.Namespace) -> None:
status_path = Path(args.status_file).expanduser().resolve()
public_key = Path(args.public_key).expanduser().resolve()
track = normalized_release_track(getattr(args, "track", "stable"))
auth_token = consume_auth_token_file(getattr(args, "auth_token_file", None))
expected_server_identity = str(
getattr(args, "expected_server_identity", "") or ""
).strip()
if not expected_server_identity:
raise RuntimeError("managed update is missing the stable server identity")
update_id = str(getattr(args, "update_id", "") or "").strip()
if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", update_id) is None:
raise RuntimeError("managed update is missing a valid update ID")
expected_service_cgroup = str(
getattr(args, "expected_service_cgroup", "") or ""
).strip() or None
if expected_service_cgroup is not None and re.fullmatch(
r"/(?:[A-Za-z0-9_.@:-]+/)*[A-Za-z0-9_.@:-]+",
expected_service_cgroup,
) is None:
raise RuntimeError("managed update has an invalid service cgroup")
repair_failed_team_hub_host = bool(
getattr(args, "repair_failed_team_hub_host", False)
)
expected_team_hub_id = str(
getattr(args, "expected_team_hub_id", "") or ""
).strip() or None
raw_team_hub_transport = getattr(args, "expected_team_hub_transport", None)
expected_team_hub_transport = (
str(raw_team_hub_transport).strip()
if raw_team_hub_transport is not None
else None
)
raw_team_hub_url = getattr(args, "expected_team_hub_url", None)
expected_team_hub_url = (
str(raw_team_hub_url).strip()
if raw_team_hub_url is not None
else None
)
raw_team_hub_direct_ip_url = getattr(
args, "expected_team_hub_direct_ip_url", None
)
expected_team_hub_direct_ip_url = (
str(raw_team_hub_direct_ip_url).strip()
if raw_team_hub_direct_ip_url is not None
else None
)
team_hub_snapshot = str(
getattr(args, "team_hub_snapshot", "") or ""
).strip() or None
team_hub_data_dir = str(
getattr(args, "team_hub_data_dir", "") or ""
).strip() or None
if any((expected_team_hub_id, team_hub_snapshot, team_hub_data_dir)) and not all(
(expected_team_hub_id, team_hub_snapshot, team_hub_data_dir)
):
raise RuntimeError("managed Team Hub rollback arguments must be complete")
if repair_failed_team_hub_host and any(
(expected_team_hub_id, team_hub_snapshot, team_hub_data_dir)
):
raise RuntimeError(
"failed Team Hub repair cannot reuse live-host rollback arguments"
)
if expected_team_hub_id is None and not repair_failed_team_hub_host:
if (
expected_team_hub_transport is not None
or expected_team_hub_url is not None
or expected_team_hub_direct_ip_url is not None
):
raise RuntimeError("managed Team Hub transport arguments require a Hub identity")
if repair_failed_team_hub_host and (
raw_team_hub_transport is None
or raw_team_hub_url is None
or raw_team_hub_direct_ip_url is None
):
raise RuntimeError(
"failed Team Hub repair requires exact transport continuity arguments"
)
if (
expected_team_hub_id is not None or repair_failed_team_hub_host
) and expected_team_hub_transport is None:
if raw_team_hub_url is not None:
raise RuntimeError("legacy loopback Team Hub cannot have a remote URL")
elif expected_team_hub_id is not None or repair_failed_team_hub_host:
if expected_team_hub_transport == "loopback":
if raw_team_hub_url is None or expected_team_hub_url != "":
raise RuntimeError("loopback Team Hub cannot have a remote URL")
expected_team_hub_url = None
elif expected_team_hub_transport in {"tailscale_serve", "direct_ip"}:
if expected_team_hub_url is None:
raise RuntimeError("remote Team Hub transport requires its exact URL")
try:
from team_hub_host import ( # Imported only for the managed-Hub path.
TEAM_HUB_MODE_HOST,
configured_team_hub_endpoint,
)
resolved_transport, resolved_url, _host, config_error = (
configured_team_hub_endpoint(
TEAM_HUB_MODE_HOST,
expected_team_hub_url,
expected_team_hub_transport,
args.port,
)
)
except Exception as exc:
raise RuntimeError("could not validate the expected Team Hub URL") from exc
if (
config_error is not None
or resolved_transport != expected_team_hub_transport
or resolved_url != expected_team_hub_url
):
raise RuntimeError("expected Team Hub URL is invalid")
else:
raise RuntimeError("managed Team Hub transport is invalid")
if expected_team_hub_direct_ip_url is not None:
if expected_team_hub_id is None and not repair_failed_team_hub_host:
raise RuntimeError("managed Team Hub direct-IP route requires a Hub identity")
if expected_team_hub_direct_ip_url:
try:
from team_hub_host import (
TEAM_HUB_MODE_HOST,
configured_team_hub_endpoint,
)
direct_transport, direct_url, _host, direct_error = (
configured_team_hub_endpoint(
TEAM_HUB_MODE_HOST,
expected_team_hub_direct_ip_url,
"direct_ip",
args.port,
)
)
except Exception as exc:
raise RuntimeError(
"could not validate the expected Team Hub direct-IP route"
) from exc
if (
direct_error is not None
or direct_transport != "direct_ip"
or direct_url != expected_team_hub_direct_ip_url
):
raise RuntimeError("expected Team Hub direct-IP route is invalid")
if (
expected_team_hub_transport == "direct_ip"
and expected_team_hub_direct_ip_url != expected_team_hub_url
):
raise RuntimeError("primary direct-IP Team Hub route changed")
if repair_failed_team_hub_host:
with server_update_status_lock(status_path):
admitted = _read_status_unlocked(status_path)
expected_repair_routes = [
{
"transport": expected_team_hub_transport,
"hub_url": expected_team_hub_url,
}
]
if (
expected_team_hub_direct_ip_url
and expected_team_hub_transport != "direct_ip"
):
expected_repair_routes.append(
{
"transport": "direct_ip",
"hub_url": expected_team_hub_direct_ip_url,
}
)
expected_status = {
"update_id": update_id,
"phase": "starting",
"team_hub_repair_mode": "failed_start",
"team_hub_host_server_identity": expected_server_identity,
"team_hub_transport": expected_team_hub_transport,
"team_hub_url": expected_team_hub_url,
"team_hub_direct_ip_url": expected_team_hub_direct_ip_url,
"team_hub_routes": expected_repair_routes,
}
if any(admitted.get(name) != value for name, value in expected_status.items()):
raise RuntimeError(
"failed Team Hub repair is not owned by the exact admitted update"
)
update_status(
status_path,
expected_update_id=update_id,
phase="checking",
track=track,
runner_pid=os.getpid(),
heartbeat_at=utc_now(),
message=f"Checking the signed {track} release manifest.",
)
expected_version = (
str(getattr(args, "expected_version", "") or "").strip() or None
)
current_version = str(
getattr(args, "current_version", "") or ""
).strip()
require_latest = (
track == "stable"
and bool(current_version)
and version_is_prerelease(current_version)
)
manifest = check_release(
public_key,
track,
expected_version=expected_version,
require_latest=require_latest,
)
version = str(manifest["version"])
if expected_version and version != expected_version:
raise RuntimeError(
f"resolved signed release is {version}, not {expected_version}"
)
if current_version and not release_transition_allowed(current_version, version, track):
raise RuntimeError(
f"resolved release {version} is not newer than installed version {current_version}; "
"managed updates only permit forward updates or an explicit beta-to-stable channel switch"
)
with tempfile.TemporaryDirectory(prefix="agents-server-update-") as temporary:
root = Path(temporary)
archive_path = root / str(manifest["archive"]["name"])
update_status(
status_path,
expected_update_id=update_id,
phase="downloading",
track=track,
target_version=version,
message=f"Downloading AgentsServer {version}.",
)
archive_bytes = download_bytes(str(manifest["archive"]["url"]), MAX_ARCHIVE_BYTES, timeout=120.0)
digest = hashlib.sha256(archive_bytes).hexdigest()
if digest != manifest["archive"]["sha256"]:
raise RuntimeError("release archive checksum does not match the signed manifest")
archive_path.write_bytes(archive_bytes)
update_status(
status_path,
expected_update_id=update_id,
phase="verifying",
message="Signature and archive checksum verified.",
)
source = safe_extract(archive_path, root / "extracted")
install = source / "install.sh"
install.chmod(0o755)
command = [
str(install),
"--non-interactive",
"--release-version", version,
"--port", str(args.port),
"--bind", args.bind,
"--expected-server-identity", expected_server_identity,
]
command.extend(instance_installer_arguments(source))
if expected_team_hub_id is not None:
command.extend(
[
"--expected-team-hub-id", expected_team_hub_id,
"--team-hub-snapshot", str(team_hub_snapshot),
"--team-hub-data-dir", str(team_hub_data_dir),
"--team-hub-operation-id", update_id,
]
)
if expected_team_hub_transport is not None:
command.extend(
[
"--expected-team-hub-transport",
expected_team_hub_transport,
"--expected-team-hub-url",
expected_team_hub_url or "",
]
)
if expected_team_hub_direct_ip_url is not None:
command.extend(
[
"--expected-team-hub-direct-ip-url",
expected_team_hub_direct_ip_url,
]
)
elif repair_failed_team_hub_host:
command.extend(
[
"--repair-failed-team-hub-host",
"--managed-update-id",
update_id,
"--expected-team-hub-transport",
expected_team_hub_transport or "",
"--expected-team-hub-url",
expected_team_hub_url or "",
"--expected-team-hub-direct-ip-url",
expected_team_hub_direct_ip_url or "",
]
)
wait_for_server_idle(
args.port,
status_path=status_path,
expected_update_id=update_id,
expected_server_identity=expected_server_identity,
token=auth_token,
require_verified_service_cgroup=expected_service_cgroup is not None,
)
update_status(
status_path,
expected_update_id=update_id,
phase="installing",
message=f"Installing AgentsServer {version} with rollback protection.",
)
log_path = status_path.with_name("server-update.log")
run_installer(
command,
cwd=source,
status_path=status_path,
log_path=log_path,
version=version,
expected_update_id=update_id,
managed_update_id=(
update_id
if expected_service_cgroup is not None
or repair_failed_team_hub_host
else None
),
expected_service_cgroup=expected_service_cgroup,
)
identity_arguments: dict[str, Any] = {
"token": auth_token,
"expected_server_identity": expected_server_identity,
"expected_team_hub_id": expected_team_hub_id,
}
if expected_team_hub_transport is not None and not repair_failed_team_hub_host:
identity_arguments["expected_team_hub_transport"] = (
expected_team_hub_transport
)
if expected_team_hub_url is not None and not repair_failed_team_hub_host:
identity_arguments["expected_team_hub_url"] = expected_team_hub_url
if (
expected_team_hub_direct_ip_url is not None
and not repair_failed_team_hub_host
):
identity_arguments["expected_team_hub_direct_ip_url"] = (
expected_team_hub_direct_ip_url
)
assert_post_update_identity(args.port, **identity_arguments)
if repair_failed_team_hub_host:
assert_repaired_team_hub_identity(
args.port,
token=auth_token,
expected_server_identity=expected_server_identity,
expected_team_hub_transport=expected_team_hub_transport or "",
expected_team_hub_url=expected_team_hub_url,
expected_team_hub_direct_ip_url=(
expected_team_hub_direct_ip_url or ""
),
)
# install.sh owns the success clear while it can still stop the
# candidate, restore the verified snapshot, and restart the old
# release. The runner clears only failures before install starts.
update_status(
status_path,
expected_update_id=update_id,
phase="complete",
update_available=False,
message=f"AgentsServer {version} is installed and healthy.",
track=track,
installed_version=version,
heartbeat_at=None,
elapsed_seconds=None,
runner_pid=None,
error_code=None,
error_action=None,
retryable=None,
finished_at=utc_now(),
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--status-file", required=True)
parser.add_argument("--public-key", required=True)
parser.add_argument("--port", required=True, type=int)
parser.add_argument("--bind", required=True)
parser.add_argument("--expected-version")
parser.add_argument("--current-version")
parser.add_argument("--track", choices=sorted(RELEASE_TRACKS), default="stable")
parser.add_argument("--auth-token-file")
parser.add_argument("--expected-server-identity", required=True)
parser.add_argument("--update-id", required=True)
parser.add_argument("--expected-service-cgroup")
parser.add_argument("--expected-team-hub-id")
parser.add_argument("--expected-team-hub-transport")
parser.add_argument("--expected-team-hub-url")
parser.add_argument("--expected-team-hub-direct-ip-url")
parser.add_argument("--team-hub-snapshot")
parser.add_argument("--team-hub-data-dir")
parser.add_argument("--repair-failed-team-hub-host", action="store_true")
args = parser.parse_args()
try:
run_update(args)
return 0
except Exception as exc:
status_path = Path(args.status_file).expanduser().resolve()
update_id = str(getattr(args, "update_id", "") or "").strip()
try:
with server_update_status_lock(status_path):
current = _read_status_unlocked(status_path)
if (
str(current.get("update_id") or "") != update_id
or str(current.get("phase") or "")
not in RUNNER_OWNED_ACTIVE_PHASES
):
return 1
phase = str(current.get("phase") or "")
if phase in {
"starting",
"checking",
"downloading",
"verifying",
}:
# Keep the owning active status in place until its exact
# Hub fence is cleared. A clear failure therefore remains
# fail-closed instead of publishing a false terminal row.
clear_team_hub_maintenance(args)
elif phase in {"installing", "restarting"} and \
team_hub_maintenance_fence_present(args):
# Once install.sh starts, only its verified rollback or
# successful candidate handoff may clear the fence. Keep
# the active row if recovery was not proven complete.
return 1
_update_status_unlocked(
status_path,
current,
phase="failed",
message=str(exc),
heartbeat_at=None,
runner_pid=None,
finished_at=utc_now(),
)
except (UpdateOwnershipLostError, RuntimeError, OSError):
pass
return 1
if __name__ == "__main__":
raise SystemExit(main())