app/Models/Server.php
Rama main · réplica de lectura del repositorio original.
<?php
namespace App\Models;
use App\Actions\Application\StopApplicationOneServer;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\InstallDocker;
use App\Actions\Server\InstallPrerequisites;
use App\Actions\Server\StartSentinel;
use App\Actions\Server\ValidatePrerequisites;
use App\Enums\ApplicationDeploymentStatus;
use App\Enums\ProxyTypes;
use App\Enums\ServerRole;
use App\Events\ServerReachabilityChanged;
use App\Helpers\SshMultiplexingHelper;
use App\Helpers\SslHelper;
use App\Jobs\CheckAndStartSentinelJob;
use App\Jobs\CheckTraefikVersionForServerJob;
use App\Jobs\RegenerateSslCertJob;
use App\Jobs\ServerConnectionCheckJob;
use App\Livewire\Server\Proxy;
use App\Notifications\Server\HostKeyChanged;
use App\Notifications\Server\Reachable;
use App\Notifications\Server\Unreachable;
use App\Services\DigitalOceanService;
use App\Services\HetznerService;
use App\Services\HostingerService;
use App\Services\VultrService;
use App\Support\ValidationPatterns;
use App\Traits\Auditable;
use App\Traits\ClearsGlobalSearchCache;
use App\Traits\HasMetrics;
use App\Traits\HasSafeStringAttribute;
use App\Traits\HasStars;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Database\Eloquent\Casts\Attribute;
use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Database\Eloquent\Relations\HasMany;
use Illuminate\Database\Eloquent\Relations\HasOne;
use Illuminate\Database\Eloquent\Relations\MorphMany;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Http\Exceptions\ThrottleRequestsException;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\RateLimiter;
use Illuminate\Support\Facades\Storage;
use Illuminate\Support\Str;
use Illuminate\Support\Stringable;
use OpenApi\Attributes as OA;
use Spatie\SchemalessAttributes\Casts\SchemalessAttributes;
use Spatie\SchemalessAttributes\SchemalessAttributesTrait;
use Spatie\Url\Url;
use Stevebauman\Purify\Facades\Purify;
use Symfony\Component\Yaml\Exception\ParseException;
use Symfony\Component\Yaml\Yaml;
/**
* @property array{
* current: string,
* latest: string,
* type: 'patch_update'|'minor_upgrade',
* checked_at: string,
* newer_branch_target?: string,
* newer_branch_latest?: string,
* upgrade_target?: string
* }|null $traefik_outdated_info Traefik version tracking information.
*
* This JSON column stores information about outdated Traefik proxy versions on this server.
* The structure varies depending on the type of update available:
*
* **For patch updates** (e.g., 3.5.0 → 3.5.2):
* ```php
* [
* 'current' => '3.5.0', // Current version (without 'v' prefix)
* 'latest' => '3.5.2', // Latest patch version available
* 'type' => 'patch_update', // Update type identifier
* 'checked_at' => '2025-11-14T10:00:00Z', // ISO8601 timestamp
* 'newer_branch_target' => 'v3.6', // (Optional) Available major/minor version
* 'newer_branch_latest' => '3.6.2' // (Optional) Latest version in that branch
* ]
* ```
*
* **For minor/major upgrades** (e.g., 3.5.6 → 3.6.2):
* ```php
* [
* 'current' => '3.5.6', // Current version
* 'latest' => '3.6.2', // Latest version in target branch
* 'type' => 'minor_upgrade', // Update type identifier
* 'upgrade_target' => 'v3.6', // Target branch (with 'v' prefix)
* 'checked_at' => '2025-11-14T10:00:00Z' // ISO8601 timestamp
* ]
* ```
*
* **Null value**: Set to null when:
* - Server is fully up-to-date with the latest version
* - Traefik image uses the 'latest' tag (no fixed version tracking)
* - No Traefik version detected on the server
*
* @see CheckTraefikVersionForServerJob Where this data is populated
* @see Proxy Where this data is read and displayed
*/
#[OA\Schema(
description: 'Server model',
type: 'object',
properties: [
'id' => ['type' => 'integer', 'description' => 'The server ID.'],
'uuid' => ['type' => 'string', 'description' => 'The server UUID.'],
'name' => ['type' => 'string', 'description' => 'The server name.'],
'description' => ['type' => 'string', 'description' => 'The server description.'],
'ip' => ['type' => 'string', 'description' => 'The IP address.'],
'user' => ['type' => 'string', 'description' => 'The user.'],
'port' => ['type' => 'integer', 'description' => 'The port number.'],
'proxy' => ['type' => 'object', 'description' => 'The proxy configuration.'],
'proxy_type' => ['type' => 'string', 'enum' => ['traefik', 'caddy', 'none'], 'description' => 'The proxy type.'],
'high_disk_usage_notification_sent' => ['type' => 'boolean', 'description' => 'The flag to indicate if the high disk usage notification has been sent.'],
'unreachable_notification_sent' => ['type' => 'boolean', 'description' => 'The flag to indicate if the unreachable notification has been sent.'],
'unreachable_count' => ['type' => 'integer', 'description' => 'The unreachable count for your server.'],
'validation_logs' => ['type' => 'string', 'description' => 'The validation logs.'],
'log_drain_notification_sent' => ['type' => 'boolean', 'description' => 'The flag to indicate if the log drain notification has been sent.'],
'swarm_cluster' => ['type' => 'string', 'description' => 'The swarm cluster configuration.'],
'settings' => ['$ref' => '#/components/schemas/ServerSetting'],
]
)]
class Server extends BaseModel
{
use Auditable, ClearsGlobalSearchCache, HasFactory, HasMetrics, HasStars, SchemalessAttributesTrait, SoftDeletes;
/**
* Sentinel IP for servers that do not have a real address yet
* (cloud provisioning in progress or parked as unreachable).
* Scheduled jobs skip these servers via skipServer().
*/
public const PLACEHOLDER_IP = '1.2.3.4';
public const PLACEHOLDER_IPS = [self::PLACEHOLDER_IP, '0.0.0.0', '::'];
protected array $auditExclude = ['canvas_position'];
/**
* Serves the proxy error pages (503.html) for the Traefik and Caddy catch-all.
*/
public const PROXY_ERROR_PAGE_CONTAINER = 'coolify-proxy-error-pages';
public const PROXY_ERROR_PAGE_IMAGE = 'busybox:musl';
/**
* Maximum size of a custom error or maintenance page. The page goes to the server with scp
* (see uploadProxyFile()), so this limit only keeps the Livewire payload reasonable.
*/
public const PROXY_ERROR_PAGE_MAX_BYTES = 1048576;
/**
* Default Caddy proxy image. caddy-docker-proxy 2.13 ships Caddy 2.11.
*/
public const RECOMMENDED_CADDY_PROXY_IMAGE = 'lucaslorentz/caddy-docker-proxy:2.13-alpine';
/**
* First caddy-docker-proxy version that ships Caddy 2.8+ (`log_append`, `basic_auth`).
* Version 2.8 of the image still runs Caddy 2.7.6.
*/
public const MINIMUM_CURRENT_CADDY_PROXY_VERSION = [2, 9];
/**
* Address of the development `testing-host` server (docker-compose.dev*.yml, ServerSeeder).
*/
public const DEV_TESTING_HOST_IP = 'coolify-testing-host';
public static $batch_counter = 0;
/**
* Identity map cache for request-scoped Server lookups.
* Prevents N+1 queries when the same Server is accessed multiple times.
*/
private static ?array $identityMapCache = null;
protected $appends = ['is_coolify_host'];
protected static function booted()
{
static::saving(function ($server) {
$payload = [];
if ($server->user) {
$payload['user'] = trim($server->user);
}
if ($server->ip) {
$payload['ip'] = trim($server->ip);
// Update ip_previous when ip is being changed
if ($server->isDirty('ip') && $server->getOriginal('ip')) {
$payload['ip_previous'] = $server->getOriginal('ip');
}
}
$server->fill($payload);
});
static::saved(function ($server) {
if ($server->wasChanged(['ip', 'port'])) {
$server->settings?->forgetSshHostKey('address_changed');
}
if ($server->wasChanged('private_key_id') || $server->privateKey?->isDirty()) {
refresh_server_connection($server->privateKey);
} elseif ($server->wasChanged(['user', 'ip', 'port'])) {
// An open multiplexed connection keeps the old user and host until it is closed.
SshMultiplexingHelper::removeMuxFile($server);
}
});
static::created(function ($server) {
ServerSetting::create([
'server_id' => $server->id,
]);
if ($server->id === 0) {
if ($server->isSwarm()) {
(new SwarmDocker)->forceFill([
'id' => 0,
'name' => 'coolify',
'network' => 'coolify-overlay',
'server_id' => $server->id,
])->save();
} else {
(new StandaloneDocker)->forceFill($server->defaultStandaloneDockerAttributes(id: 0))->saveQuietly();
}
} else {
if ($server->isSwarm()) {
SwarmDocker::create([
'name' => 'coolify-overlay',
'network' => 'coolify-overlay',
'server_id' => $server->id,
]);
} else {
$standaloneDocker = new StandaloneDocker;
$standaloneDocker->forceFill($server->defaultStandaloneDockerAttributes());
$standaloneDocker->saveQuietly();
}
}
if (! isset($server->proxy->redirect_enabled)) {
$server->proxy->redirect_enabled = true;
}
// Create predefined server shared variables
SharedEnvironmentVariable::create([
'key' => 'COOLIFY_SERVER_UUID',
'value' => $server->uuid,
'type' => 'server',
'server_id' => $server->id,
'team_id' => $server->team_id,
'is_literal' => true,
]);
SharedEnvironmentVariable::create([
'key' => 'COOLIFY_SERVER_NAME',
'value' => $server->name,
'type' => 'server',
'server_id' => $server->id,
'team_id' => $server->team_id,
'is_literal' => true,
]);
});
static::retrieved(function ($server) {
if (! isset($server->proxy->redirect_enabled)) {
$server->proxy->redirect_enabled = true;
}
});
static::forceDeleting(function ($server) {
$server->destinations()->each(function ($destination) {
$destination->delete();
});
// Leftover active runner rows would block deleting the GitHub App.
// The runners of a transferred server now belong to the other instance, so keep them registered at GitHub.
$executions = GithubRunnerExecution::query()->where('server_id', $server->id);
if ($server->isTransferredAway()) {
$executions->delete();
} else {
GithubRunnerExecution::deleteAndDeregister($executions);
}
$server->githubRunnerConfig()->delete();
$server->settings()->delete();
SshMultiplexingHelper::deleteStaleKnownHostsFiles($server->uuid, null);
$server->sslCertificates()->delete();
$server->notificationThrottles()->delete();
ImageScanPolicy::query()->where('scope_type', $server->getMorphClass())->where('scope_id', $server->id)->delete();
});
static::deleted(function (Server $server) {
$server->failQueuedDeployments();
});
static::updated(function () {
static::flushIdentityMap();
});
}
/**
* Fail the queued deployments of a deleted server, because they can never start.
*/
public function failQueuedDeployments(): void
{
ApplicationDeploymentQueue::query()
->where('server_id', $this->id)
->where('status', ApplicationDeploymentStatus::QUEUED->value)
->eachById(function (ApplicationDeploymentQueue $deployment) {
$updated = ApplicationDeploymentQueue::query()
->whereKey($deployment->id)
->where('status', ApplicationDeploymentStatus::QUEUED->value)
->update(['status' => ApplicationDeploymentStatus::FAILED->value]);
if ($updated > 0) {
$deployment->addLogEntry('The server was deleted.', 'stderr');
}
});
}
/**
* Find a Server by ID using the identity map cache.
* This prevents N+1 queries when the same Server is accessed multiple times.
*/
public static function findCached($id): ?static
{
if ($id === null) {
return null;
}
if (static::$identityMapCache === null) {
static::$identityMapCache = [];
}
if (! isset(static::$identityMapCache[$id])) {
static::$identityMapCache[$id] = static::query()->find($id);
}
return static::$identityMapCache[$id];
}
/**
* Flush the identity map cache.
* Called automatically on update, and should be called in tests.
*/
public static function flushIdentityMap(): void
{
static::$identityMapCache = null;
}
protected $casts = [
'proxy' => SchemalessAttributes::class,
'traefik_outdated_info' => 'array',
'server_metadata' => 'array',
'logdrain_axiom_api_key' => 'encrypted',
'logdrain_newrelic_license_key' => 'encrypted',
'delete_unused_volumes' => 'boolean',
'delete_unused_networks' => 'boolean',
'force_disabled' => 'boolean',
'sentinel_waiting_since' => 'datetime',
'canvas_position' => 'array',
];
/**
* Sensitive fields hidden by default in serialized output (toArray/toJson).
* API controllers should call makeVisible([...]) for callers with the
* `read:sensitive` or `root` token ability.
*/
protected $hidden = [
'logdrain_axiom_api_key',
'logdrain_newrelic_license_key',
'proxy_error_page',
];
protected $schemalessAttributes = [
'proxy',
];
protected $fillable = [
'name',
'ip',
'port',
'user',
'description',
'private_key_id',
'cloud_provider_token_id',
'team_id',
'hetzner_server_id',
'hetzner_server_status',
'vultr_instance_id',
'vultr_instance_status',
'digitalocean_droplet_id',
'digitalocean_droplet_status',
'hostinger_virtual_machine_id',
'hostinger_virtual_machine_status',
'is_validating',
'validation_logs',
'detected_traefik_version',
'traefik_outdated_info',
'server_metadata',
'ip_previous',
];
use HasSafeStringAttribute;
public function setValidationLogsAttribute($value): void
{
$this->attributes['validation_logs'] = $value !== null
? Purify::config('validation_logs')->clean(remove_iip($value))
: null;
}
public function type()
{
return 'server';
}
public function hasPlaceholderIp(): bool
{
// Cast: the saving hook stores the ip as a Stringable in memory.
return self::isPlaceholderIp((string) $this->ip);
}
public static function isPlaceholderIp(?string $ip): bool
{
return blank($ip) || in_array($ip, self::PLACEHOLDER_IPS, true);
}
/**
* Replace a placeholder IP with the real address once the cloud
* provider reports one. Returns true when the IP was updated.
*/
public function backfillPlaceholderIp(?string $ip): bool
{
if ($this->isTransferredAway()) {
return false;
}
$ip = blank($ip) ? null : self::normalizeIp($ip);
if (self::isPlaceholderIp($ip)) {
return false;
}
$updated = static::query()
->whereKey($this->getKey())
->where(function (Builder $query): void {
$query->whereNull('ip')
->orWhere('ip', '')
->orWhereIn('ip', self::PLACEHOLDER_IPS);
})
->update(['ip' => $ip]);
if ($updated === 0) {
return false;
}
$this->forceFill(['ip' => $ip]);
$this->syncOriginalAttribute('ip');
static::flushIdentityMap();
// The query update skips the saved hook, which forgets the host key of the old address.
$this->settings?->forgetSshHostKey('address_changed');
return true;
}
/**
* Persist provider status without saving a stale in-memory IP value.
*
* @param array<string, mixed> $updates
*/
private function persistProviderState(array $updates): void
{
if (empty($updates)) {
return;
}
static::query()->whereKey($this->getKey())->update($updates);
$this->forceFill($updates);
$this->syncOriginalAttributes(array_keys($updates));
static::flushIdentityMap();
}
/**
* Refresh the status of the linked cloud server from its provider.
*/
public function refreshCloudProviderState(): ?string
{
return match ($this->cloudProviderToken?->provider) {
'hetzner' => $this->hetzner_server_id ? $this->refreshHetznerState() : null,
'vultr' => $this->vultr_instance_id ? $this->refreshVultrState() : null,
'digitalocean' => $this->digitalocean_droplet_id ? $this->refreshDigitalOceanState() : null,
'hostinger' => $this->hostinger_virtual_machine_id ? $this->refreshHostingerState() : null,
default => null,
};
}
public function refreshHetznerState(): ?string
{
// Another instance manages the cloud server of an archive; keep the stored status.
if ($this->isTransferredAway()) {
return $this->hetzner_server_status;
}
if (! $this->hetzner_server_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'hetzner') {
return $this->hetzner_server_status;
}
$hetznerService = new HetznerService($this->cloudProviderToken->token);
$server = $hetznerService->getServer($this->hetzner_server_id);
$status = $server['status'] ?? null;
$assignedIp = data_get($server, 'public_net.ipv4.ip') ?? hetznerServerIpv6(data_get($server, 'public_net.ipv6.ip'));
$updates = [];
if ($this->hetzner_server_status !== $status) {
$updates['hetzner_server_status'] = $status;
}
$this->persistProviderState($updates);
$this->backfillPlaceholderIp($assignedIp);
return $status;
}
public function refreshVultrState(): ?string
{
// Another instance manages the cloud server of an archive; keep the stored status.
if ($this->isTransferredAway()) {
return $this->vultr_instance_status;
}
if (! $this->vultr_instance_id || ! $this->cloudProviderToken) {
return null;
}
$vultrService = new VultrService($this->cloudProviderToken->token);
try {
$instance = $vultrService->getInstance($this->vultr_instance_id);
} catch (\Throwable $e) {
if ((int) $e->getCode() !== 404) {
throw $e;
}
if ($this->vultr_instance_status !== 'deleted') {
$this->persistProviderState(['vultr_instance_status' => 'deleted']);
}
return 'deleted';
}
$status = ($instance['power_status'] ?? null) === 'stopped'
? 'stopped'
: ($instance['status'] ?? null);
$publicIp = $vultrService->getPublicIp($instance);
$updates = [];
if ($this->vultr_instance_status !== $status) {
$updates['vultr_instance_status'] = $status;
}
$this->persistProviderState($updates);
$this->backfillPlaceholderIp($publicIp);
return $status;
}
public function refreshDigitalOceanState(): ?string
{
// Another instance manages the cloud server of an archive; keep the stored status.
if ($this->isTransferredAway()) {
return $this->digitalocean_droplet_status;
}
if (! $this->digitalocean_droplet_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'digitalocean') {
return $this->digitalocean_droplet_status;
}
$digitalOceanService = new DigitalOceanService($this->cloudProviderToken->token);
try {
$droplet = $digitalOceanService->getDroplet((int) $this->digitalocean_droplet_id);
} catch (RequestException $e) {
if ($e->response?->status() === 404) {
$this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
return 'deleted';
}
throw $e;
} catch (\Throwable $e) {
if ((int) $e->getCode() === 404) {
$this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
return 'deleted';
}
throw $e;
}
if (empty($droplet)) {
return $this->digitalocean_droplet_status;
}
$status = $droplet['status'] ?? null;
$ip = $digitalOceanService->getPublicIpAddress($droplet);
$this->persistProviderState(['digitalocean_droplet_status' => $status]);
$this->backfillPlaceholderIp($ip);
return $status;
}
public function refreshHostingerState(): ?string
{
// Another instance manages the cloud server of an archive; keep the stored status.
if ($this->isTransferredAway()) {
return $this->hostinger_virtual_machine_status;
}
if (! $this->hostinger_virtual_machine_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'hostinger') {
return $this->hostinger_virtual_machine_status;
}
$hostingerService = new HostingerService($this->cloudProviderToken->token);
$virtualMachine = $hostingerService->getVirtualMachine((int) $this->hostinger_virtual_machine_id);
if (empty($virtualMachine)) {
return $this->hostinger_virtual_machine_status;
}
$status = $virtualMachine['state'] ?? null;
$ip = $hostingerService->getPublicIpAddress($virtualMachine);
$this->persistProviderState(['hostinger_virtual_machine_status' => $status]);
$this->backfillPlaceholderIp($ip);
return $status;
}
protected function isCoolifyHost(): Attribute
{
return Attribute::make(
get: function () {
return $this->id === 0;
}
);
}
public static function isReachable()
{
return Server::ownedByCurrentTeam()->whereRelation('settings', 'is_reachable', true);
}
/**
* Get query builder for servers owned by current team.
* If you need all servers without further query chaining, use ownedByCurrentTeamCached() instead.
*/
public static function ownedByCurrentTeam(array $select = ['*'])
{
$teamId = currentTeam()->id;
$selectArray = collect($select)->concat(['id']);
return Server::whereTeamId($teamId)->with('settings', 'swarmDockers', 'standaloneDockers')->select($selectArray->all())->orderBy('name');
}
/**
* Get all servers owned by current team (cached for request duration).
*/
public static function ownedByCurrentTeamCached()
{
return once(function () {
return Server::ownedByCurrentTeam()->get();
});
}
public static function isUsable(): Builder
{
return self::usableByBuildServerStatus(false);
}
public static function isUsableBuildServer(): Builder
{
return self::usableByBuildServerStatus(true);
}
private static function usableByBuildServerStatus(bool $isBuildServer): Builder
{
$query = Server::ownedByCurrentTeam()
->whereRelation('settings', 'is_reachable', true)
->whereRelation('settings', 'is_usable', true)
->whereRelation('settings', 'is_swarm_worker', false)
->whereRelation('settings', 'force_disabled', false);
return $isBuildServer
? self::whereServerRole($query, ServerRole::BUILD)
: self::whereServerRole($query, ServerRole::DEPLOYMENT, ServerRole::BOTH);
}
/**
* Filters by the effective server role. A null role falls back to the legacy
* is_build_server flag, like ServerSetting::effectiveServerRole().
*/
private static function whereServerRole(Builder $query, ServerRole ...$roles): Builder
{
$legacyBuildServerFlags = collect($roles)
->reject(fn (ServerRole $role) => $role === ServerRole::DEPLOYMENT)
->map(fn (ServerRole $role) => $role === ServerRole::BUILD)
->values()
->all();
return $query->whereHas('settings', fn (Builder $settings) => $settings
->whereIn('server_role', array_map(fn (ServerRole $role) => $role->value, $roles))
->orWhere(fn (Builder $legacy) => $legacy
->whereNull('server_role')
->whereIn('is_build_server', $legacyBuildServerFlags)));
}
public function canHostResources(): bool
{
return $this->settings->effectiveServerRole()->canDeploy();
}
public function settings()
{
return $this->hasOne(ServerSetting::class);
}
public function imageScans(): HasMany
{
return $this->hasMany(ImageScan::class);
}
public function imageSboms(): HasMany
{
return $this->hasMany(ImageSbom::class);
}
public function hostScan(): HasOne
{
return $this->hasOne(ServerHostScan::class);
}
public function dockerCleanupExecutions()
{
return $this->hasMany(DockerCleanupExecution::class);
}
public function proxySet()
{
return $this->proxyType() && $this->proxyType() !== 'NONE' && $this->isFunctional() && ! $this->isSwarmWorker() && $this->canHostResources();
}
/**
* Writes the catch-all proxy configuration for requests that no resource handles.
* Priority: redirect URL > custom error page > Coolify default error page.
*/
public function setupDefaultRedirect()
{
$banner =
"# This file is generated by Coolify, do not edit it manually.\n".
"# Disable the default redirect to customize (only if you know what are you doing).\n\n";
$dynamic_conf_path = $this->proxyPath().'/dynamic';
$proxy_type = $this->proxyType();
$redirect_enabled = $this->proxy->redirect_enabled ?? true;
if ($proxy_type === ProxyTypes::TRAEFIK->value) {
$default_redirect_file = "$dynamic_conf_path/default_redirect_503.yaml";
} elseif ($proxy_type === ProxyTypes::CADDY->value) {
$default_redirect_file = "$dynamic_conf_path/default_redirect_503.caddy";
}
instant_remote_process([
"mkdir -p $dynamic_conf_path",
"rm -f $dynamic_conf_path/default_redirect_404.yaml",
"rm -f $dynamic_conf_path/default_redirect_404.caddy",
], $this);
// The error page container must run before the catch-all points to it.
$this->setupProxyErrorPageContainer();
if ($redirect_enabled === false) {
instant_remote_process(["rm -f $default_redirect_file"], $this);
} else {
$conf = $banner.$this->defaultRedirectConfiguration();
$base64 = base64_encode($conf);
instant_remote_process([
"echo '$base64' | base64 -d | tee $default_redirect_file > /dev/null",
], $this);
}
if ($proxy_type === 'CADDY') {
$this->reloadCaddy();
}
}
/**
* Catch-all proxy configuration (without the banner) for an enabled default redirect.
*/
public function defaultRedirectConfiguration(): string
{
$redirect_url = $this->proxy->redirect_url;
$usesErrorPage = $this->usesProxyErrorPage();
if ($this->proxyType() === ProxyTypes::CADDY->value) {
if (filled($redirect_url)) {
return ":80, :443 {
redir $redirect_url
}";
}
if ($usesErrorPage) {
// busybox httpd answers 200, so copy_response sets the 503 status.
return ':80, :443 {
reverse_proxy '.self::PROXY_ERROR_PAGE_CONTAINER.':80 {
rewrite /503.html
handle_response {
copy_response 503
}
}
}';
}
return ':80, :443 {
respond 503
}';
}
$dynamic_conf = [
'http' => [
'routers' => [
'catchall' => [
'entryPoints' => [
0 => 'http',
1 => 'https',
],
'service' => 'noop',
'rule' => 'PathPrefix(`/`)',
'tls' => $this->traefikCertificateResolver() ? ['certResolver' => $this->traefikCertificateResolver()] : [],
'priority' => -1000,
],
],
'services' => [
'noop' => [
'loadBalancer' => [
'servers' => [],
],
],
],
],
];
if (filled($redirect_url)) {
$dynamic_conf['http']['routers']['catchall']['middlewares'] = [
0 => 'redirect-regexp',
];
$dynamic_conf['http']['services']['noop']['loadBalancer']['servers'][0] = [
'url' => '',
];
$dynamic_conf['http']['middlewares'] = [
'redirect-regexp' => [
'redirectRegex' => [
'regex' => '(.*)',
'replacement' => $redirect_url,
'permanent' => false,
],
],
];
} elseif ($usesErrorPage) {
// The noop service (no servers) keeps the 503; the errors middleware only replaces the body.
$dynamic_conf['http']['routers']['catchall']['middlewares'] = [
0 => 'error-pages',
];
$dynamic_conf['http']['middlewares'] = [
'error-pages' => [
'errors' => [
'status' => ['503'],
'service' => 'error-pages',
'query' => '/503.html',
],
],
];
$dynamic_conf['http']['services']['error-pages'] = [
'loadBalancer' => [
'servers' => [
['url' => 'http://'.self::PROXY_ERROR_PAGE_CONTAINER.':80'],
],
],
];
}
// A router with `tls` matches only HTTPS, so plain HTTP needs its own catch-all router.
if (! $this->isSwarm()) {
$catchall = $dynamic_conf['http']['routers']['catchall'];
$dynamic_conf['http']['routers']['catchall']['entryPoints'] = ['https'];
unset($catchall['tls']);
$dynamic_conf['http']['routers']['catchall-http'] = array_merge($catchall, ['entryPoints' => ['http']]);
}
return Yaml::dump($dynamic_conf, 12, 2);
}
/**
* True when the catch-all shows the error page: a standalone Traefik or Caddy proxy with
* the default redirect enabled and no redirect URL. Swarm servers keep the plain 503.
*/
public function usesProxyErrorPage(): bool
{
return ! $this->isSwarm()
&& in_array($this->proxyType(), [ProxyTypes::TRAEFIK->value, ProxyTypes::CADDY->value], true)
&& ($this->proxy->redirect_enabled ?? true) !== false
&& blank($this->proxy->redirect_url);
}
public function proxyErrorPagesPath(): string
{
return base_configuration_dir().'/proxy/error-pages';
}
/**
* The custom error page HTML, or the Coolify default page.
*/
public function proxyErrorPageHtml(): string
{
return filled($this->proxy_error_page) ? $this->proxy_error_page : view('proxy.error-page')->render();
}
/**
* Applications and services on this server that are in maintenance mode.
*
* @return Collection<int, Application|Service>
*/
public function maintenanceResources(): Collection
{
if ($this->isSwarm() || ! in_array($this->proxyType(), [ProxyTypes::TRAEFIK->value, ProxyTypes::CADDY->value], true)) {
return collect();
}
return $this->applications()->where('is_maintenance_enabled', true)->values()
->concat($this->services()->where('is_maintenance_enabled', true)->get());
}
/**
* Writes 503.html and starts the error page container, or removes the container when
* neither the catch-all nor a maintenance page uses it. Swarm servers are not supported.
*/
public function setupProxyErrorPageContainer(): void
{
if ($this->isSwarm()) {
return;
}
$container = self::PROXY_ERROR_PAGE_CONTAINER;
if (! $this->usesProxyErrorPage() && $this->maintenanceResources()->isEmpty()) {
instant_remote_process(["docker rm -f $container >/dev/null 2>&1 || true"], $this);
return;
}
$path = $this->proxyErrorPagesPath();
$mount = devHostDockerPath($this, $path);
$restart = RESTART_MODE;
instant_remote_process(["mkdir -p $path"], $this);
$this->uploadProxyFile("$path/503.html", $this->proxyErrorPageHtml());
instant_remote_process([
dockerNetworkEnsureCommand('coolify'),
"docker start $container >/dev/null 2>&1 || docker run -d --name $container --network coolify --restart $restart --label coolify.managed=true -v $mount:/www:ro ".self::PROXY_ERROR_PAGE_IMAGE.' httpd -f -p 80 -h /www',
// A proxy on custom networks only must still reach the error page container.
'docker network connect coolify coolify-proxy >/dev/null 2>&1 || true',
], $this);
}
/**
* Writes $contents to $path on this server with scp. A shell command is limited to 128 KiB,
* so large pages and proxy files cannot go inside one command.
*/
public function uploadProxyFile(string $path, string $contents): void
{
$uploadPath = '/tmp/coolify-upload-'.bin2hex(random_bytes(8));
$localPath = tempnam(sys_get_temp_dir(), 'coolify-proxy-file-');
try {
file_put_contents($localPath, $contents);
instant_scp($localPath, $uploadPath, $this);
} finally {
@unlink($localPath);
}
// The proxy and the page container read the file as root, other users do not need it.
instant_remote_process(["mv -f $uploadPath $path", "chmod 644 $path"], $this);
}
/**
* Writes the maintenance pages and the proxy configuration that shows them instead of the
* applications and services in maintenance mode. Swarm servers are not supported.
*/
public function setupMaintenancePages(): void
{
$proxyType = $this->proxyType();
if ($this->isSwarm() || ! in_array($proxyType, [ProxyTypes::TRAEFIK->value, ProxyTypes::CADDY->value], true)) {
return;
}
$resources = $this->maintenanceResources();
$path = $this->proxyErrorPagesPath();
$keep = $resources->map(fn ($resource) => "! -name '{$resource->maintenancePageFile()}' ")->implode('');
instant_remote_process([
"mkdir -p $path",
"find $path -maxdepth 1 -name 'maintenance-*.html' {$keep}-delete",
], $this);
$this->setupProxyErrorPageContainer();
foreach ($resources as $resource) {
$this->uploadProxyFile("$path/{$resource->maintenancePageFile()}", $resource->maintenancePageHtml());
}
$dynamicPath = $this->proxyPath().'/dynamic';
instant_remote_process(["mkdir -p $dynamicPath"], $this);
if ($proxyType === ProxyTypes::TRAEFIK->value) {
$file = "$dynamicPath/coolify-maintenance.yaml";
$configuration = $this->maintenanceTraefikConfiguration($resources);
if ($configuration === null) {
instant_remote_process(["rm -f $file"], $this);
} else {
$this->uploadProxyFile($file, $configuration);
}
} else {
// caddy-docker-proxy reads the base Caddyfile again on its next poll.
$this->uploadProxyFile("$dynamicPath/Caddyfile", $this->caddyBaseCaddyfile($resources));
}
}
/**
* Traefik routers that use the same rule as the resource routers (from the domains, or
* from custom labels), with a priority one higher. Traefik uses the rule length as the
* default priority, so a resource on a longer path of the same host keeps its own router.
*
* @param Collection<int, Application|Service> $resources
*/
public function maintenanceTraefikConfiguration(Collection $resources): ?string
{
$routers = [];
$middlewares = [];
foreach ($resources as $resource) {
$routes = $resource->maintenanceTraefikRouters();
if ($routes->isEmpty()) {
continue;
}
$middleware = "coolify-maintenance-{$resource->uuid}";
$middlewares[$middleware] = [
'errors' => [
'status' => ['503'],
'service' => 'coolify-maintenance-pages',
'query' => '/'.$resource->maintenancePageFile(),
],
];
foreach ($routes as $route) {
$routers["coolify-maintenance-{$resource->uuid}-{$route['name']}"] = array_filter([
'entryPoints' => $route['entryPoints'],
'rule' => $route['rule'],
'service' => 'coolify-maintenance-noop',
'middlewares' => ['coolify-maintenance-headers', $middleware],
'priority' => $route['priority'],
'tls' => $route['tls'],
], fn ($value) => $value !== null);
}
}
if ($routers === []) {
return null;
}
$middlewares['coolify-maintenance-headers'] = [
'headers' => ['customResponseHeaders' => ['Retry-After' => '300']],
];
return "# This file is generated by Coolify, do not edit it manually.\n\n".Yaml::dump([
'http' => [
'routers' => $routers,
'middlewares' => $middlewares,
'services' => [
// No servers: Traefik answers 503 and the errors middleware adds the page.
'coolify-maintenance-noop' => ['loadBalancer' => ['servers' => []]],
'coolify-maintenance-pages' => [
'loadBalancer' => ['servers' => [['url' => 'http://'.self::PROXY_ERROR_PAGE_CONTAINER.':80']]],
],
],
],
], 12, 2);
}
/**
* The base Caddyfile of caddy-docker-proxy. A maintenance block uses the same site address
* as the resource labels, so caddy-docker-proxy merges both into one site block. Its named
* matcher has the same path as the resource handle, so Caddy runs it first.
*
* @param Collection<int, Application|Service>|null $resources
*/
public function caddyBaseCaddyfile(?Collection $resources = null): string
{
$resources ??= $this->maintenanceResources();
$sites = [];
foreach ($resources as $resource) {
foreach ($resource->maintenanceCaddySites() as $index => $site) {
$matcher = "@coolify-maintenance-{$resource->uuid}-{$index}";
$sites[$site['address']][] = implode("\n", [
"\t{$matcher} path {$site['path']}",
"\thandle {$matcher} {",
"\t\treverse_proxy ".self::PROXY_ERROR_PAGE_CONTAINER.':80 {',
"\t\t\trewrite /{$resource->maintenancePageFile()}",
"\t\t\theader_down Retry-After 300",
"\t\t\thandle_response {",
"\t\t\t\tcopy_response 503",
"\t\t\t}",
"\t\t}",
"\t}",
]);
}
}
$caddyfile = 'import /dynamic/*.caddy';
foreach ($sites as $address => $blocks) {
$caddyfile .= "\n\n{$address} {\n".implode("\n", $blocks)."\n}";
}
return $caddyfile."\n";
}
public function setupDynamicProxyConfiguration()
{
$settings = instanceSettings();
$dynamic_config_path = $this->proxyPath().'/dynamic';
if ($this->proxyType() === ProxyTypes::TRAEFIK->value) {
$file = "$dynamic_config_path/coolify.yaml";
if (empty($settings->fqdn) || (isCloud() && $this->id !== 0) || ! $this->isLocalhost()) {
instant_remote_process([
"rm -f $file",
], $this);
} else {
$url = Url::fromString($settings->fqdn);
$host = $this->dashboardTraefikHost($url->getHost());
$schema = $url->getScheme();
$traefik_dynamic_conf = [
'http' => [
'middlewares' => [
'redirect-to-https' => [
'redirectscheme' => [
'scheme' => 'https',
],
],
'gzip' => [
'compress' => true,
],
],
'routers' => [
'coolify-http' => [
'middlewares' => [
0 => 'gzip',
],
'entryPoints' => [
0 => 'http',
],
'service' => 'coolify',
'rule' => "Host(`{$host}`)",
],
'coolify-reverb-ws' => [
'entryPoints' => [
0 => 'http',
],
'service' => 'coolify-reverb',
'rule' => "Host(`{$host}`) && PathPrefix(`/app/`)",
],
'coolify-terminal-ws' => [
'entryPoints' => [
0 => 'http',
],
'service' => 'coolify-terminal',
'rule' => "Host(`{$host}`) && PathPrefix(`/terminal/ws`)",
],
],
'services' => [
'coolify' => [
'loadBalancer' => [
'servers' => [
0 => [
'url' => 'http://coolify:8080',
],
],
],
],
'coolify-reverb' => [
'loadBalancer' => [
'servers' => [
0 => [
'url' => 'http://coolify:6001',
],
],
],
],
'coolify-terminal' => [
'loadBalancer' => [
'servers' => [
0 => [
'url' => 'http://coolify:6002',
],
],
],
],
],
],
];
if ($schema === 'https') {
$certificateResolver = $this->traefikCertificateResolver();
$tls = $certificateResolver ? ['certresolver' => $certificateResolver] : [];
$traefik_dynamic_conf['http']['routers']['coolify-http']['middlewares'] = $this->dashboardHttpMiddlewares($settings);
$traefik_dynamic_conf['http']['routers']['coolify-https'] = [
'middlewares' => [
0 => 'gzip',
],
'entryPoints' => [
0 => 'https',
],
'service' => 'coolify',
'rule' => "Host(`{$host}`)",
'tls' => $tls,
];
$traefik_dynamic_conf['http']['routers']['coolify-reverb-wss'] = [
'entryPoints' => [
0 => 'https',
],
'service' => 'coolify-reverb',
'rule' => "Host(`{$host}`) && PathPrefix(`/app/`)",
'tls' => $tls,
];
$traefik_dynamic_conf['http']['routers']['coolify-terminal-wss'] = [
'entryPoints' => [
0 => 'https',
],
'service' => 'coolify-terminal',
'rule' => "Host(`{$host}`) && PathPrefix(`/terminal/ws`)",
'tls' => $tls,
];
}
$yaml = Yaml::dump($traefik_dynamic_conf, 12, 2);
$yaml =
"# This file is automatically generated by Coolify.\n".
"# Do not edit it manually (only if you know what are you doing).\n\n".
$yaml;
$base64 = base64_encode($yaml);
instant_remote_process([
"mkdir -p $dynamic_config_path",
"echo '$base64' | base64 -d | tee $file > /dev/null",
], $this);
}
} elseif ($this->proxyType() === 'CADDY') {
$file = "$dynamic_config_path/coolify.caddy";
if (empty($settings->fqdn) || (isCloud() && $this->id !== 0) || ! $this->isLocalhost()) {
instant_remote_process([
"rm -f $file",
], $this);
$this->reloadCaddy();
} else {
$url = Url::fromString($settings->fqdn);
$host = $url->getHost();
$schema = $url->getScheme();
$siteAddress = $this->dashboardCaddySiteAddress($settings, $schema, $host);
$caddy_file = "
$siteAddress {
encode zstd gzip
handle /app/* {
reverse_proxy coolify:6001 {
stream_close_delay ".CADDY_STREAM_CLOSE_DELAY.'
}
}
handle /terminal/ws {
reverse_proxy coolify:6002 {
stream_close_delay '.CADDY_STREAM_CLOSE_DELAY.'
}
}
reverse_proxy coolify:8080
}';
$base64 = base64_encode($caddy_file);
instant_remote_process([
"echo '$base64' | base64 -d | tee $file > /dev/null",
], $this);
$this->reloadCaddy();
}
}
}
public function reloadCaddy()
{
return instant_remote_process([
'docker exec coolify-proxy caddy reload --config /config/caddy/Caddyfile.autosave',
], $this);
}
public function dashboardHttpMiddlewares(InstanceSettings $settings): array
{
if ($settings->is_dashboard_force_https_enabled) {
return ['redirect-to-https'];
}
return ['gzip'];
}
/**
* Traefik matches an IPv6 Host() rule only without brackets (Host(`2a01:4f8::1`)).
*/
public function dashboardTraefikHost(string $host): string
{
return trim($host, '[]');
}
public function dashboardCaddySiteAddress(InstanceSettings $settings, string $schema, string $host): string
{
if ($schema === 'https' && ! $settings->is_dashboard_force_https_enabled) {
return "http://{$host}, https://{$host}";
}
return "{$schema}://{$host}";
}
public function proxyPath()
{
$base_path = config('constants.coolify.base_config_path');
$proxyType = $this->proxyType();
$proxy_path = "$base_path/proxy";
if ($proxyType === ProxyTypes::TRAEFIK->value) {
$proxy_path = $proxy_path.'/';
} elseif ($proxyType === ProxyTypes::CADDY->value) {
$proxy_path = $proxy_path.'/caddy';
} elseif ($proxyType === ProxyTypes::NGINX->value) {
$proxy_path = $proxy_path.'/nginx';
}
return $proxy_path;
}
public function proxyType()
{
return data_get($this->proxy, 'type');
}
public function hasPendingProxyConfiguration(): bool
{
if ($this->proxy->get('status') !== 'running') {
return false;
}
if ($this->proxy->get('certificates_restart_required')) {
return true;
}
$savedSettings = $this->proxy->get('last_saved_settings');
$appliedSettings = $this->proxy->get('last_applied_settings');
return filled($savedSettings) && filled($appliedSettings) && $savedSettings !== $appliedSettings;
}
/**
* Record the configuration the proxy runs with after a start or restart.
*/
public function markProxyConfigurationApplied(string $configuration): void
{
$this->proxy->last_applied_settings = str(base64_encode($configuration))->pipe('md5')->value();
$this->proxy->certificates_restart_required = false;
$this->save();
}
public function hasCurrentTraefikOutdatedInfo(): bool
{
if ($this->proxyType() !== ProxyTypes::TRAEFIK->value) {
return false;
}
$detectedVersion = ltrim((string) $this->detected_traefik_version, 'v');
$storedVersion = ltrim((string) data_get($this->traefik_outdated_info, 'current'), 'v');
$type = data_get($this->traefik_outdated_info, 'type');
return filled($detectedVersion)
&& $storedVersion === $detectedVersion
&& in_array($type, ['patch_update', 'minor_upgrade'], true);
}
public function scopeWithProxy(): Builder
{
return $this->proxy->modelScope();
}
/**
* Servers that this instance manages. A transferred server is only an archive and does not use a server slot.
*/
public function scopeNotTransferredAway(Builder $query): Builder
{
return $query->where(fn (Builder $query) => $query
->whereNull('server_metadata->transfer->status')
->orWhere('server_metadata->transfer->status', '!=', 'transferred'));
}
public function scopeWhereProxyType(Builder $query, string $proxyType): Builder
{
return $query->where('proxy->type', $proxyType);
}
public function isLocalhost()
{
return $this->ip === 'host.docker.internal' || $this->id === 0;
}
/**
* True only in development for the `testing-host` server. That container runs containers on the
* host Docker daemon (/var/run/docker.sock), but its /data/coolify is a Docker named volume. The host
* daemon must therefore mount the volume's host path instead of /data/coolify (see devHostDockerPath()).
*
* Dev KVM VMs and all other servers have their own Docker daemon and their own /data/coolify.
* A `host.docker.internal` server writes to the real host /data/coolify, so it also needs no change.
*/
public function sharesDevHostDocker(): bool
{
// The saving hook can leave a Stringable in `ip`, so compare the string value.
return isDev() && (string) $this->ip === self::DEV_TESTING_HOST_IP;
}
/**
* Usable dedicated (build-only) servers of a team. Servers with the combined role
* host deployments, so they are never picked as build servers. Servers dedicated to
* GitHub Actions runners are also left out.
*/
public static function buildServers($teamId): Builder
{
$query = Server::whereTeamId($teamId)
->whereRelation('settings', 'is_reachable', true)
->whereRelation('settings', 'is_usable', true)
->whereRelation('settings', 'is_swarm_worker', false)
->whereRelation('settings', 'force_disabled', false)
->whereDoesntHave('githubRunnerConfig', fn (Builder $config) => $config->where('is_enabled', true)->where('is_dedicated', true));
return self::whereServerRole($query, ServerRole::BUILD);
}
public function isForceDisabled()
{
return $this->settings->force_disabled;
}
/**
* On self-hosted, only the unreachable cleanup disables a server, so a working connection can enable it again.
* On Cloud the flag enforces the plan server limit, and a transferred server stays an archive.
*/
public function canBeReEnabled(): bool
{
return $this->settings->force_disabled === true && ! isCloud() && ! $this->isTransferredAway();
}
/**
* Server was migrated away from this Coolify instance (source side).
* Must not be revalidated or re-enabled as a live managed host.
*/
public function isTransferredAway(): bool
{
return data_get($this->server_metadata, 'transfer.status') === 'transferred';
}
/**
* Stops an action that changes the host, its cloud server, its Cloudflare tunnel, or its DNS records.
* Another Coolify instance manages a transferred server; this instance keeps only an archive of it.
*
* @throws \RuntimeException
*/
public function assertNotTransferredAway(): void
{
if ($this->isTransferredAway()) {
throw new \RuntimeException('This server is managed by another Coolify instance.');
}
}
/**
* Whether this server may be validated / installed against from this instance.
*/
public function canBeValidated(): bool
{
return ! $this->isTransferredAway();
}
/**
* On Coolify Cloud, limit how often SSH validations can run against one IP and from one team.
* The window and the maximums are server_validation_* instance settings.
* Without a limit, users can make Cloud try SSH logins on hosts that they do not own.
* A successful connection gives its attempt back, so only failed logins use up the limit.
*
* @throws ThrottleRequestsException
*/
public static function throttleValidation(int $teamId, string $ip): void
{
if (! isCloud()) {
return;
}
$limits = self::validationRateLimits($teamId, $ip);
foreach ($limits as $key => $maxAttempts) {
if (RateLimiter::tooManyAttempts($key, $maxAttempts)) {
$seconds = RateLimiter::availableIn($key);
throw new ThrottleRequestsException("Too many server validations. Please try again in {$seconds} seconds.", headers: ['Retry-After' => $seconds]);
}
}
foreach (array_keys($limits) as $key) {
RateLimiter::hit($key, self::validationRateLimitSeconds());
}
}
private static function validationRateLimitSeconds(): int
{
return instanceSettings()->server_validation_rate_limit_seconds;
}
/**
* @return array<string, int> rate limiter key => max attempts
*/
private static function validationRateLimits(int $teamId, string $ip): array
{
$settings = instanceSettings();
return [
'server-validation:ip:'.strtolower(self::normalizeIp($ip)) => $settings->server_validation_max_failures_per_ip,
'server-validation:team:'.$teamId => $settings->server_validation_max_failures_per_team,
];
}
private function refundValidationAttempt(): void
{
if (! isCloud()) {
return;
}
foreach (array_keys(self::validationRateLimits($this->team_id, $this->ip)) as $key) {
if (RateLimiter::attempts($key) > 0) {
RateLimiter::decrement($key, self::validationRateLimitSeconds());
}
}
}
public function forceEnableServer()
{
if ($this->isTransferredAway()) {
return;
}
$this->settings->force_disabled = false;
$this->settings->save();
}
public function forceDisableServer()
{
$this->settings->force_disabled = true;
$this->settings->save();
$sshKeyFileLocation = "id.root@{$this->uuid}";
Storage::disk('ssh-keys')->delete($sshKeyFileLocation);
}
public function sentinelHeartbeat(bool $isReset = false)
{
$this->sentinel_updated_at = $isReset ? now()->subMinutes(6000) : now();
$this->save();
}
/**
* Get the wait time for Sentinel to push before performing an SSH check.
*
* @return int The wait time in seconds.
*/
public function waitBeforeDoingSshCheck(): int
{
$wait = $this->settings->sentinel_push_interval_seconds * 3;
if ($wait < 120) {
$wait = 120;
}
return $wait;
}
public function firstSentinelReportTimeoutSeconds(): int
{
return max(30, $this->settings->sentinel_push_interval_seconds + 30);
}
public static function sentinelReportedVersionCacheKey(int $serverId): string
{
return "sentinel:reported-version:{$serverId}";
}
/**
* The last known reason why Sentinel pushes do not arrive. It is shown while Sentinel is out of sync.
*/
public function sentinelPushProblem(): ?string
{
return Cache::get("sentinel:push-problem:{$this->id}");
}
public function rememberSentinelPushProblem(?string $problem): void
{
if (blank($problem)) {
Cache::forget("sentinel:push-problem:{$this->id}");
return;
}
Cache::put("sentinel:push-problem:{$this->id}", Str::limit($problem, 500), now()->addDay());
}
public function isSentinelLive()
{
return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subSeconds($this->waitBeforeDoingSshCheck()));
}
public function sentinelStatus(): string
{
if ($this->sentinel_waiting_since !== null) {
return $this->sentinel_waiting_since->isAfter(now()->subSeconds($this->firstSentinelReportTimeoutSeconds()))
? 'waiting'
: 'out_of_sync';
}
return $this->isSentinelLive() ? 'in_sync' : 'out_of_sync';
}
public function isSentinelEnabled(): bool
{
return ! $this->isBuildServer()
&& ! $this->isSwarm()
&& ! $this->isForceDisabled()
&& ! $this->isTransferredAway();
}
public function isMetricsEnabled(): bool
{
return $this->settings->is_metrics_enabled;
}
/**
* Traefik certificate resolver for generated HTTPS routers, or null when the proxy uses its own certificates.
* A resource value overrides the server value; "none" turns the resolver off.
*/
public function traefikCertificateResolver(?string $resourceResolver = null): ?string
{
$resolver = filled($resourceResolver) ? $resourceResolver : data_get($this, 'settings.traefik_certificate_resolver');
return filled($resolver) && $resolver !== ServerSetting::TRAEFIK_CERTIFICATE_RESOLVER_NONE ? $resolver : null;
}
public function isTrafficAnalyticsEnabled(): bool
{
return (bool) data_get($this, 'settings.is_traffic_analytics_enabled', false);
}
/**
* Traffic analytics reads the access log of a Coolify-managed Traefik or Caddy proxy.
*/
public function hasTrafficAnalyticsProxy(): bool
{
return in_array($this->proxyType(), [ProxyTypes::TRAEFIK->value, ProxyTypes::CADDY->value], true);
}
/**
* Why traffic analytics cannot be enabled on this server, or null when it can.
*/
public function trafficAnalyticsUnsupportedReason(): ?string
{
if ($this->isSwarm() || $this->isBuildServer()) {
return 'Traffic analytics is not supported on Swarm/Build servers.';
}
if (! $this->hasTrafficAnalyticsProxy()) {
return 'Traffic analytics needs the Traefik or Caddy proxy.';
}
return null;
}
public function supportsTrafficAnalytics(): bool
{
return $this->trafficAnalyticsUnsupportedReason() === null;
}
/**
* Major and minor version from a caddy-docker-proxy image tag, for example [2, 8] for
* `lucaslorentz/caddy-docker-proxy:2.8-alpine`. Other images, `latest`, and digests without a tag give null.
*
* @return array{0: int, 1: int}|null
*/
public static function caddyDockerProxyImageVersion(?string $image): ?array
{
if ($image === null || preg_match('#(?:^|/)caddy-docker-proxy:(\d+)\.(\d+)#', $image, $version) !== 1) {
return null;
}
return [(int) $version[1], (int) $version[2]];
}
/**
* Caddy image in the saved proxy configuration. Null for other proxies or a configuration that cannot be read.
*/
public function configuredCaddyProxyImage(): ?string
{
if ($this->proxyType() !== ProxyTypes::CADDY->value) {
return null;
}
try {
$image = data_get(Yaml::parse((string) $this->proxy->get('last_saved_proxy_configuration')), 'services.caddy.image');
} catch (ParseException) {
return null;
}
return is_string($image) && $image !== '' ? $image : null;
}
/**
* The saved Caddy image when it is caddy-docker-proxy older than 2.9 (Caddy 2.7), else null.
* Unknown versions (custom images, `latest`, digests) are not reported.
*/
public function outdatedCaddyProxyImage(): ?string
{
$image = $this->configuredCaddyProxyImage();
$version = self::caddyDockerProxyImageVersion($image);
return $version !== null && $version < self::MINIMUM_CURRENT_CADDY_PROXY_VERSION ? $image : null;
}
/**
* True when the Caddy proxy runs caddy-docker-proxy 2.9+ (Caddy 2.8+). The 2.8 image (the default before 2.13)
* runs Caddy 2.7.6, which rejects the whole Caddyfile when it contains newer directives. A saved change that
* is not applied yet may still run the old image, so it counts as unsupported.
*/
private function caddyRunsCurrentVersion(): bool
{
if ($this->hasPendingProxyConfiguration()) {
return false;
}
$version = self::caddyDockerProxyImageVersion($this->configuredCaddyProxyImage());
return $version !== null && $version >= self::MINIMUM_CURRENT_CADDY_PROXY_VERSION;
}
/**
* Caddy's `log_append` tags access-log lines with the app UUID for traffic analytics. It needs Caddy 2.8+.
*/
public function caddySupportsLogAppend(): bool
{
return $this->caddyRunsCurrentVersion();
}
/**
* Caddy 2.8 renamed `basicauth` to `basic_auth`. Caddy 2.7 knows only `basicauth`, and Caddy 2.8+ still
* accepts it as a deprecated name, so `basicauth` is the safe fallback.
*/
public function caddySupportsBasicAuthDirective(): bool
{
return $this->caddyRunsCurrentVersion();
}
public function caddySupportsArgon2idBasicAuth(): bool
{
return $this->caddyRunsCurrentVersion()
&& self::caddyDockerProxyImageVersion($this->configuredCaddyProxyImage()) >= [2, 11];
}
public function isServerApiEnabled(): bool
{
return $this->settings->is_sentinel_enabled;
}
public function checkSentinel()
{
CheckAndStartSentinelJob::dispatch($this);
}
public function getDiskUsage(): ?string
{
return instant_remote_process(['df / --output=pcent | tr -cd 0-9'], $this, false);
// return instant_remote_process(["df /| tail -1 | awk '{ print $5}' | sed 's/%//g'"], $this, false);
}
public function definedResources()
{
$applications = $this->applications();
$databases = $this->databases();
$services = $this->services();
return $applications->concat($databases)->concat($services->get());
}
/**
* The resources that deleting this server with "Delete all resources" deletes. An application that has this server
* only as an additional destination runs on another server, so it is detached instead.
*
* @return Collection<int, Model>
*/
public function primaryResources(): Collection
{
return $this->primaryApplications()
->concat($this->databases())
->concat($this->services()->get());
}
/**
* Remove this server from the applications that use it only as an additional destination.
* The applications keep running on their other servers.
*
* @param bool $stopContainers stop and remove the containers of the applications on this server (best effort)
*/
public function detachAdditionalApplications(bool $stopContainers = true): void
{
$applications = $this->additionalApplications();
if ($stopContainers) {
foreach ($applications as $application) {
try {
StopApplicationOneServer::run($application, $this);
} catch (\Throwable $e) {
logger()->warning('Could not stop the containers of an application on a deleted server.', [
'application_id' => $application->id,
'server_id' => $this->id,
'error' => $e->getMessage(),
]);
}
}
}
DB::table('additional_destinations')
->where('server_id', $this->id)
->whereIn('application_id', $applications->pluck('id'))
->delete();
}
public function stopUnmanaged($id)
{
return instant_remote_process([dockerStopCommand(0, escapeshellarg($id), $this)], $this);
}
public function restartUnmanaged($id)
{
return instant_remote_process(['docker restart '.escapeshellarg($id)], $this);
}
public function startUnmanaged($id)
{
return instant_remote_process(['docker start '.escapeshellarg($id)], $this);
}
public function getContainers()
{
$containers = collect([]);
$containerReplicates = collect([]);
if ($this->isSwarm()) {
$containers = instant_remote_process_with_timeout(["docker service inspect $(docker service ls -q) --format '{{json .}}'"], $this, false);
$containers = format_docker_command_output_to_json($containers);
$containerReplicates = instant_remote_process_with_timeout(["docker service ls --format '{{json .}}'"], $this, false);
if ($containerReplicates) {
$containerReplicates = format_docker_command_output_to_json($containerReplicates);
foreach ($containerReplicates as $containerReplica) {
$name = data_get($containerReplica, 'Name');
$containers = $containers->map(function ($container) use ($name, $containerReplica) {
if (data_get($container, 'Spec.Name') === $name) {
$replicas = data_get($containerReplica, 'Replicas');
$running = str($replicas)->explode('/')[0];
$total = str($replicas)->explode('/')[1];
if ($running === $total) {
data_set($container, 'State.Status', 'running');
data_set($container, 'State.Health.Status', 'healthy');
} else {
data_set($container, 'State.Status', 'starting');
data_set($container, 'State.Health.Status', 'unhealthy');
}
}
return $container;
});
}
}
} else {
$containers = instant_remote_process_with_timeout(["docker container inspect $(docker container ls -aq) --format '{{json .}}'"], $this, false);
$containers = format_docker_command_output_to_json($containers);
$containerReplicates = collect([]);
}
return [
'containers' => collect($containers) ?? collect([]),
'containerReplicates' => collect($containerReplicates) ?? collect([]),
];
}
public function loadAllContainers(): Collection
{
if ($this->isFunctional()) {
$containers = instant_remote_process(["docker ps -a --format '{{json .}}'"], $this);
$containers = format_docker_command_output_to_json($containers);
return collect($containers);
}
return collect([]);
}
public function loadUnmanagedContainers(): Collection
{
if ($this->isFunctional()) {
$containers = instant_remote_process(["docker ps -a --format '{{json .}}'"], $this);
$containers = format_docker_command_output_to_json($containers);
$containers = $containers->map(function ($container) {
$labels = data_get($container, 'Labels');
if (! str($labels)->contains('coolify.managed')) {
return $container;
}
return null;
});
$containers = $containers->filter();
return collect($containers);
} else {
return collect([]);
}
}
public function databases()
{
// Get destination IDs for this server in two efficient queries
$standaloneDockerIds = StandaloneDocker::where('server_id', $this->id)->pluck('id');
$swarmDockerIds = SwarmDocker::where('server_id', $this->id)->pluck('id');
$destinationCondition = function ($query) use ($standaloneDockerIds, $swarmDockerIds) {
$query->where(function ($q) use ($standaloneDockerIds) {
$q->where('destination_type', StandaloneDocker::class)
->whereIn('destination_id', $standaloneDockerIds);
})->orWhere(function ($q) use ($swarmDockerIds) {
$q->where('destination_type', SwarmDocker::class)
->whereIn('destination_id', $swarmDockerIds);
});
};
// Query each database type with the destination condition
$postgresqls = StandalonePostgresql::where($destinationCondition)->get();
$redis = StandaloneRedis::where($destinationCondition)->get();
$mongodbs = StandaloneMongodb::where($destinationCondition)->get();
$mysqls = StandaloneMysql::where($destinationCondition)->get();
$mariadbs = StandaloneMariadb::where($destinationCondition)->get();
$keydbs = StandaloneKeydb::where($destinationCondition)->get();
$dragonflies = StandaloneDragonfly::where($destinationCondition)->get();
$clickhouses = StandaloneClickhouse::where($destinationCondition)->get();
$sqlites = StandaloneSqlite::where($destinationCondition)->get();
return $postgresqls
->concat($redis)
->concat($mongodbs)
->concat($mysqls)
->concat($mariadbs)
->concat($keydbs)
->concat($dragonflies)
->concat($clickhouses)
->concat($sqlites)
->reject(fn ($item) => $item instanceof StandalonePostgresql && $item->isInstanceDatabase());
}
/**
* The applications whose primary destination is on this server, plus the applications that use this server as an
* additional destination.
*/
public function applications()
{
$applications = $this->primaryApplications();
// Get additional server applications
$additionalApplicationIds = DB::table('additional_destinations')
->where('server_id', $this->id)
->pluck('application_id');
if ($additionalApplicationIds->isNotEmpty()) {
$additionalApps = Application::whereIn('id', $additionalApplicationIds)->get();
$applications = $applications->concat($additionalApps);
}
return $applications;
}
/**
* The applications whose primary destination is on this server.
*
* @return Collection<int, Application>
*/
public function primaryApplications(): Collection
{
// Get destination IDs for this server in two efficient queries
$standaloneDockerIds = StandaloneDocker::where('server_id', $this->id)->pluck('id');
$swarmDockerIds = SwarmDocker::where('server_id', $this->id)->pluck('id');
// Query all applications in a single query using polymorphic conditions
return Application::where(function ($query) use ($standaloneDockerIds, $swarmDockerIds) {
$query->where(function ($q) use ($standaloneDockerIds) {
$q->where('destination_type', StandaloneDocker::class)
->whereIn('destination_id', $standaloneDockerIds);
})->orWhere(function ($q) use ($swarmDockerIds) {
$q->where('destination_type', SwarmDocker::class)
->whereIn('destination_id', $swarmDockerIds);
});
})->get();
}
/**
* The applications that use this server only as an additional destination. Their primary destination is on
* another server.
*
* @return Collection<int, Application>
*/
public function additionalApplications(): Collection
{
$applicationIds = DB::table('additional_destinations')
->where('server_id', $this->id)
->whereNotIn('application_id', $this->primaryApplications()->pluck('id'))
->pluck('application_id');
return Application::query()->with('destination.server')->whereIn('id', $applicationIds)->get();
}
public function dockerComposeBasedApplications()
{
return $this->applications()->filter(function ($application) {
return data_get($application, 'build_pack') === 'dockercompose';
});
}
public function dockerComposeBasedPreviewDeployments()
{
return $this->previews()->filter(function ($preview) {
$applicationId = data_get($preview, 'application_id');
$application = Application::find($applicationId);
if (! $application) {
return false;
}
return data_get($application, 'build_pack') === 'dockercompose';
});
}
public function services()
{
return $this->hasMany(Service::class);
}
public function port(): Attribute
{
return Attribute::make(
get: function ($value) {
return (int) preg_replace('/[^0-9]/', '', $value);
},
set: function ($value) {
return (int) preg_replace('/[^0-9]/', '', (string) $value);
}
);
}
public function user(): Attribute
{
return Attribute::make(
get: function ($value) {
return preg_replace(ValidationPatterns::INVALID_SERVER_USERNAME_CHARACTERS_PATTERN, '', $value);
},
set: function ($value) {
return preg_replace(ValidationPatterns::INVALID_SERVER_USERNAME_CHARACTERS_PATTERN, '', $value);
}
);
}
public function ip(): Attribute
{
return Attribute::make(
get: function ($value) {
return preg_replace('/[^0-9a-zA-Z.:%-]/', '', $value);
},
set: fn ($value) => self::normalizeIp($value),
);
}
/**
* The form a server IP is stored in: unsafe characters removed (including IPv6 brackets) and
* IPv6 addresses in their short lower-case form (2A01:04F8::0001 -> 2a01:4f8::1).
*/
public static function normalizeIp(?string $value): string
{
return normalizeIpAddress(preg_replace('/[^0-9a-zA-Z.:%-]/', '', (string) $value));
}
/**
* Whether a server of any team already uses this SSH endpoint. The port is part of the endpoint,
* so servers behind one NAT address with different forwarded SSH ports are different servers.
*/
public static function sshEndpointInUse(?string $ip, int|string|null $port, ?int $exceptServerId = null): bool
{
return self::query()
->where('ip', self::normalizeIp($ip))
->where('port', (int) ($port ?? 22))
->when($exceptServerId !== null, fn ($query) => $query->whereKeyNot($exceptServerId))
->exists();
}
public function getIp(): Attribute
{
return Attribute::make(
get: function () {
if (isDev()) {
return '127.0.0.1';
}
if ($this->isLocalhost()) {
return base_ip();
}
return $this->ip;
}
);
}
public function previews()
{
return $this->destinations()->map(function ($standaloneDocker) {
return $standaloneDocker->applications->map(function ($application) {
return $application->previews;
})->flatten();
})->flatten();
}
public function destinations()
{
$standalone_docker = $this->hasMany(StandaloneDocker::class)->get();
$swarm_docker = $this->hasMany(SwarmDocker::class)->get();
return $standalone_docker->concat($swarm_docker);
}
public function githubRunnerConfig()
{
return $this->hasOne(GithubRunnerConfig::class);
}
public function githubRunnerExecutions()
{
return $this->hasMany(GithubRunnerExecution::class);
}
public function hasEnabledGithubRunners(): bool
{
return $this->githubRunnerConfig()->where('is_enabled', true)->exists();
}
public function standaloneDockers()
{
return $this->hasMany(StandaloneDocker::class);
}
public function swarmDockers()
{
return $this->hasMany(SwarmDocker::class);
}
public function privateKey()
{
return $this->belongsTo(PrivateKey::class);
}
public function cloudProviderToken()
{
return $this->belongsTo(CloudProviderToken::class);
}
public function notificationThrottles(): MorphMany
{
return $this->morphMany(NotificationThrottle::class, 'notifiable');
}
/**
* True while an Unreachable notification was sent and no Reachable notification followed.
*/
protected function unreachableNotificationSent(): Attribute
{
return Attribute::get(fn (): bool => NotificationThrottle::wasSent($this, Unreachable::class));
}
public function sslCertificates()
{
return $this->hasMany(SslCertificate::class);
}
public function muxFilename()
{
return 'mux_'.$this->uuid;
}
public function team()
{
return $this->belongsTo(Team::class);
}
/**
* @return array{id?: int, name: string, uuid: string, network: string, server_id: int}
*/
public function defaultStandaloneDockerAttributes(?int $id = null): array
{
$attributes = [
'name' => 'coolify',
'uuid' => new_public_id(),
'network' => 'coolify',
'server_id' => $this->id,
];
if (! is_null($id)) {
$attributes['id'] = $id;
}
return $attributes;
}
public function environment_variables()
{
return $this->hasMany(SharedEnvironmentVariable::class)->where('type', 'server');
}
public function isProxyShouldRun()
{
// TODO: Do we need "|| $this->proxy->force_stop" here?
if ($this->proxyType() === ProxyTypes::NONE->value || $this->isBuildServer()) {
return false;
}
return true;
}
/**
* Marks a proxy start or restart as running, so status checks do not start a second proxy meanwhile.
* ProxyStatusChangedNotification clears the mark when the start or restart finishes.
*/
public function markProxyTransitionStarted(): void
{
Cache::put($this->proxyTransitionCacheKey(), true, now()->addMinutes(5));
}
public function markProxyTransitionFinished(): void
{
Cache::forget($this->proxyTransitionCacheKey());
}
public function isProxyTransitionInProgress(): bool
{
return Cache::has($this->proxyTransitionCacheKey());
}
private function proxyTransitionCacheKey(): string
{
return "proxy-transition:{$this->uuid}";
}
public function skipServer()
{
if ($this->hasPlaceholderIp()) {
return true;
}
if ($this->settings->force_disabled === true) {
return true;
}
return false;
}
public function isFunctional()
{
$isFunctional = data_get($this->settings, 'is_reachable') && data_get($this->settings, 'is_usable') && data_get($this->settings, 'force_disabled') === false && ! $this->hasPlaceholderIp();
if ($isFunctional === false) {
Storage::disk('ssh-mux')->delete($this->muxFilename());
}
return $isFunctional;
}
/**
* Like isFunctional(), but runs a live SSH and Docker check when the server is marked
* unreachable. The flag can be stale after one failed scheduled check.
*/
public function isFunctionalAfterRecheck(): bool
{
if ($this->isFunctional()) {
return true;
}
if ($this->settings->is_reachable || $this->settings->force_disabled || $this->hasPlaceholderIp()) {
return false;
}
(new ServerConnectionCheckJob($this))->handle();
$this->settings->refresh();
return $this->isFunctional();
}
public function isLogDrainEnabled()
{
return $this->isFluentBitLogDrainEnabled() || $this->settings->is_logdrain_cloudwatch_enabled;
}
/**
* Whether the active log drain relies on the coolify-log-drain Fluent Bit container.
*/
public function isFluentBitLogDrainEnabled(): bool
{
return $this->settings->is_logdrain_newrelic_enabled || $this->settings->is_logdrain_axiom_enabled || $this->settings->is_logdrain_custom_enabled;
}
public function validateOS(): bool|Stringable
{
$os_release = instant_remote_process(['cat /etc/os-release'], $this);
return self::supportedOsFromOsRelease($os_release);
}
/**
* Match /etc/os-release against SUPPORTED_OS: ID first, then each ID_LIKE token (e.g. Manjaro: ID=manjaro, ID_LIKE=arch).
*/
public static function supportedOsFromOsRelease(?string $osRelease): bool|Stringable
{
$collectedData = collect(explode("\n", (string) $osRelease))
->mapWithKeys(function ($line) {
$item = str($line)->trim();
return [$item->before('=')->value() => $item->after('=')->lower()->replace(['"', "'"], '')->value()];
});
$candidates = collect([data_get($collectedData, 'ID')])
->merge(explode(' ', (string) data_get($collectedData, 'ID_LIKE')))
->filter();
foreach ($candidates as $candidate) {
$supported = collect(SUPPORTED_OS)->filter(fn ($supportedOs) => str($supportedOs)->contains($candidate));
if ($supported->count() === 1) {
return str($supported->first());
}
}
return false;
}
public function gatherServerMetadata(): ?array
{
if (! $this->isFunctional()) {
return null;
}
try {
$output = instant_remote_process([
'echo "---PRETTY_NAME---" && grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d \'"\' && echo "---ARCH---" && uname -m && echo "---KERNEL---" && uname -r && echo "---CPUS---" && nproc && echo "---MEMORY---" && free -b | awk \'/Mem:/{print $2}\' && echo "---UPTIME_SINCE---" && uptime -s && echo "---DOCKER---" && (docker version --format \'{{.Server.Version}}\' 2>/dev/null || true) && echo "---COMPOSE---" && (docker compose version --short 2>/dev/null || true) && echo "---GPUS---" && (nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits 2>/dev/null || true) && echo "---NVIDIA_TOOLKIT---" && (command -v nvidia-container-runtime-hook nvidia-ctk 2>/dev/null || true)',
], $this, false);
if (! $output) {
return null;
}
$sections = [];
$sectionLines = [];
$currentKey = null;
foreach (explode("\n", trim($output)) as $line) {
$line = trim($line);
if (preg_match('/^---(\w+)---$/', $line, $m)) {
$currentKey = $m[1];
} elseif ($currentKey) {
$sections[$currentKey] = $line;
if ($line !== '') {
$sectionLines[$currentKey][] = $line;
}
}
}
$metadata = [
'os' => $sections['PRETTY_NAME'] ?? 'Unknown',
'arch' => $sections['ARCH'] ?? 'Unknown',
'kernel' => $sections['KERNEL'] ?? 'Unknown',
'cpus' => (int) ($sections['CPUS'] ?? 0),
'memory_bytes' => (int) ($sections['MEMORY'] ?? 0),
'uptime_since' => $sections['UPTIME_SINCE'] ?? null,
'gpus' => self::parseNvidiaSmiGpus($sectionLines['GPUS'] ?? []),
'nvidia_container_toolkit' => ($sectionLines['NVIDIA_TOOLKIT'] ?? []) !== [],
'collected_at' => now()->toIso8601String(),
];
// Merge into the saved row: server_metadata also holds the transfer and migration state.
DB::transaction(function () use ($metadata) {
$locked = self::query()->lockForUpdate()->findOrFail($this->id);
$locked->update(['server_metadata' => array_merge($locked->server_metadata ?? [], $metadata)]);
$this->server_metadata = $locked->server_metadata;
$this->syncOriginalAttribute('server_metadata');
});
try {
$detectedDockerVersion = parseDockerEngineVersion($sections['DOCKER'] ?? null);
if ($detectedDockerVersion !== null) {
$this->rememberDockerVersion($detectedDockerVersion);
}
$detectedComposeVersion = parseDockerEngineVersion($sections['COMPOSE'] ?? null);
if ($detectedComposeVersion !== null) {
$this->rememberComposeVersion($detectedComposeVersion);
}
} catch (\Throwable $e) {
Log::debug('Failed to store server runtime versions', [
'server_id' => $this->id,
'error' => $e->getMessage(),
]);
}
return $metadata;
} catch (\Throwable $e) {
Log::debug('Failed to gather server metadata', [
'server_id' => $this->id,
'error' => $e->getMessage(),
]);
return null;
}
}
/**
* Parse `nvidia-smi --query-gpu=index,uuid,memory.total,name --format=csv,noheader,nounits` lines.
* The name is the last column because it can contain commas.
*
* @param list<string> $lines
* @return list<array{index: string, uuid: string, memory_mb: int, name: string}>
*/
public static function parseNvidiaSmiGpus(array $lines): array
{
$gpus = [];
foreach ($lines as $line) {
$columns = array_map('trim', explode(',', $line, 4));
if (count($columns) !== 4 || ! ctype_digit($columns[0])) {
continue;
}
[$index, $uuid, $memory, $name] = $columns;
$gpus[] = ['index' => $index, 'uuid' => $uuid, 'memory_mb' => (int) $memory, 'name' => $name];
}
return $gpus;
}
/**
* GPUs found when the server details were last collected, or null when they were never checked.
*
* @return list<array{index: string, uuid: string, memory_mb: int, name: string}>|null
*/
public function detectedGpus(): ?array
{
return data_get($this->server_metadata, 'gpus');
}
public function isTerminalEnabled()
{
return $this->settings->is_terminal_enabled ?? false;
}
public function isSwarm()
{
return data_get($this, 'settings.is_swarm_manager') || data_get($this, 'settings.is_swarm_worker');
}
public function isSwarmManager()
{
return data_get($this, 'settings.is_swarm_manager');
}
public function isSwarmWorker()
{
return data_get($this, 'settings.is_swarm_worker');
}
public function serverStatus(): bool
{
if ($this->status() === false) {
return false;
}
if ($this->isFunctional() === false) {
return false;
}
return true;
}
public function status(): bool
{
['uptime' => $uptime] = $this->validateConnection();
if ($uptime === false) {
$this->markResourcesAsExited();
return false;
}
return true;
}
/**
* Mark all resources on this server as exited, because their containers cannot be checked.
*/
public function markResourcesAsExited(): void
{
foreach ($this->applications() as $application) {
$application->status = 'exited';
$application->save();
}
foreach ($this->databases() as $database) {
$database->status = 'exited';
$database->save();
}
foreach ($this->services() as $service) {
$apps = $service->applications()->get();
$dbs = $service->databases()->get();
foreach ($apps as $app) {
$app->status = 'exited';
$app->save();
}
foreach ($dbs as $db) {
$db->status = 'exited';
$db->save();
}
}
}
public function isReachableChanged()
{
$this->refresh();
$unreachableNotificationSent = (bool) $this->unreachable_notification_sent;
$isReachable = (bool) $this->settings->is_reachable;
if ($isReachable === true) {
if ($unreachableNotificationSent === true) {
$this->sendReachableNotification();
}
return;
}
if ($this->unreachable_count >= ServerConnectionCheckJob::UNREACHABLE_THRESHOLD && ! $unreachableNotificationSent) {
$this->sendUnreachableNotification();
}
}
public function sendReachableNotification()
{
if (NotificationThrottle::release($this, Unreachable::class)) {
$this->team->notify(new Reachable($this));
}
}
public function sendUnreachableNotification()
{
NotificationThrottle::sendOnce($this, Unreachable::class, null, fn () => $this->team->notify(new Unreachable($this)));
}
public function validateConnection(bool $justCheckingNewKey = false)
{
// A validation enables a disabled self-hosted server again, but only when the connection works.
$isReEnabling = $this->canBeReEnabled() && ! $this->hasPlaceholderIp();
if ($isReEnabling) {
$this->forceEnableServer();
}
if ($this->skipServer()) {
return ['uptime' => false, 'error' => 'Server skipped.'];
}
try {
// Enforce mode cannot connect without the key, so it reads the key first. Warn mode learns it after the connection.
if (SshMultiplexingHelper::hostKeyCheckingMode($this) === 'enforce') {
SshMultiplexingHelper::learnHostKey($this, evenIfUnreachable: true);
}
// One attempt only: retries would repeat failed logins on a host that may not belong to the user.
instant_remote_process(['ls /'], $this, disableMultiplexing: true, maxAttempts: 1);
SshMultiplexingHelper::learnHostKey($this, evenIfUnreachable: true);
$this->refundValidationAttempt();
if ($this->settings->is_reachable === false) {
$this->settings->is_reachable = true;
$this->settings->save();
ServerReachabilityChanged::dispatch($this);
}
return ['uptime' => true, 'error' => null];
} catch (\Throwable $e) {
if ($isReEnabling) {
$this->forceDisableServer();
}
$error = $e->getMessage();
if ($error === SshMultiplexingHelper::HOST_KEY_CHANGED_MESSAGE && ! $this->recordHostKeyMismatch()) {
$error = 'SSH host key verification failed, but the server still sends the trusted host key.';
}
if (! $justCheckingNewKey && $this->settings->is_reachable === true) {
$this->settings->is_reachable = false;
$this->settings->save();
ServerReachabilityChanged::dispatch($this);
}
return ['uptime' => false, 'error' => $error];
}
}
/**
* ssh reported a host key problem. Returns true when the server sends a host key that is not pinned, and
* saves that key as pending, so that an admin can compare and trust it. Returns false when the server
* still sends the pinned key or the key cannot be read: then it is a normal connection error.
* A key is reported once a day, and the team gets at most one notification an hour, because a load
* balancer in front of several hosts can switch between keys on every connection. While a key is
* pending, the server is scanned at most every ten minutes.
*
* @param string|null $sshErrorOutput The stderr of ssh. When it shows the fingerprint of the pending key, no scan is needed.
*/
public function recordHostKeyMismatch(?string $sshErrorOutput = null): bool
{
$settings = $this->settings;
if (filled($settings->ssh_host_key_pending) && preg_match_all('/SHA256:[A-Za-z0-9+\/]+/', (string) $sshErrorOutput, $matches) > 0) {
$pendingFingerprints = array_map(fn (string $fingerprint) => strtok($fingerprint, ' '), ServerSetting::sshHostKeyFingerprints($settings->ssh_host_key_pending));
if (array_diff($matches[0], $pendingFingerprints) === []) {
return true;
}
}
// Each scan is a failed login on the server (fail2ban), so a pending key is scanned again at most every ten minutes.
$scannedCacheKey = 'ssh_host_key_mismatch_scanned_'.$this->uuid.'_'.hash('sha256', (string) $settings->ssh_host_key);
if (filled($settings->ssh_host_key_pending) && Cache::has($scannedCacheKey)) {
return true;
}
$newKey = SshMultiplexingHelper::scanHostKey($this);
Cache::put($scannedCacheKey, true, now()->addMinutes(10));
if ($newKey === null || array_diff(explode("\n", $newKey), explode("\n", (string) $settings->ssh_host_key)) === []) {
return false;
}
if ($newKey === $settings->ssh_host_key_pending) {
return true;
}
$settings->forceFill(['ssh_host_key_pending' => $newKey])->save();
if (! Cache::add('ssh_host_key_reported_'.$this->uuid.'_'.hash('sha256', $newKey), true, now()->addDay())) {
return true;
}
Log::warning('SSH host key of server changed', ['server_uuid' => $this->uuid, 'mode' => SshMultiplexingHelper::hostKeyCheckingMode($this)]);
auditLog('server.ssh_host_key_changed', [
'team_id' => $this->team_id,
'server_uuid' => $this->uuid,
'server_name' => $this->name,
'fingerprints' => ServerSetting::sshHostKeyFingerprints($newKey),
], 'warning');
if (Cache::add('ssh_host_key_changed_notified_'.$this->uuid, true, now()->addHour())) {
$this->team?->notify(new HostKeyChanged($this));
}
return true;
}
public function installDocker()
{
return InstallDocker::run($this);
}
/**
* Validate that required commands are available on the server.
*
* @return array{success: bool, missing: array<string>, found: array<string>}
*/
public function validatePrerequisites(): array
{
return ValidatePrerequisites::run($this);
}
public function installPrerequisites()
{
return InstallPrerequisites::run($this);
}
public function validateDockerEngine($throwError = false)
{
$dockerBinary = instant_remote_process(['command -v docker'], $this, false, no_sudo: true);
if (is_null($dockerBinary)) {
$this->settings->is_usable = false;
$this->settings->save();
if ($throwError) {
throw new \Exception('Server is not usable. Docker Engine is not installed.');
}
return false;
}
try {
instant_remote_process(['docker version'], $this);
} catch (\Throwable $e) {
$this->settings->is_usable = false;
$this->settings->save();
if ($throwError) {
throw new \Exception('Server is not usable. Docker Engine is not running.');
}
return false;
}
$this->settings->is_usable = true;
$this->settings->save();
$this->validateCoolifyNetwork(isSwarm: false, isBuildServer: $this->isBuildServer());
return true;
}
public function validateDockerCompose($throwError = false)
{
$dockerCompose = instant_remote_process(['docker compose version'], $this, false);
if (is_null($dockerCompose)) {
$this->settings->is_usable = false;
$this->settings->save();
if ($throwError) {
throw new \Exception('Server is not usable. Docker Compose is not installed.');
}
return false;
}
$this->settings->is_usable = true;
$this->settings->save();
return true;
}
public function validateDockerSwarm()
{
$swarmStatus = instant_remote_process(['docker info|grep -i swarm'], $this, false);
$swarmStatus = str($swarmStatus)->trim()->after(':')->trim();
if ($swarmStatus === 'inactive') {
throw new \Exception('Docker Swarm is not initiated. Please join the server to a swarm before continuing.');
return false;
}
$this->settings->is_usable = true;
$this->settings->save();
$this->validateCoolifyNetwork(isSwarm: true);
return true;
}
public function dockerVersion(): ?string
{
return $this->settings?->docker_version;
}
public function rememberDockerVersion(?string $version): void
{
$this->settings->update([
'docker_version' => parseDockerEngineVersion($version),
'docker_version_checked_at' => now(),
]);
}
public function composeVersion(): ?string
{
return $this->settings?->compose_version;
}
public function rememberComposeVersion(?string $version): void
{
$this->settings->update([
'compose_version' => parseDockerEngineVersion($version),
'compose_version_checked_at' => now(),
]);
}
public function validateDockerEngineVersion()
{
$dockerVersionRaw = instant_remote_process(['docker version --format json'], $this, false);
$dockerVersion = dockerEngineVersionFromJson($dockerVersionRaw);
$this->rememberDockerVersion($dockerVersion);
$composeVersionRaw = instant_remote_process(['docker compose version --short'], $this, false);
$this->rememberComposeVersion(is_string($composeVersionRaw) ? $composeVersionRaw : null);
if (is_null(checkMinimumDockerEngineVersion($dockerVersion))) {
$this->settings->is_usable = false;
$this->settings->save();
return false;
}
$this->settings->is_reachable = true;
$this->settings->is_usable = true;
$this->settings->save();
ServerReachabilityChanged::dispatch($this);
return true;
}
/**
* Explains why validateDockerEngineVersion() failed: the version could not be read, or it is too old.
*/
public function dockerEngineVersionError(): string
{
$requiredVersion = str(config('constants.docker.minimum_required_version'))->before('.');
$documentation = '<a target="_blank" class="underline" href="https://docs.docker.com/engine/install/#server">documentation</a>';
$dockerVersion = $this->dockerVersion();
if (is_null($dockerVersion)) {
$command = $this->isNonRoot() ? 'sudo docker version --format json' : 'docker version --format json';
$user = e($this->user);
return "Coolify could not read the Docker Engine version. Run \"{$command}\" on the server as the {$user} user. It must exit without an error and print only JSON. Text from shell startup files (for example ~/.bashrc) breaks the output.";
}
return "Docker Engine {$dockerVersion} is installed, but Coolify needs version {$requiredVersion} or newer. Please upgrade Docker: {$documentation}.";
}
public function validateCoolifyNetwork($isSwarm = false, $isBuildServer = false)
{
if ($isBuildServer) {
return;
}
if ($isSwarm) {
return instant_remote_process(['docker network create --attachable --driver overlay coolify-overlay >/dev/null 2>&1 || true'], $this, false);
} else {
return instant_remote_process(['docker network create coolify --attachable >/dev/null 2>&1 || true'], $this, false);
}
}
public function isNonRoot()
{
if ($this->user instanceof Stringable) {
return $this->user->value() !== 'root';
}
return $this->user !== 'root';
}
public function isBuildServer()
{
return $this->settings->effectiveServerRole() === ServerRole::BUILD;
}
public function canBuildApplications(): bool
{
return $this->settings->effectiveServerRole()->canBuild();
}
public static function createWithPrivateKey(array $data, PrivateKey $privateKey)
{
$server = new self($data);
$server->privateKey()->associate($privateKey);
$server->save();
return $server;
}
public function updateWithPrivateKey(array $data, ?PrivateKey $privateKey = null)
{
$this->update($data);
if ($privateKey) {
$this->privateKey()->associate($privateKey);
$this->save();
}
return $this;
}
public function storageCheck(): ?string
{
$commands = [
'df / --output=pcent | tr -cd 0-9',
];
return instant_remote_process($commands, $this, false);
}
public function isIpv6(): bool
{
return str($this->ip)->contains(':');
}
public function restartSentinel(?string $customImage = null, bool $async = true)
{
try {
if ($async) {
StartSentinel::dispatch($this, true, null, $customImage);
} else {
StartSentinel::run($this, true, null, $customImage);
}
} catch (\Throwable $e) {
return handleError($e);
}
}
public function url()
{
return base_url().'/server/'.$this->uuid;
}
public function restartContainer(string $containerName)
{
return instant_remote_process(['docker restart '.escapeshellarg($containerName)], $this, false);
}
public function changeProxy(string $proxyType, bool $async = true)
{
$validProxyTypes = collect(ProxyTypes::cases())->map(function ($proxyType) {
return str($proxyType->value)->lower();
});
if ($validProxyTypes->contains(str($proxyType)->lower())) {
$previousProxyType = $this->proxyType();
$this->proxy->set('type', str($proxyType)->upper());
$this->proxy->set('status', 'exited');
$this->proxy->set('last_saved_proxy_configuration', null);
$this->proxy->set('last_saved_settings', null);
$this->proxy->set('last_applied_settings', null);
$this->proxy->set('certificates_restart_required', false);
$this->detected_traefik_version = null;
$this->traefik_outdated_info = null;
$this->save();
if ($this->proxySet()) {
if ($async) {
StartProxy::dispatch($this);
} else {
StartProxy::run($this);
}
}
if ($previousProxyType !== $this->proxyType() && $this->shouldRestartSentinelForTrafficAnalytics()) {
// Sentinel keeps the traffic log mount, path and log format of the old proxy until it is recreated.
$this->restartSentinel();
}
} else {
throw new \Exception('Invalid proxy type.');
}
}
/**
* Sentinel reads the proxy access log only when it runs and traffic analytics is on.
* The raw setting is used, because a switch to a proxy without analytics support must also drop the old log mount.
*/
private function shouldRestartSentinelForTrafficAnalytics(): bool
{
return (bool) $this->settings->is_sentinel_enabled
&& $this->isSentinelEnabled()
&& $this->isTrafficAnalyticsEnabled();
}
public function isEmpty()
{
return $this->applications()->count() == 0 &&
$this->databases()->count() == 0 &&
$this->services()->count() == 0;
}
/**
* Return the server's CA certificate, generating it first when it does not exist yet.
*/
public function ensureCaCertificate(): ?SslCertificate
{
$caCertificate = $this->sslCertificates()->where('is_ca_certificate', true)->first();
if ($caCertificate) {
return $caCertificate;
}
$this->generateCaCertificate();
return $this->sslCertificates()->where('is_ca_certificate', true)->first();
}
public function generateCaCertificate()
{
try {
SslHelper::generateSslCertificate(
commonName: 'Coolify CA Certificate',
serverId: $this->id,
isCaCertificate: true,
validityDays: 10 * 365
);
$caCertificate = $this->sslCertificates()->where('is_ca_certificate', true)->first();
if ($caCertificate) {
$certificateContent = $caCertificate->ssl_certificate;
$commands = SslHelper::caCertificateFileCommands($certificateContent);
instant_remote_process($commands, $this, false);
dispatch(new RegenerateSslCertJob(
server_id: $this->id,
force_regeneration: true
));
}
} catch (\Throwable $e) {
return handleError($e);
}
}
}