app/Jobs/PushServerUpdateJob.php
Rama main · réplica de lectura del repositorio original.
<?php
namespace App\Jobs;
use App\Actions\Application\StopApplication;
use App\Actions\Application\StopApplicationPreview;
use App\Actions\Database\StartDatabaseProxy;
use App\Actions\Database\StopDatabaseProxy;
use App\Actions\Proxy\CheckProxy;
use App\Actions\Proxy\StartProxy;
use App\Actions\Server\StartLogDrain;
use App\Actions\Service\StopServiceApplication;
use App\Actions\Shared\ComplexStatusCheck;
use App\Events\ServiceChecked;
use App\Models\Application;
use App\Models\ApplicationPreview;
use App\Models\NotificationThrottle;
use App\Models\Server;
use App\Models\ServiceApplication;
use App\Models\ServiceDatabase;
use App\Models\StandaloneClickhouse;
use App\Models\StandaloneDocker;
use App\Models\StandaloneDragonfly;
use App\Models\StandaloneKeydb;
use App\Models\StandaloneMariadb;
use App\Models\StandaloneMongodb;
use App\Models\StandaloneMysql;
use App\Models\StandalonePostgresql;
use App\Models\StandaloneRedis;
use App\Models\StandaloneSqlite;
use App\Models\SwarmDocker;
use App\Notifications\Application\RestartLimitReached as ApplicationRestartLimitReached;
use App\Notifications\Container\ContainerRestarted;
use App\Notifications\Server\HighDiskUsage;
use App\Services\ContainerStatusAggregator;
use App\Services\RestartCountTracker;
use App\Traits\CalculatesExcludedStatus;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
{
use CalculatesExcludedStatus;
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 30;
public Collection $containers;
public Collection $applications;
public Collection $previews;
public Collection $databases;
public Collection $services;
public Collection $applicationsById;
public Collection $previewsByKey;
public Collection $databasesByUuid;
public Collection $servicesById;
public Collection $serviceApplicationsById;
public Collection $serviceDatabasesById;
public Collection $allApplicationIds;
public Collection $allDatabaseUuids;
public Collection $allTcpProxyUuids;
public Collection $allServiceApplicationIds;
public Collection $allApplicationPreviewsIds;
public Collection $allServiceDatabaseIds;
public Collection $allApplicationsWithAdditionalServers;
public Collection $foundApplicationIds;
public Collection $foundDatabaseUuids;
public Collection $foundServiceApplicationIds;
public Collection $foundServiceDatabaseIds;
public Collection $foundApplicationPreviewsIds;
public Collection $applicationContainerStatuses;
public Collection $applicationContainerRestartCounts;
public Collection $serviceContainerStatuses;
public Collection $previewContainerRestartCounts;
public Collection $serviceContainerRestartCounts;
public bool $foundProxy = false;
public bool $foundLogDrainContainer = false;
public array $logDrainContainerLabels = [];
private ?array $cachedDestinationIds = null;
public function middleware(): array
{
return [(new WithoutOverlapping('push-server-update-'.$this->server->uuid))->expireAfter(30)->dontRelease()];
}
public function backoff(): int
{
return isDev() ? 1 : 3;
}
public function __construct(public Server $server, public $data)
{
$this->onQueue(sentinel_queue());
$this->containers = collect();
$this->foundApplicationIds = collect();
$this->foundDatabaseUuids = collect();
$this->foundServiceApplicationIds = collect();
$this->foundApplicationPreviewsIds = collect();
$this->foundServiceDatabaseIds = collect();
$this->applicationContainerStatuses = collect();
$this->applicationContainerRestartCounts = collect();
$this->serviceContainerStatuses = collect();
$this->previewContainerRestartCounts = collect();
$this->serviceContainerRestartCounts = collect();
$this->allApplicationIds = collect();
$this->allDatabaseUuids = collect();
$this->allTcpProxyUuids = collect();
$this->allServiceApplicationIds = collect();
$this->allServiceDatabaseIds = collect();
$this->applicationsById = collect();
$this->previewsByKey = collect();
$this->databasesByUuid = collect();
$this->servicesById = collect();
$this->serviceApplicationsById = collect();
$this->serviceDatabasesById = collect();
}
public function handle()
{
// Defensive initialization for Collection properties to handle queue deserialization edge cases
$this->serviceContainerStatuses ??= collect();
$this->previewContainerRestartCounts ??= collect();
$this->serviceContainerRestartCounts ??= collect();
$this->applicationContainerStatuses ??= collect();
$this->applicationContainerRestartCounts ??= collect();
$this->foundApplicationIds ??= collect();
$this->foundDatabaseUuids ??= collect();
$this->foundServiceApplicationIds ??= collect();
$this->foundApplicationPreviewsIds ??= collect();
$this->foundServiceDatabaseIds ??= collect();
$this->allApplicationIds ??= collect();
$this->allDatabaseUuids ??= collect();
$this->allTcpProxyUuids ??= collect();
$this->allServiceApplicationIds ??= collect();
$this->allServiceDatabaseIds ??= collect();
$this->applicationsById ??= collect();
$this->previewsByKey ??= collect();
$this->databasesByUuid ??= collect();
$this->servicesById ??= collect();
$this->serviceApplicationsById ??= collect();
$this->serviceDatabasesById ??= collect();
// Eager-load relations the job touches repeatedly to avoid lazy-load queries
// (settings: disk threshold, isProxyShouldRun, isLogDrainEnabled; team: notifications).
$this->server->loadMissing(['settings', 'team']);
// TODO: Swarm is not supported yet
if (! $this->data) {
throw new \Exception('No data provided');
}
$data = collect($this->data);
// Heartbeat is updated by SentinelController on every push, before dispatch.
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
// Only dispatch the storage check when disk usage is at/above the notification
// threshold AND the value changed. Below the threshold ServerStorageCheckJob
// has nothing to do (it only sends a HighDiskUsage notification), so dispatching
// it is wasted work — and most servers sit well below the threshold.
$diskThreshold = data_get($this->server, 'settings.server_disk_usage_notification_threshold', 80);
$storageCacheKey = 'storage-check:'.$this->server->id;
// Set while usage was high, so the throttle is released only once when usage recovers.
$highUsageCacheKey = 'storage-high:'.$this->server->id;
$lastPercentage = Cache::get($storageCacheKey);
if ($filesystemUsageRoot !== null
&& $filesystemUsageRoot >= $diskThreshold
&& (string) $lastPercentage !== (string) $filesystemUsageRoot) {
Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
Cache::forever($highUsageCacheKey, true);
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
} elseif ($filesystemUsageRoot !== null && $filesystemUsageRoot < $diskThreshold) {
Cache::forget($storageCacheKey);
// The storage check does not run below the threshold, so the next spike alerts again only
// when the throttle is released here.
if (HighDiskUsage::hasRecovered($filesystemUsageRoot, $diskThreshold) && Cache::pull($highUsageCacheKey)) {
NotificationThrottle::release($this->server, HighDiskUsage::class);
}
}
if ($this->containers->isEmpty() && ! $this->isCompleteSnapshot()) {
return;
}
$this->applications = $this->loadApplications();
$this->databases = $this->loadDatabases();
$this->previews = $this->loadPreviews();
$this->services = $this->loadServices();
$this->applicationsById = $this->applications->keyBy(fn ($application) => (string) $application->id);
$this->previewsByKey = $this->previews->keyBy(fn ($preview) => $preview->application_id.':'.$preview->pull_request_id);
$this->databasesByUuid = $this->databases->keyBy('uuid');
$this->servicesById = $this->services->keyBy(fn ($service) => (string) $service->id);
$this->serviceApplicationsById = $this->services->flatMap(fn ($service) => $service->applications)->keyBy(fn ($application) => (string) $application->id);
$this->serviceDatabasesById = $this->services->flatMap(fn ($service) => $service->databases)->keyBy(fn ($database) => (string) $database->id);
$this->allApplicationIds = $this->applications->filter(function ($application) {
return $application->additional_servers_count === 0;
})->pluck('id');
$this->allApplicationsWithAdditionalServers = $this->applications->filter(function ($application) {
return $application->additional_servers_count > 0;
});
$this->allApplicationPreviewsIds = $this->previews->map(function ($preview) {
return $preview->application_id.':'.$preview->pull_request_id;
});
$this->allDatabaseUuids = $this->databases->pluck('uuid');
$this->allTcpProxyUuids = $this->databases->where('is_public', true)->pluck('uuid');
$this->allServiceApplicationIds = $this->serviceApplicationsById->keys();
$this->allServiceDatabaseIds = $this->serviceDatabasesById->keys();
$upResourceIds = NotifyResourceStoppedJob::upResourceIds($this->applications->whereIn('id', $this->allApplicationIds)
->concat($this->previews)
->concat($this->databases)
->concat($this->serviceApplicationsById)
->concat($this->serviceDatabasesById));
// Owners of application containers outside this server's applications, in one query.
$foreignApplicationIdsByUuid = containerApplicationIdsByUuid(
$this->applications,
$this->containers
->map(fn ($container) => collect(data_get($container, 'labels')))
->filter(fn (Collection $labels) => $labels->has('coolify.managed') && isContainerOfType($labels, 'application'))
);
foreach ($this->containers as $container) {
$containerStatus = data_get($container, 'state', 'exited');
$rawHealthStatus = data_get($container, 'health_status');
$containerHealth = $rawHealthStatus ?? 'unknown';
// Only append health status if container is not exited
if ($containerStatus !== 'exited') {
$containerStatus = "$containerStatus:$containerHealth";
}
$labels = collect(data_get($container, 'labels'));
$coolify_managed = $labels->has('coolify.managed');
if (! $coolify_managed) {
continue;
}
if (filter_var($labels->get('com.docker.compose.oneoff'), FILTER_VALIDATE_BOOLEAN)) {
continue;
}
$name = data_get($container, 'name');
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
$this->foundLogDrainContainer = true;
$this->logDrainContainerLabels = $labels->all();
}
// Containers are matched by owner UUID; numeric ids differ between instances.
if (isContainerOfType($labels, 'application')) {
$applicationId = resolveContainerApplicationId($this->applications, $labels, $foreignApplicationIdsByUuid);
if ($applicationId === null) {
continue;
}
$applicationId = (string) $applicationId;
$pullRequestId = $labels->get('coolify.pullRequestId', '0');
try {
if ($pullRequestId === '0') {
$application = $this->applicationsById->get((string) $applicationId);
if ($application && $application->container_present !== true) {
$application->update(['container_present' => true]);
}
if ($this->allApplicationIds->contains($applicationId)) {
$this->foundApplicationIds->push($applicationId);
}
// Store container status for aggregation
if (! $this->applicationContainerStatuses->has($applicationId)) {
$this->applicationContainerStatuses->put($applicationId, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
$restartCount = data_get($container, 'restart_count');
if (is_numeric($restartCount)) {
if (! $this->applicationContainerRestartCounts->has($applicationId)) {
$this->applicationContainerRestartCounts->put($applicationId, collect());
}
$this->applicationContainerRestartCounts->get($applicationId)->put($containerName, (int) $restartCount);
}
}
} else {
$previewKey = $applicationId.':'.$pullRequestId;
if ($this->allApplicationPreviewsIds->contains($previewKey)) {
$this->foundApplicationPreviewsIds->push($previewKey);
}
$this->updateApplicationPreviewStatus($applicationId, $pullRequestId, $containerStatus);
$restartCount = data_get($container, 'restart_count');
if (is_numeric($restartCount)) {
$this->previewContainerRestartCounts->push([
'key' => $previewKey,
'count' => (int) $restartCount,
]);
}
}
} catch (\Exception $e) {
}
} elseif (isContainerOfType($labels, 'service')) {
[$service, $subType, $servicePart] = resolveServiceContainerOwner($this->services, $labels);
if (! $service || ! $servicePart) {
continue;
}
$serviceId = (string) $service->id;
$subId = (string) $servicePart->id;
if ($subType === 'application') {
$this->foundServiceApplicationIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count'));
}
} elseif ($subType === 'database') {
$this->foundServiceDatabaseIds->push($subId);
// Store container status for aggregation
$key = $serviceId.':'.$subType.':'.$subId;
if (! $this->serviceContainerStatuses->has($key)) {
$this->serviceContainerStatuses->put($key, collect());
}
$containerName = $labels->get('com.docker.compose.service');
if ($containerName) {
$this->serviceContainerStatuses->get($key)->put($containerName, $containerStatus);
$this->storeServiceRestartCount($key, $containerName, data_get($container, 'restart_count'));
}
}
} else {
$uuid = $labels->get('com.docker.compose.service');
$type = $labels->get('coolify.type');
if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) {
$this->foundProxy = true;
} elseif ($type === 'service' && $this->isRunning($containerStatus)) {
} else {
if ($this->allDatabaseUuids->contains($uuid) && $this->isActiveOrTransient($containerStatus)) {
$this->foundDatabaseUuids->push($uuid);
// TCP proxy should only be started/managed when database is actually running
if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) {
$this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: true);
} else {
$this->updateDatabaseStatus($uuid, $containerStatus, data_get($container, 'restart_count'), tcpProxy: false);
}
}
}
}
}
if (! $this->isCompleteSnapshot()) {
ServiceChecked::dispatch($this->server->team_id);
return;
}
$this->updateProxyStatus();
Application::whereIn('id', $this->foundApplicationIds->unique())
->update(['container_present' => true]);
$this->updateNotFoundApplicationStatus();
$this->updateNotFoundApplicationPreviewStatus();
$this->updateNotFoundDatabaseStatus();
$this->updateNotFoundServiceStatus();
$this->updateAdditionalServersStatus();
$this->trackPreviewRestartCounts();
// Aggregate multi-container application statuses
$this->aggregateMultiContainerStatuses();
// Aggregate multi-container service statuses
$this->aggregateServiceContainerStatuses();
NotifyResourceStoppedJob::dispatchForStopped($upResourceIds);
$this->checkLogDrainContainer();
ServiceChecked::dispatch($this->server->team_id);
}
private function isCompleteSnapshot(): bool
{
return data_get($this->data, 'snapshot.complete', true) !== false;
}
private function loadApplications(): Collection
{
[$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
$applications = ($standaloneDockerIds->isNotEmpty() || $swarmDockerIds->isNotEmpty())
? Application::withoutGlobalScope('withRelations')
->select([
'id',
'uuid',
'name',
'status',
'container_present',
'build_pack',
'docker_compose_raw',
'environment_id',
'destination_id',
'destination_type',
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
])
->withCount('additional_servers')
->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
->get()
: collect();
$additionalApplicationIds = DB::table('additional_destinations')
->where('server_id', $this->server->id)
->pluck('application_id');
if ($additionalApplicationIds->isNotEmpty()) {
$applications = $applications->concat(
Application::withoutGlobalScope('withRelations')
->select([
'id',
'uuid',
'name',
'status',
'container_present',
'build_pack',
'docker_compose_raw',
'environment_id',
'destination_id',
'destination_type',
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
])
->withCount('additional_servers')
->whereIn('id', $additionalApplicationIds)
->get()
);
}
return $applications->unique('id')->values();
}
private function loadPreviews(): Collection
{
$applicationIds = $this->applications->pluck('id');
if ($applicationIds->isEmpty()) {
return collect();
}
return ApplicationPreview::query()
->select([
'id',
'application_id',
'pull_request_id',
'status',
'last_online_at',
'restart_count',
'max_restart_count',
'restart_limit_reached',
'last_restart_at',
'last_restart_type',
])
->whereIn('application_id', $applicationIds)
->get();
}
private function loadServices(): Collection
{
return $this->server->services()
->select([
'id',
'server_id',
'uuid',
'docker_compose_raw',
])
->with([
'applications:id,uuid,name,service_id,status,last_online_at,restart_count,max_restart_count,restart_limit_reached,last_restart_at,last_restart_type',
'databases:id,uuid,name,service_id,status,last_online_at,is_public',
])
->get();
}
private function loadDatabases(): Collection
{
[$standaloneDockerIds, $swarmDockerIds] = $this->serverDestinationIds();
if ($standaloneDockerIds->isEmpty() && $swarmDockerIds->isEmpty()) {
return collect();
}
$databaseColumns = [
'id',
'uuid',
'name',
'status',
'is_public',
'destination_id',
'destination_type',
'last_online_at',
'restart_count',
'last_restart_at',
'last_restart_type',
];
return collect([
StandalonePostgresql::class,
StandaloneRedis::class,
StandaloneMongodb::class,
StandaloneMysql::class,
StandaloneMariadb::class,
StandaloneKeydb::class,
StandaloneDragonfly::class,
StandaloneClickhouse::class,
StandaloneSqlite::class,
])->flatMap(function (string $databaseClass) use ($databaseColumns, $standaloneDockerIds, $swarmDockerIds) {
return $databaseClass::query()
->select($databaseColumns)
->where(fn ($query) => $this->scopeDestination($query, $standaloneDockerIds, $swarmDockerIds))
->get();
})->reject(fn ($database) => $database instanceof StandalonePostgresql && $database->isInstanceDatabase())->values();
}
private function serverDestinationIds(): array
{
if ($this->cachedDestinationIds !== null) {
return $this->cachedDestinationIds;
}
return $this->cachedDestinationIds = [
StandaloneDocker::where('server_id', $this->server->id)->pluck('id'),
SwarmDocker::where('server_id', $this->server->id)->pluck('id'),
];
}
private function scopeDestination($query, Collection $standaloneDockerIds, Collection $swarmDockerIds): void
{
$query->where(function ($query) use ($standaloneDockerIds) {
$query->where('destination_type', StandaloneDocker::class)
->whereIn('destination_id', $standaloneDockerIds);
})->orWhere(function ($query) use ($swarmDockerIds) {
$query->where('destination_type', SwarmDocker::class)
->whereIn('destination_id', $swarmDockerIds);
});
}
private function aggregateMultiContainerStatuses()
{
if ($this->applicationContainerStatuses->isEmpty()) {
return;
}
foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
$application = $this->applicationsById->get((string) $applicationId);
if (! $application) {
continue;
}
$maxRestartCount = 0;
$restartCountsAvailable = $this->applicationContainerRestartCounts->has($applicationId);
if ($restartCountsAvailable) {
$maxRestartCount = $this->applicationContainerRestartCounts->get($applicationId)->max() ?? 0;
$restartState = (new RestartCountTracker)->evaluate(
previousRestartCount: $application->restart_count ?? 0,
observedRestartCount: $maxRestartCount,
maxRestartCount: $application->max_restart_count ?? 0,
);
if ($restartState['restart_count_changed']) {
$hasCrashRestarts = $restartState['restart_count'] > 0;
$application->update([
'restart_count' => $restartState['restart_count'],
'last_restart_at' => $hasCrashRestarts ? now() : null,
'last_restart_type' => $hasCrashRestarts ? 'crash' : null,
]);
}
if ($restartState['restart_limit_reached']) {
$restartLimitClaimed = Application::query()
->whereKey($application->getKey())
->where('restart_limit_reached', false)
->update(['restart_limit_reached' => true]) === 1;
// Stop again while a container still runs: an earlier stop can fail or come too late.
$stillRunning = $containerStatuses->contains(fn (string $status): bool => ! str($status)->contains('exited'));
if ($restartLimitClaimed || $stillRunning) {
$application->refresh();
StopApplication::dispatch(
application: $application,
previewDeployments: false,
dockerCleanup: false,
resetRestartCount: false,
removeContainers: false,
);
}
if ($restartLimitClaimed) {
$application->environment->project->team?->notify(new ApplicationRestartLimitReached($application));
}
}
}
if ($application->stoppedAfterRestartLimit() && $containerStatuses->every(
fn (string $status): bool => str($status)->contains('exited')
)) {
$application->update(['status' => 'exited']);
continue;
}
// Parse docker compose to check for excluded containers
$dockerComposeRaw = data_get($application, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
// Use preserveRestarting: true so applications show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $maxRestartCount, preserveRestarting: true);
// Update application status with aggregated result
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
$application->status = $aggregatedStatus;
$application->save();
}
}
}
private function aggregateServiceContainerStatuses()
{
if ($this->serviceContainerStatuses->isEmpty()) {
return;
}
foreach ($this->serviceContainerStatuses as $key => $containerStatuses) {
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
if (empty($subId)) {
continue;
}
$service = $this->servicesById->get((string) $serviceId);
if (! $service) {
continue;
}
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
$subResource = $this->serviceApplicationsById->get((string) $subId);
} elseif ($subType === 'database') {
$subResource = $this->serviceDatabasesById->get((string) $subId);
}
if (! $subResource) {
continue;
}
$restartCount = $this->serviceContainerRestartCounts->get($key)?->max() ?? 0;
if (! $subResource instanceof ServiceDatabase && $subResource->trackRestartCount($restartCount)) {
StopServiceApplication::dispatch($subResource, false, false);
$subResource->team()?->notify(new ApplicationRestartLimitReached($subResource));
continue;
}
// Parse docker compose from service to check for excluded containers
$dockerComposeRaw = data_get($service, 'docker_compose_raw');
$excludedContainers = $this->getExcludedContainersFromDockerCompose($dockerComposeRaw);
// Filter out excluded containers
$relevantStatuses = $containerStatuses->filter(function ($status, $containerName) use ($excludedContainers) {
return ! $excludedContainers->contains($containerName);
});
// If all containers are excluded, calculate status from excluded containers
if ($relevantStatuses->isEmpty()) {
$aggregatedStatus = $this->calculateExcludedStatusFromStrings($containerStatuses);
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
continue;
}
// Use ContainerStatusAggregator service for state machine logic
// Use preserveRestarting: true so individual sub-resources show "Restarting" instead of "Degraded"
$aggregator = new ContainerStatusAggregator;
$aggregatedStatus = $aggregator->aggregateFromStrings($relevantStatuses, $restartCount, preserveRestarting: true);
// Update service sub-resource status with aggregated result
if ($aggregatedStatus && $subResource->status !== $aggregatedStatus) {
$subResource->status = $aggregatedStatus;
$subResource->save();
}
}
}
private function updateApplicationStatus(string $applicationId, string $containerStatus)
{
$application = $this->applicationsById->get((string) $applicationId);
if (! $application) {
return;
}
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
$application->save();
}
}
private function updateApplicationPreviewStatus(string $applicationId, string $pullRequestId, string $containerStatus)
{
$application = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
if (! $application) {
return;
}
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
$application->save();
}
}
private function updateNotFoundApplicationStatus()
{
$notFoundApplicationIds = $this->allApplicationIds->diff($this->foundApplicationIds);
if ($notFoundApplicationIds->isEmpty()) {
return;
}
// Batch update: mark all not-found applications as exited (excluding already exited ones)
Application::whereIn('id', $notFoundApplicationIds)
->update([
'status' => 'exited',
'container_present' => false,
'restart_limit_reached' => false,
]);
}
private function updateNotFoundApplicationPreviewStatus()
{
$notFoundApplicationPreviewsIds = $this->allApplicationPreviewsIds->diff($this->foundApplicationPreviewsIds);
if ($notFoundApplicationPreviewsIds->isEmpty()) {
return;
}
// Collect IDs of previews that need to be marked as exited
$previewIdsToUpdate = collect();
foreach ($notFoundApplicationPreviewsIds as $previewKey) {
// Parse the previewKey format "application_id:pull_request_id"
$parts = explode(':', $previewKey);
if (count($parts) !== 2) {
continue;
}
$applicationId = $parts[0];
$pullRequestId = $parts[1];
$applicationPreview = $this->previewsByKey->get($applicationId.':'.$pullRequestId);
if ($applicationPreview && ! str($applicationPreview->status)->startsWith('exited')) {
$previewIdsToUpdate->push($applicationPreview->id);
}
}
// Batch update all collected preview IDs
if ($previewIdsToUpdate->isNotEmpty()) {
ApplicationPreview::whereIn('id', $previewIdsToUpdate)->update(['status' => 'exited']);
}
}
private function updateProxyStatus()
{
// If proxy is not found, start it
if ($this->server->isProxyShouldRun() && ! $this->server->isProxyTransitionInProgress()) {
if ($this->foundProxy === false) {
try {
if (CheckProxy::run($this->server)) {
StartProxy::run($this->server, async: false);
$this->server->team?->notify(new ContainerRestarted('coolify-proxy', $this->server, restartedResource: $this->server));
}
} catch (\Throwable $e) {
}
} elseif ($this->server->proxy->get('status') !== 'running') {
$this->server->proxy->set('status', 'running');
$this->server->save();
}
}
}
private function updateDatabaseStatus(string $databaseUuid, string $containerStatus, mixed $restartCount = null, bool $tcpProxy = false): void
{
$database = $this->databasesByUuid->get($databaseUuid);
if (! $database) {
return;
}
if ($database->status !== $containerStatus) {
$database->status = $containerStatus;
$database->save();
}
if (is_numeric($restartCount) && $restartCount > ($database->restart_count ?? 0)) {
$database->update([
'restart_count' => (int) $restartCount,
'last_restart_at' => now(),
'last_restart_type' => 'crash',
]);
}
if (! $this->isCompleteSnapshot()) {
return;
}
if ($this->isRunning($containerStatus) && $tcpProxy) {
$tcpProxyContainerFound = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
return data_get($value, 'name') === "$databaseUuid-proxy" && data_get($value, 'state') === 'running';
})->first();
if (! $tcpProxyContainerFound) {
StartDatabaseProxy::dispatch($database);
$this->server->team?->notify(new ContainerRestarted("TCP Proxy for {$database->name}", $this->server, restartedResource: $database));
} elseif (StartDatabaseProxy::claimOutdatedRefresh($databaseUuid, data_get($tcpProxyContainerFound, 'labels'))) {
StartDatabaseProxy::dispatch($database);
}
} elseif ($this->isRunning($containerStatus) && ! $tcpProxy) {
// Clean up orphaned proxy containers when is_public=false
$orphanedProxy = $this->containers->filter(function ($value, $key) use ($databaseUuid) {
return data_get($value, 'name') === "$databaseUuid-proxy" && data_get($value, 'state') === 'running';
})->first();
if ($orphanedProxy) {
StopDatabaseProxy::dispatch($database);
}
}
}
private function storeServiceRestartCount(string $key, string $containerName, mixed $restartCount): void
{
if (! is_numeric($restartCount)) {
return;
}
if (! $this->serviceContainerRestartCounts->has($key)) {
$this->serviceContainerRestartCounts->put($key, collect());
}
$this->serviceContainerRestartCounts->get($key)->put($containerName, (int) $restartCount);
}
private function trackPreviewRestartCounts(): void
{
$this->previewContainerRestartCounts
->groupBy('key')
->each(function (Collection $counts, string $key): void {
$preview = $this->previewsByKey->get($key);
if ($preview?->trackRestartCount((int) $counts->max('count'))) {
StopApplicationPreview::dispatch($preview, false, false);
$preview->application->environment->project->team?->notify(new ApplicationRestartLimitReached($preview));
}
});
}
private function updateNotFoundDatabaseStatus()
{
$notFoundDatabaseUuids = $this->allDatabaseUuids->diff($this->foundDatabaseUuids);
if ($notFoundDatabaseUuids->isEmpty()) {
return;
}
$notFoundDatabaseUuids->each(function ($databaseUuid) {
$database = $this->databasesByUuid->get($databaseUuid);
if ($database) {
if (! str($database->status)->startsWith('exited')) {
$database->update([
'status' => 'exited',
'restart_count' => 0,
'last_restart_at' => null,
'last_restart_type' => null,
]);
}
if ($database->is_public) {
StopDatabaseProxy::dispatch($database);
}
}
});
}
private function updateNotFoundServiceStatus()
{
$notFoundServiceApplicationIds = $this->allServiceApplicationIds->diff($this->foundServiceApplicationIds);
$notFoundServiceDatabaseIds = $this->allServiceDatabaseIds->diff($this->foundServiceDatabaseIds);
// Batch update service applications
if ($notFoundServiceApplicationIds->isNotEmpty()) {
ServiceApplication::whereIn('id', $notFoundServiceApplicationIds)
->where('restart_limit_reached', false)
->where('status', '!=', 'exited')
->update(['status' => 'exited', 'restart_count' => 0, 'last_restart_at' => null, 'last_restart_type' => null]);
}
// Batch update service databases
if ($notFoundServiceDatabaseIds->isNotEmpty()) {
ServiceDatabase::whereIn('id', $notFoundServiceDatabaseIds)
->where('status', '!=', 'exited')
->update(['status' => 'exited']);
}
}
private function updateAdditionalServersStatus()
{
$this->allApplicationsWithAdditionalServers->each(function ($application) {
ComplexStatusCheck::run($application);
});
}
private function isRunning(string $containerStatus)
{
return str($containerStatus)->contains('running');
}
/**
* Check if container is in an active or transient state.
* Active states: running
* Transient states: restarting, starting, created, paused
*
* These states indicate the container exists and should be tracked.
* Terminal states (exited, dead, removing) should NOT be tracked.
*/
private function isActiveOrTransient(string $containerStatus): bool
{
return str($containerStatus)->contains('running') ||
str($containerStatus)->contains('restarting') ||
str($containerStatus)->contains('starting') ||
str($containerStatus)->contains('created') ||
str($containerStatus)->contains('paused');
}
private function checkLogDrainContainer()
{
if (! $this->server->isFluentBitLogDrainEnabled()) {
return;
}
if ($this->foundLogDrainContainer === false || StartLogDrain::claimOutdatedRefresh($this->server, $this->logDrainContainerLabels)) {
StartLogDrain::dispatch($this->server);
}
}
}