app/Services/CheckRunner.php
Rama main · réplica de lectura del repositorio original.
<?php
namespace App\Services;
use App\Enums\CheckType;
use App\Enums\ComponentStatus;
use App\Enums\IncidentStatus;
use App\Models\Check;
use App\Models\CheckResult;
use App\Models\Incident;
use App\Models\IncidentUpdate;
use App\Models\Maintenance;
use App\Models\UptimeDay;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
/**
* Turns probe results into component status and, where asked for, into incidents.
* This is the part Cachet does not have: nobody has to press a button.
*/
class CheckRunner
{
/** Consecutive healthy results before an auto-opened incident closes itself. */
public const RECOVERY_STREAK = 3;
public function __construct(protected Probe $probe, protected OutgoingWebhook $webhook) {}
/** @return int number of checks executed */
public function runDue(?\DateTimeInterface $now = null): int
{
$now ??= now();
$ran = 0;
foreach (Check::with('component')->whereHas('component')->where('enabled', true)->get() as $check) {
if (! $check->isDue($now) || ! $check->component?->enabled) {
continue;
}
$this->runOne($check, $now);
$ran++;
}
return $ran;
}
public function runOne(Check $check, ?\DateTimeInterface $now = null): ProbeResult
{
$now ??= now();
$result = app(ProbeQuorum::class)->combine($check, $this->probe->run($check));
if ($result->inconclusive) {
if ($check->component->status !== ComponentStatus::UnderMaintenance) {
$check->component->update(['status' => ComponentStatus::PerformanceIssues]);
}
if ($check->type !== CheckType::Heartbeat) {
$check->update(['last_run_at' => $now]);
}
return $result;
}
if ($check->type === CheckType::Http) {
$check->tls_expires_at = $result->tlsExpiresAt;
$check->tls_warning = $result->tlsExpiresAt && $result->tlsExpiresAt <= now()->addDays(14) ? $result->message : null;
}
// Read before this run's own result is written: it is the reference for how
// much wall time this evaluation stands for.
$previousRun = CheckResult::where('component_id', $check->component_id)->max('checked_at');
CheckResult::create([
'component_id' => $check->component_id,
'ok' => $result->ok,
'latency_ms' => $result->latencyMs,
'message' => $result->message,
'checked_at' => $now,
]);
if ($result->ok) {
$check->consecutive_successes++;
$check->consecutive_failures = 0;
} else {
$check->consecutive_failures++;
$check->consecutive_successes = 0;
}
// A heartbeat's last_run_at is owned by whoever pings in, not by the runner.
if ($check->type !== CheckType::Heartbeat) {
$check->last_run_at = $now;
}
$check->save();
$this->recordUptime($check, $result, $now, $previousRun);
$this->applyStatus($check, $result, $now);
return $result;
}
protected function recordUptime(Check $check, ProbeResult $result, \DateTimeInterface $now, ?string $previousRun): void
{
$now = Carbon::instance($now)->utc();
$dayStart = $now->copy()->startOfDay();
// Preserve the existing observation cap and UTC midnight boundary.
$since = $previousRun ? Carbon::parse($previousRun, 'UTC') : $now->copy()->subSeconds($check->interval_seconds);
$since = $since->max($dayStart)->max($now->copy()->subSeconds($check->interval_seconds));
$seconds = (int) max(0, $since->diffInSeconds($now));
$windows = Maintenance::whereNotNull('started_at')->where('starts_at', '<', $now)->where('ends_at', '>', $since)
->whereHas('components', fn ($q) => $q->whereKey($check->component_id))
->with(['components' => fn ($q) => $q->whereKey($check->component_id)])->limit(2001)->get();
if ($windows->count() > 2000) {
return; // Too many windows to establish accurate eligible credit; keep raw evidence only.
}
$excluded = MaintenanceIntervals::excluded($windows, $check->component_id, $since, $now);
// A manually held maintenance state has no reliable earlier boundary.
if ($check->component->status === ComponentStatus::UnderMaintenance && $excluded === 0) {
return;
}
$seconds = max(0, $seconds - $excluded);
if ($seconds === 0) {
return;
}
$day = UptimeDay::firstOrCreate(
['component_id' => $check->component_id, 'day' => $dayStart],
['up_seconds' => 0, 'down_seconds' => 0, 'worst_status' => ComponentStatus::Operational->value],
);
$column = $result->ok ? 'up_seconds' : 'down_seconds';
$day->increment($column, $seconds);
if (! $result->ok) {
$day->worst_status = ComponentStatus::MajorOutage;
$day->save();
}
}
protected function applyStatus(Check $check, ProbeResult $result, \DateTimeInterface $now): void
{
$component = $check->component;
// Planned work: a probe failing inside a running maintenance window is
// expected, and must neither flip the component nor open an incident.
if (! $result->ok && $component->status === ComponentStatus::UnderMaintenance && Maintenance::holds($component->id)) {
return;
}
if (! $result->ok && $check->consecutive_failures >= $check->retries) {
if ($component->status !== ComponentStatus::MajorOutage) {
$component->update(['status' => ComponentStatus::MajorOutage]);
$this->openIncident($check, $result, $now);
}
return;
}
if (! $result->ok) {
return;
}
// A passing check owns the status — including a hand-set "Degraded", which
// the form says it will overwrite. Under maintenance is the one deliberate
// state a probe cannot judge, so that stays until someone clears it.
if (! in_array($component->status, [ComponentStatus::Operational, ComponentStatus::UnderMaintenance], true)) {
$component->update(['status' => $result->degraded ? ComponentStatus::PerformanceIssues : ComponentStatus::Operational]);
}
if ($result->degraded && $component->status === ComponentStatus::Operational) {
$component->update(['status' => ComponentStatus::PerformanceIssues]);
}
// Checked independently of the component status: the first healthy result
// already flipped the component back, so nesting this inside that branch
// meant the incident never closed.
if ($check->consecutive_successes >= $this->recoveryStreak($check)) {
$this->resolveIncident($check, $now);
}
}
/**
* A heartbeat that calls in has proved it is alive; the streak is hysteresis
* against a flapping poll, and applying it here would keep a nightly backup
* red for two more days after it demonstrably ran.
*/
protected function recoveryStreak(Check $check): int
{
return $check->type === CheckType::Heartbeat ? 1 : self::RECOVERY_STREAK;
}
protected function openIncident(Check $check, ProbeResult $result, \DateTimeInterface $now): Incident
{
$incident = DB::transaction(function () use ($check, $result, $now) {
$incident = Incident::create([
'name' => __(':component unreachable', ['component' => $check->component->name], Localization::page()),
'status' => IncidentStatus::Investigating,
'impact' => 'major',
'source' => 'check',
'auto_resolve' => true,
'grouping_key' => 'check:'.$check->component_id,
'occurred_at' => $now,
]);
$incident->components()->attach($check->component_id, [
'status' => ComponentStatus::MajorOutage->value,
]);
IncidentUpdate::create([
'incident_id' => $incident->id,
'status' => IncidentStatus::Investigating,
'message' => $result->message
? __('Automatic check failed: :reason.', ['reason' => __($result->message, [], Localization::page())], Localization::page())
: __('Automatic check failed.', [], Localization::page()),
'automatic' => true,
]);
return $incident;
});
$this->webhook->incidentChanged($incident->fresh('components'), 'incident.created');
return $incident;
}
protected function resolveIncident(Check $check, \DateTimeInterface $now): void
{
$incident = Incident::where('grouping_key', 'check:'.$check->component_id)
->where('auto_resolve', true)
->whereNull('resolved_at')
->latest('occurred_at')
->first();
if (! $incident) {
return;
}
DB::transaction(function () use ($incident, $now, $check) {
$incident->update(['status' => IncidentStatus::Resolved, 'resolved_at' => $now]);
IncidentUpdate::create([
'incident_id' => $incident->id,
'status' => IncidentStatus::Resolved,
'message' => __('The component responded normally again for :count consecutive checks.', ['count' => $this->recoveryStreak($check)], Localization::page()),
'automatic' => true,
]);
});
$this->webhook->incidentChanged($incident->fresh('components'), 'incident.updated');
}
}