pkg/health/pod.go
Rama main · réplica de lectura del repositorio original.
package health
import (
"fmt"
"strings"
"time"
corev1 "k8s.io/api/core/v1"
)
// Pod classifies a pod's health into a canonical Verdict. This is the single
// source of truth that ClassifyPodHealth (legacy string), the timeline, the
// dashboards, and MCP all reduce to.
//
// Precedence (broken-state checks first, benign last — a benign branch must
// never be ordered ahead of a failure check):
//
// Succeeded → neutral (completed; lifecycle, not a problem)
// Failed → unhealthy
// stable crashloop → unhealthy when down, degraded when Running+Ready
// fatal waiting reason → unhealthy
// active OOMKilled → unhealthy
// scheduling-gated → neutral (deliberately held; not aged into a problem)
// Pending > 5m → degraded ; Pending < 5m → healthy (startup grace)
// readiness probe/gate → degraded
// active thrash → degraded
// owned by a Job → neutral (batch work in flight; not an all-clear)
// default → healthy
//
// Unschedulable and stuck-termination are NOT folded in here: they are detected
// by callers that own their own pre-emption (the scheduling detector, the
// terminating detector) and folding them in would change health-counter outputs.
// They move into this classifier in a later migration where those changes are
// owned explicitly.
func Pod(pod *corev1.Pod, now time.Time) Verdict {
level := classifyPodLevel(pod, now)
v := Verdict{Level: level}
switch level {
case LevelDegraded, LevelUnhealthy:
v.Reason = PodProblemReason(pod, now)
v.Message = PodProblemMessage(pod)
if v.Reason == reasonReadinessGateFail {
for _, c := range pod.Status.Conditions {
if c.Type == corev1.PodReady {
v.Message = c.Message
break
}
}
}
case LevelNeutral:
if pod.Status.Phase == corev1.PodSucceeded {
v.Reason = "Completed"
} else if PodHasSchedulingGates(pod) {
// Explain the hold so the pod doesn't read as a bare, unexplained
// Pending: name the state and the gate(s) blocking scheduling.
v.Reason = corev1.PodReasonSchedulingGated
if gates := SchedulingGateNames(pod); len(gates) > 0 {
v.Message = "Held by scheduling gate(s): " + strings.Join(gates, ", ")
}
}
}
return v
}
// PodDisplayLevel is the pod level for the DISPLAY surfaces — topology node
// color, timeline events, and the AI/MCP summary. On top of the canonical Pod()
// verdict it folds the scheduling + stuck-terminating signals Pod() deliberately
// leaves to its caller, as a FLOOR: escalate to at least degraded so a
// scheduler-failed or wedged pod doesn't read healthy, but never downgrade a real
// unhealthy (a crashlooping pod mid-deletion stays red).
//
// The dashboard / MCP health COUNTERS deliberately do NOT use this — they bucket
// Pod().Level directly and handle scheduling through their own rollup.
func PodDisplayLevel(pod *corev1.Pod, now time.Time) Level {
base := Pod(pod, now).Level
if IsPodUnschedulable(pod) || IsStuckTerminating(pod, now) {
return WorseOf(base, LevelDegraded)
}
return base
}
func classifyPodLevel(pod *corev1.Pod, now time.Time) Level {
if pod.Status.Phase == corev1.PodSucceeded {
return LevelNeutral
}
if pod.Status.Phase == corev1.PodFailed {
return LevelUnhealthy
}
// Phase Unknown means the node is unreachable and the kubelet has stopped
// reporting — the container states are stale and untrustworthy, so this is
// genuinely unknown, not healthy. (Without this, the default fall-through to
// healthy paints a node-lost pod green.)
if pod.Status.Phase == corev1.PodUnknown {
return LevelUnknown
}
// Crash history keeps the problem present across the kubelet's
// Waiting→Running oscillation, while current state controls urgency.
// Unhealthy may return immediately; degraded must not mask an unrelated
// fatal state on another container.
crashLoopLevel := podStableCrashLoopLevel(pod, now)
if crashLoopLevel == LevelUnhealthy {
return LevelUnhealthy
}
for _, cs := range pod.Status.ContainerStatuses {
if cs.State.Waiting != nil && isFatalWaitingReason(cs.State.Waiting.Reason) {
return LevelUnhealthy
}
}
if podHasActiveOOMKilled(pod, now) {
return LevelUnhealthy
}
// Init container errors.
for _, cs := range pod.Status.InitContainerStatuses {
if cs.State.Waiting != nil && isFatalWaitingReason(cs.State.Waiting.Reason) {
return LevelUnhealthy
}
}
if crashLoopLevel == LevelDegraded {
return LevelDegraded
}
// A pod held by a scheduling gate is Pending on purpose — a controller is
// blocking scheduling until a condition is met (Kueue admission, topology
// assignment, a custom gate). It is an intentional lifecycle state, not an
// all-clear, so it reads Neutral (like Completed / batch work) rather than
// aging into a Pending problem. Placed before the age check so a deliberately
// held pod never turns degraded purely on elapsed time; the container-fault
// checks above still run first, so a real fault still surfaces. Once the gate
// is removed (slice empty) the pod ages normally again.
if PodHasSchedulingGates(pod) {
return LevelNeutral
}
// Pods pending past the startup grace window.
if pod.Status.Phase == corev1.PodPending {
if now.Sub(pod.CreationTimestamp.Time) > 5*time.Minute {
return LevelDegraded
}
return LevelHealthy
}
if podHasReadinessProbeFailure(pod, now) {
return LevelDegraded
}
if _, blocked := PodReadinessGateFailure(pod, now); blocked {
return LevelDegraded
}
// A container actively thrashing — high cumulative restarts AND currently
// not ready AND still churning. A plain RestartCount>N check also fires on a
// pod that crashed at startup and has since been Ready for hours
// (RestartCount never resets), and on nodes whose restarts are stale
// laptop-sleep / reboot artifacts — both are healthy now. The thrash gate
// (not-ready + recent/Waiting) excludes those.
if podActiveThrashContainer(pod, now) {
return LevelDegraded
}
// Batch work is in-flight, not "healthy". Workload() already draws exactly
// this line for the Job itself — jobVerdict returns Neutral/"Running" for an
// active Job (workload.go), documented there as "in-progress, not healthy".
// Without this, the same work gets two answers: the Job reads Neutral while
// every one of its pods reads Healthy. A Job pod sitting at 1/2 ready
// because its sidecar outlived the main container then reads as an
// affirmative all-clear, which is the claim we most want not to make.
//
// LAST, after every failure check, so a crashlooping / OOMKilled / fatally
// waiting Job pod keeps its real verdict — Neutral must never mask a
// failure. From here we can see the owner reference but not the Job's
// status, so the honest claim is "this is batch work", not "the Job is
// in-flight" — which is why it is Neutral (no assertion) rather than
// Degraded (an assertion of impairment).
if isOwnedByJob(pod) {
return LevelNeutral
}
return LevelHealthy
}
// isOwnedByJob reports whether the pod was created by a batch Job. The
// APIVersion group is checked so a CRD that happens to be Kind "Job" doesn't
// match.
func isOwnedByJob(pod *corev1.Pod) bool {
for _, ref := range pod.OwnerReferences {
if ref.Kind == "Job" && strings.HasPrefix(ref.APIVersion, "batch/") {
return true
}
}
return false
}
// PodProblemReason returns a short reason token for a problematic pod. Walks
// init containers first because when init is failing the pod stays Pending and
// main ContainerStatuses haven't been populated yet — without the init check the
// reason would fall through to "Pending", masking CrashLoopBackOff /
// ImagePullBackOff / etc. on the actual failing init container.
//
// now is injected (callers pass the same clock they classify with) so the reason
// is reproducible and the crashloop/thrash overrides use a single consistent
// time base.
func PodProblemReason(pod *corev1.Pod, now time.Time) string {
reason := podProblemReasonRaw(pod, now)
// Stable-crashloop normalization: a crashlooping container oscillates
// Waiting("CrashLoopBackOff") → Running (just restarted) → Terminated →
// Waiting between polls. On the "Running" tick the raw walk returns a bare
// phase ("Running") — which category classification maps to `unknown`,
// flipping the category (and the category-hashed issue_id) mid-cycle. When
// the stable history fields say this is a crashloop, emit the canonical
// reason so the row's category stays stable across the whole oscillation. We
// only override when the raw reason isn't already a more-specific, stable
// signal (ImagePullBackOff, OOMKilled, an init failure, …) — those win.
if podHasStableCrashLoop(pod, now) && isPhaseOrCrashReason(reason) {
return reasonCrashLoop
}
if podHasActiveOOMKilled(pod, now) && isPhaseOnlyReason(reason) {
return "OOMKilled"
}
// Actively-thrashing-but-not-a-classic-backoff: a container churning on
// failed readiness probes with clean (exit 0) terminations isn't a stable
// crashloop, so the raw walk returns a bare phase ("Running") that would
// classify as `unknown`. Name it HighRestartCount so the row lands in a
// runtime category instead of the catch-all. Only override a bare phase — a
// specific reason (ImagePullBackOff, an init failure, a real crash) wins.
if podActiveThrashContainer(pod, now) && isPhaseOnlyReason(reason) {
return reasonHighRestart
}
return reason
}
// podProblemReasonRaw is the phase/state walk: init containers first (they block
// the pod Pending before main ContainerStatuses populate), then main containers,
// falling back to the bare phase string.
//
// Containers that exited 0 are skipped on the first pass. A completed init
// container did its job, and naming its "Completed" as the pod's problem reason
// reports a success as the fault, hides the container that actually failed
// below it, and classifies to the catch-all category — leaving a critical row
// with nothing to act on. The second pass restores the plain walk so a pod whose
// containers all succeeded still reports what it did.
func podProblemReasonRaw(pod *corev1.Pod, now time.Time) string {
if IsEvictedPod(pod) {
if cs := oomKilledContainer(pod); cs != nil {
return containerStateReason(cs)
}
return PodEvictedReason
}
if cs := problemContainer(pod, true); cs != nil {
return containerStateReason(cs)
}
// Derived pod-level signals outrank a container that merely finished, so a
// completed init container cannot shadow a readiness failure below it.
if podHasReadinessProbeFailure(pod, now) {
return reasonReadinessProbeFail
}
if _, blocked := PodReadinessGateFailure(pod, now); blocked && !podHasStableCrashLoop(pod, now) && !podHasActiveOOMKilled(pod, now) {
return reasonReadinessGateFail
}
// A failed pod is never described by a container that succeeded. The kubelet
// puts the cause on the pod itself for the failures that have no failing
// container at all — Evicted, DeadlineExceeded, Shutdown, NodeAffinity — and
// without this an evicted pod reported "Completed" from the container the
// kubelet stopped cleanly on its way out.
if pod.Status.Phase == corev1.PodFailed {
if pod.Status.Reason != "" {
return pod.Status.Reason
}
return string(corev1.PodFailed)
}
// A succeeded pod, though, IS described by its completed containers.
if pod.Status.Phase == corev1.PodSucceeded {
if cs := problemContainer(pod, false); cs != nil {
return containerStateReason(cs)
}
}
// Still running: the phase is the honest answer, and it is a phase-like
// reason, so the crashloop / OOM / thrash normalizations in
// PodProblemReason can still replace it. "Completed" is in neither of their
// predicate sets, so returning it here would silence every one of them.
return string(pod.Status.Phase)
}
// problemContainer picks the one container status that explains the pod, so the
// reason and the message always describe the same container rather than being
// gathered from two. Init containers first (they block the pod Pending before
// main ContainerStatuses populate). When skipSucceeded is set, a container
// terminated with exit code 0 is passed over.
func problemContainer(pod *corev1.Pod, skipSucceeded bool) *corev1.ContainerStatus {
for _, statuses := range [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses} {
for i := range statuses {
cs := &statuses[i]
if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" {
return cs
}
if cs.State.Terminated != nil && cs.State.Terminated.Reason != "" {
if skipSucceeded && cs.State.Terminated.ExitCode == 0 {
continue
}
return cs
}
}
}
return nil
}
// oomKilledContainer is the first container whose current state is an
// OOMKilled termination, in any position. On an Evicted pod it is the only
// container that outranks the pod-level reason: the kubelet kills the other
// containers on its way out, so they end as Error or ContainerStatusUnknown
// with exit 137 — a consequence of the eviction, not its cause — while the
// kernel OOM-killed this one before the kubelet chose the pod. Scanning every
// position keeps a killed sidecar listed first from hiding it.
func oomKilledContainer(pod *corev1.Pod) *corev1.ContainerStatus {
for _, statuses := range [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses} {
for i := range statuses {
if t := statuses[i].State.Terminated; t != nil && t.Reason == "OOMKilled" {
return &statuses[i]
}
}
}
return nil
}
func containerStateReason(cs *corev1.ContainerStatus) string {
if cs.State.Waiting != nil {
return cs.State.Waiting.Reason
}
if cs.State.Terminated != nil {
return cs.State.Terminated.Reason
}
return ""
}
func containerStateMessage(cs *corev1.ContainerStatus) string {
if cs.State.Waiting != nil {
return cs.State.Waiting.Message
}
if cs.State.Terminated != nil {
return cs.State.Terminated.Message
}
return ""
}
// PodProblemMessage returns the kubelet's waiting/terminated message for the
// first container in a problem state (init containers first, mirroring
// podProblemReasonRaw's walk). This is the actionable detail behind an otherwise
// bare reason — ImagePullBackOff's "Failed to pull image X: …not found",
// CreateContainerConfigError's "couldn't find key Y in Secret Z".
// It selects the same container the reason came from, so a chatty container
// that merely finished cannot supply the explanation for a different
// container's failure. A failing container with no message stays empty rather
// than borrowing one.
func PodProblemMessage(pod *corev1.Pod) string {
if IsEvictedPod(pod) {
if cs := oomKilledContainer(pod); cs != nil {
return containerStateMessage(cs)
}
return pod.Status.Message
}
if cs := problemContainer(pod, true); cs != nil {
return containerStateMessage(cs)
}
// Gated exactly as podProblemReasonRaw gates its own fallback. Without the
// phase check a still-running pod took its reason from readiness or from the
// phase — later normalized to CrashLoopBackOff / OOMKilled — while the
// message came from a container that had merely finished, which is the
// cross-container pairing this walk exists to prevent.
// Mirrors podProblemReasonRaw: the pod-level message is what explains an
// eviction or a deadline, and it belongs to the same object rather than to
// some other container.
if pod.Status.Phase == corev1.PodFailed {
return pod.Status.Message
}
if pod.Status.Phase == corev1.PodSucceeded {
if cs := problemContainer(pod, false); cs != nil {
return containerStateMessage(cs)
}
}
return ""
}
// PodRestartContext extracts crash-debugging context: total restarts across main
// + init containers, and the kubelet-recorded reason for the most recent
// container termination (OOMKilled, Error, Completed, …). Lets callers tell
// chronic-vs-acute and pick the right next step (OOMKilled → memory analysis;
// Error → previous logs).
func PodRestartContext(pod *corev1.Pod) (restartCount int32, lastTerminatedReason string) {
var newestFinish time.Time
walk := func(statuses []corev1.ContainerStatus) {
for _, cs := range statuses {
restartCount += cs.RestartCount
if t := cs.LastTerminationState.Terminated; t != nil && !t.FinishedAt.IsZero() {
if newestFinish.IsZero() || t.FinishedAt.After(newestFinish) {
newestFinish = t.FinishedAt.Time
lastTerminatedReason = t.Reason
}
}
}
}
walk(pod.Status.ContainerStatuses)
walk(pod.Status.InitContainerStatuses)
return restartCount, lastTerminatedReason
}
type crashLoopCandidate struct {
status corev1.ContainerStatus
init bool
index int
stateRank int
finishedAt time.Time
}
// PodCrashLoopDiagnosis returns a human-readable cause + suggested action for a
// pod's active crashloop candidate, or empty strings when there is no actionable
// crash (no candidate, OOM — which has its own path — or a clean exit).
func PodCrashLoopDiagnosis(pod *corev1.Pod, now time.Time) (cause, action string) {
candidate, ok := activeCrashLoopCandidate(pod, now)
if !ok {
return "", ""
}
term := candidate.status.LastTerminationState.Terminated
if term == nil || term.ExitCode == 0 || term.Reason == "OOMKilled" {
return "", ""
}
ref := fmt.Sprintf("container %q", candidate.status.Name)
if candidate.init {
ref = fmt.Sprintf("init container %q", candidate.status.Name)
}
run := shortRunContext(term)
if IsCrashLoopContainerServing(pod, candidate.status) {
exit := fmt.Sprintf("exit code %d", term.ExitCode)
action := "Watch for another restart. If it repeats, inspect previous container logs and verify the pod command, args, config, and dependencies."
switch term.ExitCode {
case 127:
exit = "exit code 127 (command not found)"
action = "Watch for another restart. If it repeats, check the image entrypoint and pod command/args; verify the binary exists in the image."
case 126:
exit = "exit code 126 (command found but not executable)"
action = "Watch for another restart. If it repeats, check executable permissions, the shebang/interpreter, and the pod command/args."
case 139:
exit = "exit code 139 (segmentation fault)"
action = "Watch for another restart. If it repeats, inspect previous container logs and recent image/code changes; check native libraries or unsafe code."
case 137:
exit = "exit code 137 (Kubernetes did not report OOMKilled)"
action = "Watch for another restart. If it repeats, check node pressure, process-level SIGKILLs, and memory limits."
}
return fmt.Sprintf("%s restarted after %s and is serving again within the 5-minute settle window.%s", ref, exit, run), action
}
switch term.ExitCode {
case 127:
return fmt.Sprintf("%s exited with code 127: command not found.%s", ref, run),
"Check the image entrypoint and pod command/args; verify the binary exists in the image."
case 126:
return fmt.Sprintf("%s exited with code 126: command found but not executable.%s", ref, run),
"Check executable permissions, the shebang/interpreter, and the pod command/args."
case 139:
return fmt.Sprintf("%s exited with code 139: segmentation fault.%s", ref, run),
"Inspect previous container logs and recent image/code changes; check native libraries or unsafe code for a segfault."
case 137:
return fmt.Sprintf("%s exited with code 137, but Kubernetes did not report OOMKilled.%s", ref, run),
"Check node pressure, process-level SIGKILLs, and memory limits; inspect previous container logs for shutdown context."
default:
return fmt.Sprintf("%s is crashlooping after exit code %d.%s", ref, term.ExitCode, run),
"Inspect previous container logs for this container and verify the pod command, args, config, and dependencies."
}
}
func activeCrashLoopCandidate(pod *corev1.Pod, now time.Time) (crashLoopCandidate, bool) {
var best crashLoopCandidate
have := false
consider := func(cs corev1.ContainerStatus, init bool, index int) {
if !isStableCrashLoop(&cs, now) {
return
}
c := crashLoopCandidate{
status: cs,
init: init,
index: index,
stateRank: crashLoopStateRank(cs),
}
if t := cs.LastTerminationState.Terminated; t != nil {
c.finishedAt = t.FinishedAt.Time
}
if !have || betterCrashLoopCandidate(c, best) {
best, have = c, true
}
}
for i := range pod.Status.InitContainerStatuses {
cs := pod.Status.InitContainerStatuses[i]
consider(cs, true, i)
}
for i := range pod.Status.ContainerStatuses {
cs := pod.Status.ContainerStatuses[i]
consider(cs, false, i)
}
return best, have
}
func crashLoopStateRank(cs corev1.ContainerStatus) int {
if cs.State.Waiting != nil {
return 0
}
if cs.State.Terminated != nil {
return 1
}
if cs.State.Running != nil {
return 2
}
return 3
}
func betterCrashLoopCandidate(cand, cur crashLoopCandidate) bool {
if cand.stateRank != cur.stateRank {
return cand.stateRank < cur.stateRank
}
if !cand.finishedAt.Equal(cur.finishedAt) {
return cand.finishedAt.After(cur.finishedAt)
}
if cand.init != cur.init {
return cand.init
}
if cand.status.Name != cur.status.Name {
return cand.status.Name < cur.status.Name
}
return cand.index < cur.index
}
func shortRunContext(term *corev1.ContainerStateTerminated) string {
if term == nil || term.StartedAt.IsZero() || term.FinishedAt.IsZero() {
return ""
}
runDuration := term.FinishedAt.Time.Sub(term.StartedAt.Time)
if runDuration <= 0 || runDuration >= shortCrashRunWindow {
return ""
}
return " It exited within seconds of starting."
}