pkg/health/reasons.go
Rama main · réplica de lectura del repositorio original.
package health
import (
"time"
corev1 "k8s.io/api/core/v1"
)
// Canonical reason tokens this package emits. They are deliberately the
// Kubernetes-canonical strings so downstream category classification stays
// stable across the kubelet's instantaneous state oscillation.
const (
reasonCrashLoop = "CrashLoopBackOff"
reasonHighRestart = "HighRestartCount"
reasonReadinessProbeFail = "ReadinessProbeFailed"
reasonReadinessGateFail = "ReadinessGatesNotReady"
)
// highRestartThreshold is the cumulative per-container RestartCount above which
// a still-unhealthy container is treated as actively thrashing.
const highRestartThreshold = 3
const (
shortCrashRunWindow = 10 * time.Second
crashSettleWindow = 5 * time.Minute
staleCrashStateGap = 2 * crashSettleWindow
)
// isStableCrashLoop reports whether a container is in an ACTIVE crashloop: it
// has restarted with a crash-class last termination (CrashLoopBackOff / generic
// Error / non-zero exit), AND it has not since recovered. It reads the stable
// history fields (RestartCount + LastTerminationState) rather than the
// instantaneous State the kubelet flips between polls — so a real loop's brief
// "Running" blip doesn't downgrade the verdict — but it must NOT fire on a
// container that crashed once and is now running healthily: RestartCount and
// LastTerminationState persist for the life of the container, so without the
// recovery guard below a pod that restarted once at startup would read as a
// crashloop forever. Two recovery signals clear it: a container Running
// continuously past the kubelet's max CrashLoopBackOff backoff (5m) has
// outlived the loop, and a container whose CURRENT state is a clean exit
// (Terminated, exit 0) has succeeded — the common
// init-container-retries-then-completes case, whose failed prior attempt lingers
// in LastTerminationState. A passing readiness probe proves the container is
// serving now, so it controls severity, but it does not erase a recent restart;
// that keeps a real flapper visible as warning across its Ready/Waiting cycle.
// OOMKilled is intentionally excluded — it has its own category/severity path
// upstream.
func isStableCrashLoop(cs *corev1.ContainerStatus, now time.Time) bool {
if cs.RestartCount == 0 {
return false
}
if r := cs.State.Running; r != nil {
if !r.StartedAt.IsZero() && now.Sub(r.StartedAt.Time) >= crashSettleWindow {
return false
}
if t := cs.LastTerminationState.Terminated; t != nil && !t.FinishedAt.IsZero() {
if r.StartedAt.IsZero() && now.Sub(t.FinishedAt.Time) >= crashSettleWindow {
return false
}
// A gap well beyond the kubelet's maximum crash backoff means the
// old termination and this new run are not one continuous loop
// (for example, a node returning after a long outage).
if !r.StartedAt.IsZero() && r.StartedAt.Sub(t.FinishedAt.Time) > staleCrashStateGap {
return false
}
}
}
if term := cs.State.Terminated; term != nil && term.ExitCode == 0 {
return false
}
t := cs.LastTerminationState.Terminated
if t == nil {
return false
}
switch t.Reason {
case "OOMKilled":
// Memory pressure is classified separately (CategoryOOMKilled); don't
// fold it into the generic crashloop bucket.
return false
case "CrashLoopBackOff", "Error":
return true
}
// A non-zero exit code with no special reason is still a crash — the app
// died and the kubelet is restarting it.
return t.ExitCode != 0
}
// ActiveCrashLoopContainerStatuses returns main and init container statuses
// that are still in an active crashloop after applying Radar's recovery guards.
func ActiveCrashLoopContainerStatuses(pod *corev1.Pod, now time.Time) []corev1.ContainerStatus {
if pod == nil {
return nil
}
var active []corev1.ContainerStatus
for i := range pod.Status.ContainerStatuses {
cs := pod.Status.ContainerStatuses[i]
if isStableCrashLoop(&cs, now) {
active = append(active, cs)
}
}
for i := range pod.Status.InitContainerStatuses {
cs := pod.Status.InitContainerStatuses[i]
if isStableCrashLoop(&cs, now) {
active = append(active, cs)
}
}
return active
}
// podHasStableCrashLoop reports whether any main or init container is in a
// stable crashloop (see isStableCrashLoop).
func podHasStableCrashLoop(pod *corev1.Pod, now time.Time) bool {
return podStableCrashLoopLevel(pod, now) != ""
}
// IsCrashLoopContainerServing reports whether a crash-history status represents
// a serving main container or a serving native sidecar.
func IsCrashLoopContainerServing(pod *corev1.Pod, cs corev1.ContainerStatus) bool {
if pod == nil || cs.State.Running == nil || !cs.Ready {
return false
}
for i := range pod.Status.ContainerStatuses {
if pod.Status.ContainerStatuses[i].Name == cs.Name {
return true
}
}
return restartableInitContainer(pod, cs.Name)
}
func podStableCrashLoopLevel(pod *corev1.Pod, now time.Time) Level {
level := Level("")
for i := range pod.Status.ContainerStatuses {
cs := &pod.Status.ContainerStatuses[i]
if !isStableCrashLoop(cs, now) {
continue
}
if IsCrashLoopContainerServing(pod, *cs) {
level = LevelDegraded
continue
}
return LevelUnhealthy
}
for i := range pod.Status.InitContainerStatuses {
cs := &pod.Status.InitContainerStatuses[i]
if !isStableCrashLoop(cs, now) {
continue
}
if IsCrashLoopContainerServing(pod, *cs) {
level = LevelDegraded
continue
}
return LevelUnhealthy
}
return level
}
func containerHasReadinessProbe(containers []corev1.Container, name string) bool {
for i := range containers {
if containers[i].Name == name {
return containers[i].ReadinessProbe != nil
}
}
return false
}
func restartableInitContainer(pod *corev1.Pod, name string) bool {
for i := range pod.Spec.InitContainers {
c := &pod.Spec.InitContainers[i]
if c.Name == name {
return c.RestartPolicy != nil && *c.RestartPolicy == corev1.ContainerRestartPolicyAlways
}
}
return false
}
// restartedRecently reports whether a container's most recent termination
// finished within the given window — i.e. it is still actively churning, not a
// container that crashed long ago and has since gone quiet (the laptop-sleep /
// node-reboot artifact where RestartCount is high but every termination is days
// old).
func restartedRecently(cs *corev1.ContainerStatus, now time.Time, within time.Duration) bool {
if t := cs.LastTerminationState.Terminated; t != nil && !t.FinishedAt.IsZero() {
return now.Sub(t.FinishedAt.Time) <= within
}
return false
}
// isActivelyThrashing reports whether a container has a high cumulative restart
// count AND is currently unhealthy AND is still churning. The Ready gate is what
// clears the recovered-after-crash false positive — a pod that restarted many
// times at startup but is now Ready and stable (its RestartCount never resets)
// no longer trips this. The Waiting/recency gate clears the slept-then-woken
// node whose restarts are days old. The 5m window matches isStableCrashLoop's
// horizon so the two guards don't drift.
func isActivelyThrashing(cs *corev1.ContainerStatus, now time.Time) bool {
if cs.RestartCount <= highRestartThreshold || cs.Ready {
return false
}
if cs.State.Waiting != nil {
return true
}
return restartedRecently(cs, now, crashSettleWindow)
}
// podActiveThrashContainer reports whether any main container is actively
// thrashing (see isActivelyThrashing). Init containers are excluded — a failing
// init container surfaces through podProblemReasonRaw's init walk with a
// specific reason already.
func podActiveThrashContainer(pod *corev1.Pod, now time.Time) bool {
for i := range pod.Status.ContainerStatuses {
if isActivelyThrashing(&pod.Status.ContainerStatuses[i], now) {
return true
}
}
return false
}
func containerHasTrustedReady(containers []corev1.Container, cs corev1.ContainerStatus) bool {
return cs.Ready && containerHasReadinessProbe(containers, cs.Name)
}
func containerRecentlyRecoveredFromOOM(containers []corev1.Container, cs corev1.ContainerStatus, now time.Time) bool {
if cs.LastTerminationState.Terminated == nil || cs.LastTerminationState.Terminated.Reason != "OOMKilled" {
return false
}
if containerHasTrustedReady(containers, cs) {
return false
}
if r := cs.State.Running; r != nil && !r.StartedAt.IsZero() {
return now.Sub(r.StartedAt.Time) <= 5*time.Minute
}
return cs.State.Waiting != nil || restartedRecently(&cs, now, 5*time.Minute)
}
// ActiveOOMKilledContainers returns the regular containers currently affected
// by an OOM kill. Init containers are excluded so callers can match the result
// to a workload template's regular containers without conflating the two lists.
func ActiveOOMKilledContainers(pod *corev1.Pod, now time.Time) []corev1.ContainerStatus {
var active []corev1.ContainerStatus
for i := range pod.Status.ContainerStatuses {
cs := pod.Status.ContainerStatuses[i]
if cs.State.Terminated != nil && cs.State.Terminated.Reason == "OOMKilled" {
active = append(active, cs)
continue
}
if containerRecentlyRecoveredFromOOM(pod.Spec.Containers, cs, now) {
active = append(active, cs)
}
}
return active
}
// PodHasActiveOOMKilled reports whether any container, init containers and
// native sidecars included, is currently affected by an OOM kill (one that has
// not since recovered). Exported for the problem detector.
func PodHasActiveOOMKilled(pod *corev1.Pod, now time.Time) bool {
return podHasActiveOOMKilled(pod, now)
}
func podHasActiveOOMKilled(pod *corev1.Pod, now time.Time) bool {
if len(ActiveOOMKilledContainers(pod, now)) > 0 {
return true
}
for _, cs := range pod.Status.InitContainerStatuses {
if cs.State.Terminated != nil && cs.State.Terminated.Reason == "OOMKilled" {
return true
}
if cs.LastTerminationState.Terminated != nil && cs.LastTerminationState.Terminated.Reason == "OOMKilled" {
if cs.State.Terminated != nil && cs.State.Terminated.ExitCode == 0 {
continue
}
if r := cs.State.Running; r != nil && !r.StartedAt.IsZero() && now.Sub(r.StartedAt.Time) > 5*time.Minute {
continue
}
return true
}
}
return false
}
// PodHasReadinessProbeFailure reports whether a running pod has been
// Ready=False past the grace window with a readiness-probed container still
// failing — exported for the problem detector, which shares this predicate.
func PodHasReadinessProbeFailure(pod *corev1.Pod, now time.Time) bool {
return podHasReadinessProbeFailure(pod, now)
}
func podHasReadinessProbeFailure(pod *corev1.Pod, now time.Time) bool {
if pod.Status.Phase != corev1.PodRunning {
return false
}
if !podReadyFalseLongEnough(pod, now, 5*time.Minute) {
return false
}
for i := range pod.Status.ContainerStatuses {
cs := &pod.Status.ContainerStatuses[i]
if cs.Ready || cs.State.Running == nil {
continue
}
if containerHasReadinessProbe(pod.Spec.Containers, cs.Name) {
return true
}
}
return false
}
func podReadyFalseLongEnough(pod *corev1.Pod, now time.Time, minAge time.Duration) bool {
for i := range pod.Status.Conditions {
cond := &pod.Status.Conditions[i]
if cond.Type != corev1.PodReady && cond.Type != corev1.ContainersReady {
continue
}
if cond.Status != corev1.ConditionFalse {
continue
}
if !cond.LastTransitionTime.IsZero() {
return now.Sub(cond.LastTransitionTime.Time) >= minAge
}
if !pod.CreationTimestamp.IsZero() {
return now.Sub(pod.CreationTimestamp.Time) >= minAge
}
return true
}
return false
}
// PodBlockingReason identifies a fatal waiting state, including init containers.
func PodBlockingReason(pod *corev1.Pod) (container, reason, message string) {
for _, statuses := range [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses} {
for _, cs := range statuses {
if w := cs.State.Waiting; w != nil && isFatalWaitingReason(w.Reason) {
return cs.Name, w.Reason, containerStateMessage(&cs)
}
}
}
return "", "", ""
}
// isFatalWaitingReason reports whether a container's Waiting reason is a hard
// failure that won't self-resolve on its own — as opposed to the transient
// PodInitializing/ContainerCreating states. InvalidImageName is permanent (a
// malformed image reference never becomes valid); the *ContainerError family
// means the container couldn't be created or started (bad command, missing
// device, runtime rejection).
func isFatalWaitingReason(reason string) bool {
switch reason {
case "CrashLoopBackOff", "ImagePullBackOff", "ErrImagePull", "InvalidImageName",
"ImageInspectError", "CreateContainerConfigError", "CreateContainerError",
"RunContainerError":
return true
}
return false
}
// isPhaseOrCrashReason reports whether reason is one that a stable-crashloop
// override may safely replace: a bare lifecycle phase / no-op waiting state
// (the instantaneous values that flap), or an already-crash-class reason
// (so the canonical string is used consistently). A distinct, more-specific
// reason like ImagePullBackOff or OOMKilled is NOT in this set and is left
// untouched.
func isPhaseOrCrashReason(reason string) bool {
switch reason {
case "Running", "Pending", "Succeeded", "Failed", "Unknown", "",
"PodInitializing", "ContainerCreating",
"CrashLoopBackOff", "Error":
return true
}
return false
}
// isPhaseOnlyReason is the narrower set the HighRestartCount override may
// replace: bare lifecycle phases / no-op waiting states only. It deliberately
// excludes the crash-class reasons (CrashLoopBackOff/Error) and terminal phases
// (Succeeded/Failed) that isPhaseOrCrashReason allows, so a thrash override can
// never clobber a real crash or terminal signal — the stable-crashloop check
// above already owns those.
func isPhaseOnlyReason(reason string) bool {
switch reason {
case "Running", "Pending", "Unknown", "",
"PodInitializing", "ContainerCreating":
return true
}
return false
}
// StuckTerminatingThreshold is how long a pod may sit with a deletionTimestamp
// before its termination is treated as stuck (wedged on a finalizer / ungraceful
// node drain) rather than a normal graceful shutdown.
const StuckTerminatingThreshold = 10 * time.Minute
// IsStuckTerminating reports whether a pod has been terminating longer than
// StuckTerminatingThreshold. The canonical Pod classifier deliberately ignores
// deletionTimestamp (folding it in would shift the dashboard/MCP health counters);
// the display surfaces (timeline, topology) fold this signal in on top so a wedged
// pod doesn't read healthy.
func IsStuckTerminating(pod *corev1.Pod, now time.Time) bool {
dt := pod.DeletionTimestamp
return dt != nil && now.Sub(dt.Time) > StuckTerminatingThreshold
}
// IsPodUnschedulable reports whether the scheduler tried and failed to place the
// pod (PodScheduled=False with reason=Unschedulable). reason=SchedulingGated is
// an intentional not-yet-scheduled state, not a placement failure, and is
// excluded.
func IsPodUnschedulable(pod *corev1.Pod) bool {
for i := range pod.Status.Conditions {
c := &pod.Status.Conditions[i]
if c.Type == corev1.PodScheduled {
return c.Status == corev1.ConditionFalse && c.Reason == corev1.PodReasonUnschedulable
}
}
return false
}
// PodHasSchedulingGates reports whether a pod is deliberately held by one or more
// scheduling gates. A non-empty spec.schedulingGates means a controller is blocking
// scheduling on purpose (Kueue admission, topology assignment, a custom gate, …); it
// is set at creation and can only be removed, and removing one of several does not
// release the pod. The spec field is used rather than the PodScheduled=SchedulingGated
// condition because it is authoritative and present before the scheduler writes any
// condition.
func PodHasSchedulingGates(pod *corev1.Pod) bool {
return pod != nil && len(pod.Spec.SchedulingGates) > 0
}
// SchedulingGateNames returns the names of the pod's scheduling gates, in order.
func SchedulingGateNames(pod *corev1.Pod) []string {
if pod == nil || len(pod.Spec.SchedulingGates) == 0 {
return nil
}
names := make([]string, 0, len(pod.Spec.SchedulingGates))
for _, g := range pod.Spec.SchedulingGates {
names = append(names, g.Name)
}
return names
}
// PodReadinessGateFailure reports a declared gate that has not passed and still
// blocks a running Pod after its containers became ready. A nil Pod has no failure.
// Both readiness transitions get the probe path's five-minute grace. Missing transition times
// use creation only for grace, never as evidence of failure onset. Onset is
// known only when both transitions are dated. With no transition or creation
// time, no grace is assumed. Up to five minutes of clock skew gets grace;
// farther-future timestamps establish neither grace nor onset.
// Unknown gate health still blocks readiness when the kubelet reports that explicitly.
func PodReadinessGateFailure(pod *corev1.Pod, now time.Time) (since time.Time, failing bool) {
if pod == nil || pod.DeletionTimestamp != nil || pod.Status.Phase != corev1.PodRunning || len(pod.Spec.ReadinessGates) == 0 {
return time.Time{}, false
}
var ready, containersReady *corev1.PodCondition
for i := range pod.Status.Conditions {
c := &pod.Status.Conditions[i]
switch c.Type {
case corev1.PodReady:
ready = c
case corev1.ContainersReady:
containersReady = c
}
}
if ready == nil || ready.Status != corev1.ConditionFalse || ready.Reason != reasonReadinessGateFail || containersReady == nil || containersReady.Status != corev1.ConditionTrue {
return time.Time{}, false
}
recent, future := false, false
for _, c := range []*corev1.PodCondition{ready, containersReady} {
transition := c.LastTransitionTime.Time
if transition.IsZero() {
transition = pod.CreationTimestamp.Time
}
if !transition.IsZero() {
future = future || transition.After(now)
recent = recent || (!transition.After(now.Add(5*time.Minute)) && now.Sub(transition) < 5*time.Minute)
}
}
if recent {
return time.Time{}, false
}
for _, gate := range pod.Spec.ReadinessGates {
found := false
for _, c := range pod.Status.Conditions {
if c.Type == gate.ConditionType {
found = true
if c.Status != corev1.ConditionTrue {
failing = true
}
break
}
}
if !found {
failing = true
}
}
if !failing {
return time.Time{}, false
}
if !future && !ready.LastTransitionTime.IsZero() && !containersReady.LastTransitionTime.IsZero() {
since = ready.LastTransitionTime.Time
if containersReady.LastTransitionTime.After(since) {
since = containersReady.LastTransitionTime.Time
}
}
return since, true
}