pkg/health/pod_test.go
Rama main · réplica de lectura del repositorio original.
package health
import (
"strings"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// TestPodGoldenVectors is the canonical, clock-injected health contract for
// pods. Every case fixes an explicit `now` and uses timestamps relative to it,
// so the table is reproducible and portable (the frontend mirrors it in vitest to
// keep the TS classifier from drifting). It asserts both the Level and, for
// problem pods, the Reason token.
func TestPodGoldenVectors(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
old := metav1.NewTime(now.Add(-10 * time.Minute))
recent := metav1.NewTime(now.Add(-1 * time.Minute))
cases := []struct {
name string
pod *corev1.Pod
wantLevel Level
wantReason string // "" = don't assert reason
}{
{
name: "healthy running pod",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{Ready: true, RestartCount: 0}},
}},
wantLevel: LevelHealthy,
},
{
name: "succeeded pod is neutral (completed)",
pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodSucceeded}},
wantLevel: LevelNeutral,
wantReason: "Completed",
},
{
// Batch work in flight is not an all-clear. Workload() already says
// this for the Job itself (jobVerdict returns Neutral/"Running");
// without the same call here the Job read Neutral while its own pods
// read Healthy — one piece of work graded two ways.
name: "job-owned pod is neutral, not an affirmative healthy",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{
{Kind: "Job", APIVersion: "batch/v1", Name: "archiver-123"},
}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{Ready: true}},
},
},
wantLevel: LevelNeutral,
},
{
// The masking guard. Neutral sits after every failure check, so a
// genuinely broken Job pod keeps its real verdict. If this ever
// returns Neutral the ordering has regressed and failing batch
// workloads have gone silent.
name: "crashlooping job-owned pod stays unhealthy",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{
{Kind: "Job", APIVersion: "batch/v1", Name: "archiver-123"},
}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
Ready: false,
RestartCount: 6,
State: corev1.ContainerState{
Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"},
},
}},
},
},
wantLevel: LevelUnhealthy,
wantReason: "CrashLoopBackOff",
},
{
// A CRD that merely happens to be Kind "Job" must not match; the
// owner's API group is checked, not just the kind string.
name: "non-batch Job kind does not trigger the neutral branch",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{
{Kind: "Job", APIVersion: "acme.example.com/v1", Name: "not-a-batch-job"},
}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{Ready: true}},
},
},
wantLevel: LevelHealthy,
},
{
name: "failed pod is unhealthy",
pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodFailed}},
wantLevel: LevelUnhealthy,
},
{
// Node unreachable / lost — container states are stale, so genuinely
// unknown, not the default healthy.
name: "node-lost (phase Unknown) is unknown",
pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodUnknown}},
wantLevel: LevelUnknown,
},
{
name: "CrashLoopBackOff is unhealthy",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{
{State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}}},
},
}},
wantLevel: LevelUnhealthy,
wantReason: "CrashLoopBackOff",
},
{
name: "OOMKilled is unhealthy",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{
{State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled"}}},
},
}},
wantLevel: LevelUnhealthy,
wantReason: "OOMKilled",
},
{
name: "recovered LastTerminationState OOMKilled is healthy",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
Ready: true,
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "OOMKilled"}},
}},
}},
wantLevel: LevelHealthy,
},
{
name: "init container fatal waiting is unhealthy",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: old},
Status: corev1.PodStatus{
Phase: corev1.PodPending,
InitContainerStatuses: []corev1.ContainerStatus{
{State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}}},
},
},
},
wantLevel: LevelUnhealthy,
wantReason: "ImagePullBackOff",
},
{
name: "pending over 5 minutes is degraded",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: old},
Status: corev1.PodStatus{Phase: corev1.PodPending},
},
wantLevel: LevelDegraded,
wantReason: "Pending",
},
{
name: "recently pending is healthy (startup grace)",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: recent},
Status: corev1.PodStatus{Phase: corev1.PodPending},
},
wantLevel: LevelHealthy,
},
{
name: "readiness probe failed long enough is degraded",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: old},
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", ReadinessProbe: &corev1.Probe{}}}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
Conditions: []corev1.PodCondition{{
Type: corev1.PodReady, Status: corev1.ConditionFalse, LastTransitionTime: old,
}},
ContainerStatuses: []corev1.ContainerStatus{{
Name: "app", Ready: false,
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: old}},
}},
},
},
wantLevel: LevelDegraded,
wantReason: "ReadinessProbeFailed",
},
{
name: "recent readiness probe failure is still starting (healthy)",
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: recent},
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app", ReadinessProbe: &corev1.Probe{}}}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
Conditions: []corev1.PodCondition{{
Type: corev1.PodReady, Status: corev1.ConditionFalse, LastTransitionTime: recent,
}},
ContainerStatuses: []corev1.ContainerStatus{{
Name: "app", Ready: false,
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: recent}},
}},
},
},
wantLevel: LevelHealthy,
},
{
name: "recovered: high restart count but now ready and stable is healthy",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
Ready: true, RestartCount: 10,
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-2 * time.Hour))}},
}},
}},
wantLevel: LevelHealthy,
},
{
name: "actively thrashing is degraded",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
Ready: false, RestartCount: 1659,
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-30 * time.Minute))}},
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Completed", ExitCode: 0, FinishedAt: metav1.NewTime(now.Add(-30 * time.Second)),
}},
}},
}},
wantLevel: LevelDegraded,
wantReason: "HighRestartCount",
},
{
name: "stale restarts (days old) is healthy",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
Ready: false, RestartCount: 200,
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-72 * time.Hour))}},
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Completed", ExitCode: 0, FinishedAt: metav1.NewTime(now.Add(-72 * time.Hour)),
}},
}},
}},
wantLevel: LevelHealthy,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
got := Pod(c.pod, now)
if got.Level != c.wantLevel {
t.Errorf("Pod().Level = %q, want %q", got.Level, c.wantLevel)
}
if c.wantReason != "" && got.Reason != c.wantReason {
t.Errorf("Pod().Reason = %q, want %q", got.Reason, c.wantReason)
}
})
}
}
// TestPodStableCrashLoopAcrossPhases pins a non-serving crashloop's continuity
// while its instantaneous State flaps Waiting → Running → Waiting.
func TestPodStableCrashLoopAcrossPhases(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
crashHistory := corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 1}}
mkPod := func(state corev1.ContainerState) *corev1.Pod {
return &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
RestartCount: 7, State: state, LastTerminationState: crashHistory,
}},
}}
}
states := []corev1.ContainerState{
{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}},
{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now)}},
{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}},
}
for i, st := range states {
v := Pod(mkPod(st), now)
if v.Level != LevelUnhealthy || v.Reason != "CrashLoopBackOff" {
t.Errorf("phase %d: got {%q,%q}, want stable {unhealthy,CrashLoopBackOff}", i, v.Level, v.Reason)
}
}
}
// TestPodProblemReasonInitWalk pins that init-container reasons win over the bare
// Pending phase (the init-blocking case) and that specific reasons aren't clobbered.
func TestPodProblemReasonInitWalk(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
cases := []struct {
name string
pod *corev1.Pod
want string
}{
{
name: "init waiting reason wins over phase",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodPending,
InitContainerStatuses: []corev1.ContainerStatus{
{State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}}},
},
}},
want: "CrashLoopBackOff",
},
{
name: "falls back to phase",
pod: &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}},
want: "Pending",
},
{
name: "active ImagePullBackOff keeps specific reason over crashloop",
pod: &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
RestartCount: 2,
State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff"}},
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 1}},
}},
}},
want: "ImagePullBackOff",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := PodProblemReason(c.pod, now); got != c.want {
t.Errorf("PodProblemReason = %q, want %q", got, c.want)
}
})
}
}
func TestPodCrashLoopDiagnosis(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
started := metav1.NewTime(now.Add(-3 * time.Second))
finished := metav1.NewTime(now.Add(-2 * time.Second))
pod := &corev1.Pod{
Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "app"}}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
ContainerStatuses: []corev1.ContainerStatus{{
Name: "app",
RestartCount: 2,
State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-1 * time.Second))}},
LastTerminationState: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Error", ExitCode: 127, StartedAt: started, FinishedAt: finished,
}},
}},
},
}
cause, action := PodCrashLoopDiagnosis(pod, now)
if !strings.Contains(cause, `container "app"`) || !strings.Contains(cause, "code 127") || !strings.Contains(cause, "within seconds") {
t.Fatalf("cause = %q, want app exit-127 diagnosis with short-run context", cause)
}
if !strings.Contains(action, "command/args") {
t.Fatalf("action = %q, want command/args guidance", action)
}
}
// An init container that exited 0 did its job. Reporting its "Completed" as the
// pod's problem reason names a success as the fault, buries the real failure in
// the container below it, and classifies to the catch-all category — so the row
// reads "critical" with nothing to act on. CNPG's bootstrap pods hit this
// (successful init, failed join), but any pod with init containers can.
func TestPodProblemReasonSkipsSuccessfulInitContainers(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodFailed,
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "bootstrap", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
},
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 1, Message: "join failed"}}},
},
}}
if got := PodProblemReason(pod, now); got != "Error" {
t.Errorf("PodProblemReason = %q, want %q — the failed container is the problem, not the init container that succeeded", got, "Error")
}
if got := PodProblemMessage(pod); got != "join failed" {
t.Errorf("PodProblemMessage = %q, want %q", got, "join failed")
}
}
// A failed init container is still the pod's problem — it blocks everything
// after it, and the main containers never start.
func TestPodProblemReasonKeepsFailedInitContainers(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodPending,
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "ok", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
{Name: "broken", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Error", ExitCode: 2, Message: "migration failed"}}},
},
}}
if got := PodProblemReason(pod, now); got != "Error" {
t.Errorf("PodProblemReason = %q, want %q", got, "Error")
}
if got := PodProblemMessage(pod); got != "migration failed" {
t.Errorf("PodProblemMessage = %q, want %q", got, "migration failed")
}
}
// The reason and the message must describe the same container. A chatty init
// container that succeeded used to supply the message while the failed one
// supplied the reason, pairing a failure with an unrelated explanation.
func TestPodProblemMessageMatchesTheReasonsContainer(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodFailed,
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "bootstrap", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Completed", ExitCode: 0, Message: "bootstrap finished cleanly"}}},
},
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Error", ExitCode: 1, Message: "could not reach primary"}}},
},
}}
if got := PodProblemReason(pod, now); got != "Error" {
t.Errorf("PodProblemReason = %q, want %q", got, "Error")
}
if got := PodProblemMessage(pod); got != "could not reach primary" {
t.Errorf("PodProblemMessage = %q, want the failed container's message", got)
}
}
// A pod whose containers all succeeded still reports what it did, rather than
// falling through to a bare phase.
func TestPodProblemReasonStillReportsAnAllSucceededPod(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodSucceeded,
ContainerStatuses: []corev1.ContainerStatus{
{Name: "job", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
},
}}
if got := PodProblemReason(pod, now); got != "Completed" {
t.Errorf("PodProblemReason = %q, want %q", got, "Completed")
}
}
// A completed init container must not shadow a derived pod-level failure. The
// readiness check used to sit behind the plain container walk, so a pod whose
// init container finished cleanly reported "Completed" while its main container
// failed readiness.
func TestPodProblemReasonSucceededInitDoesNotShadowReadinessFailure(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
// Every clause podHasReadinessProbeFailure requires must hold, or this
// passes through the phase fallback and proves nothing about precedence:
// a declared readiness probe, Running phase, and Ready=False for over 5min.
pod := &corev1.Pod{
Spec: corev1.PodSpec{Containers: []corev1.Container{{
Name: "app",
ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz"}}},
}}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
Conditions: []corev1.PodCondition{{
Type: corev1.PodReady, Status: corev1.ConditionFalse,
LastTransitionTime: metav1.NewTime(now.Add(-20 * time.Minute)),
}},
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "setup", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
},
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", Ready: false, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-20 * time.Minute))}}},
},
},
}
if got := PodProblemReason(pod, now); got != reasonReadinessProbeFail {
t.Errorf("PodProblemReason = %q, want %q — a completed init container shadowed the readiness failure", got, reasonReadinessProbeFail)
}
}
// The message must be gated the same way the reason is. A still-running pod
// takes its reason from readiness or from the phase, and used to take its
// message from a container that had merely finished.
func TestPodProblemMessageDoesNotBorrowOnAStillRunningPod(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{
Spec: corev1.PodSpec{Containers: []corev1.Container{{
Name: "app",
ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz"}}},
}}},
Status: corev1.PodStatus{
Phase: corev1.PodRunning,
Conditions: []corev1.PodCondition{{
Type: corev1.PodReady, Status: corev1.ConditionFalse,
LastTransitionTime: metav1.NewTime(now.Add(-20 * time.Minute)),
}},
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "setup", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Completed", ExitCode: 0, Message: "setup wrote 4 files"}}},
},
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", Ready: false, State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-20 * time.Minute))}}},
},
},
}
if got := PodProblemReason(pod, now); got != reasonReadinessProbeFail {
t.Fatalf("PodProblemReason = %q, want %q", got, reasonReadinessProbeFail)
}
if got := PodProblemMessage(pod); got != "" {
t.Errorf("PodProblemMessage = %q — borrowed from a container that merely finished", got)
}
}
// "Completed" is in neither override predicate set, so returning it from a
// still-running pod silences the crashloop, OOM and thrash normalizations that
// exist to name exactly this state. A running pod is described by its phase.
func TestPodProblemReasonDoesNotReportCompletedOnARunningPod(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodRunning,
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "setup", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
},
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: metav1.NewTime(now.Add(-time.Minute))}}},
},
}}
if got := PodProblemReason(pod, now); got != "Running" {
t.Errorf("PodProblemReason = %q, want %q", got, "Running")
}
}
// The message must come from the container the reason came from, even when that
// container has no message of its own. Borrowing a later container's message
// pairs a failure with an unrelated explanation.
func TestPodProblemMessageDoesNotBorrowFromAnotherContainer(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodFailed,
InitContainerStatuses: []corev1.ContainerStatus{
{Name: "setup", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Completed", ExitCode: 0, Message: "setup wrote 4 files"}}},
},
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Error", ExitCode: 1}}},
{Name: "sidecar", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "Completed", ExitCode: 0, Message: "sidecar shut down cleanly"}}},
},
}}
if got := PodProblemReason(pod, now); got != "Error" {
t.Errorf("PodProblemReason = %q, want %q", got, "Error")
}
if got := PodProblemMessage(pod); got != "" {
t.Errorf("PodProblemMessage = %q — the failing container has no message, so this was borrowed from another container", got)
}
}
// An evicted pod has no failing container: the kubelet stops its containers
// cleanly on the way out, so every one of them terminates with exit 0. Reading
// the reason off a container reported the pod as "Completed" while its verdict
// said unhealthy, and dropped the pod-level message that says WHY it was
// evicted — the only actionable text there is.
func TestPodProblemReasonUsesPodLevelReasonOnAFailedPod(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodFailed,
Reason: "Evicted",
Message: "The node was low on resource: ephemeral-storage.",
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
},
}}
if got := PodProblemReason(pod, now); got != "Evicted" {
t.Errorf("PodProblemReason = %q, want %q", got, "Evicted")
}
if got := PodProblemMessage(pod); got != "The node was low on resource: ephemeral-storage." {
t.Errorf("PodProblemMessage = %q, want the pod-level eviction message", got)
}
// The verdict and the reason must agree: unhealthy must never read as a success.
v := Pod(pod, now)
if v.Level != LevelUnhealthy || v.Reason == "Completed" {
t.Errorf("Pod() = %v/%q — an unhealthy pod reported a successful reason", v.Level, v.Reason)
}
}
// A failed pod with no pod-level reason falls back to the phase, not to a
// container that exited cleanly.
func TestPodProblemReasonFailedPodWithoutPodLevelReason(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodFailed,
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{Reason: "Completed", ExitCode: 0}}},
},
}}
if got := PodProblemReason(pod, now); got != "Failed" {
t.Errorf("PodProblemReason = %q, want %q", got, "Failed")
}
}
// A genuinely failed container still wins over the pod-level reason: it is the
// more specific explanation.
func TestPodProblemReasonPrefersAFailedContainerOverThePodReason(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
pod := &corev1.Pod{Status: corev1.PodStatus{
Phase: corev1.PodFailed,
Reason: "Evicted",
ContainerStatuses: []corev1.ContainerStatus{
{Name: "app", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{
Reason: "OOMKilled", ExitCode: 137, Message: "container was OOM killed"}}},
},
}}
if got := PodProblemReason(pod, now); got != "OOMKilled" {
t.Errorf("PodProblemReason = %q, want %q", got, "OOMKilled")
}
if got := PodProblemMessage(pod); got != "container was OOM killed" {
t.Errorf("PodProblemMessage = %q, want the container's message", got)
}
}
// TestPodSchedulingGated covers pods deliberately held by a scheduling gate: they
// must read Neutral (not an age-based Pending problem) and carry an explanation,
// while a real container fault on a gated pod still wins, and a released pod ages
// normally again.
func TestPodSchedulingGated(t *testing.T) {
now := time.Date(2026, 6, 25, 12, 0, 0, 0, time.UTC)
old := metav1.NewTime(now.Add(-30 * time.Minute))
gatedPod := func(gates ...string) *corev1.Pod {
sg := make([]corev1.PodSchedulingGate, 0, len(gates))
for _, g := range gates {
sg = append(sg, corev1.PodSchedulingGate{Name: g})
}
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: old},
Spec: corev1.PodSpec{SchedulingGates: sg},
Status: corev1.PodStatus{Phase: corev1.PodPending},
}
}
t.Run("gated pending past threshold is neutral and explained", func(t *testing.T) {
v := Pod(gatedPod("kueue.x-k8s.io/admission"), now)
if v.Level != LevelNeutral {
t.Fatalf("level = %v, want neutral", v.Level)
}
if v.Reason != corev1.PodReasonSchedulingGated {
t.Errorf("reason = %q, want %q", v.Reason, corev1.PodReasonSchedulingGated)
}
if !strings.Contains(v.Message, "kueue.x-k8s.io/admission") {
t.Errorf("message = %q, want it to name the gate", v.Message)
}
})
t.Run("gated with no PodScheduled condition is still neutral", func(t *testing.T) {
// The spec field is authoritative before the scheduler writes any condition.
if got := Pod(gatedPod("custom-gate"), now).Level; got != LevelNeutral {
t.Errorf("level = %v, want neutral", got)
}
})
t.Run("one of several gates removed is still gated", func(t *testing.T) {
if got := Pod(gatedPod("gate-b"), now).Level; got != LevelNeutral {
t.Errorf("level = %v, want neutral (pod stays held until all gates clear)", got)
}
})
t.Run("gate does not mask a real container fault", func(t *testing.T) {
pod := gatedPod("kueue.x-k8s.io/admission")
pod.Status.ContainerStatuses = []corev1.ContainerStatus{{
Name: "app",
State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{
Reason: "ImagePullBackOff",
Message: "Back-off pulling image \"nope:latest\"",
}},
}}
if got := Pod(pod, now).Level; got != LevelUnhealthy {
t.Errorf("level = %v, want unhealthy (container fault must still surface)", got)
}
})
t.Run("all gates removed ages normally", func(t *testing.T) {
released := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{CreationTimestamp: old},
Status: corev1.PodStatus{Phase: corev1.PodPending},
}
if got := Pod(released, now).Level; got != LevelDegraded {
t.Errorf("level = %v, want degraded (released pod past 5m ages normally)", got)
}
})
}