pkg/schedulinginsight/kueue_admission_test.go
Rama main · réplica de lectura del repositorio original.
package schedulinginsight
import (
"encoding/json"
"fmt"
"go/ast"
"go/parser"
"go/token"
"os"
"reflect"
"sort"
"strconv"
"strings"
"testing"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
rc "github.com/skyhook-io/radar/pkg/resourcecontext"
)
// Messages below are copied from Kueue's controllers (v0.18.0 coarse reasons,
// v0.20 granular reasons) so the parser is pinned to what Kueue writes.
const (
legacyUnusedQuota = "couldn't assign flavors to pod set main: insufficient unused quota for nvidia.com/gpu in flavor a10, 1 more needed"
legacyExceedsQuota = "couldn't assign flavors to pod set main: insufficient quota for cpu in flavor default, previously considered podsets requests (0) + current podset request (100) > maximum capacity (10)"
legacyMixedQuota = "couldn't assign flavors to pod set main: insufficient quota for cpu in flavor big, previously considered podsets requests (0) + current podset request (100) > maximum capacity (10), insufficient unused quota for cpu in flavor small, 4 more needed"
legacyTaintMismatch = "couldn't assign flavors to pod set main: untolerated taint {spot true NoSchedule <nil>} in flavor spot"
legacyNoGPUResource = "couldn't assign flavors to pod set main: resource nvidia.com/gpu unavailable in ClusterQueue"
namespaceMismatch = "Workload namespace doesn't match ClusterQueue selector"
// One flavor: memory is only short now, but cpu is over the maximum, so
// waiting cannot make this flavor fit.
legacyCoRequisite = "couldn't assign flavors to pod set main: insufficient unused quota for memory in flavor default, 1Gi more needed, insufficient quota for cpu in flavor default, previously considered podsets requests (0) + current podset request (4) > maximum capacity (2)"
// Two PodSets: the second fits nowhere, so the Workload never fits.
legacyTwoPodSets = "couldn't assign flavors to pod set driver: insufficient unused quota for cpu in flavor default, 1 more needed; couldn't assign flavors to pod set workers: insufficient quota for nvidia.com/gpu in flavor a10, previously considered podsets requests (0) + current podset request (8) > maximum capacity (4)"
)
type condition struct {
kind, status, reason, message string
observedGeneration int64
}
type workloadSpec struct {
version string
generation int64
active *bool
annotations map[string]string
conditions []condition
checks []map[string]any
noQueue bool
}
func buildWorkload(spec workloadSpec) *unstructured.Unstructured {
version := spec.version
if version == "" {
version = "v1beta2"
}
u := &unstructured.Unstructured{Object: map[string]any{
"apiVersion": "kueue.x-k8s.io/" + version,
"kind": "Workload",
"metadata": map[string]any{"name": "train", "namespace": "ml"},
"spec": map[string]any{"queueName": "gpu"},
}}
if spec.noQueue {
unstructured.RemoveNestedField(u.Object, "spec", "queueName")
}
if spec.generation != 0 {
u.SetGeneration(spec.generation)
}
if spec.active != nil {
_ = unstructured.SetNestedField(u.Object, *spec.active, "spec", "active")
}
if spec.annotations != nil {
u.SetAnnotations(spec.annotations)
}
conditions := make([]any, 0, len(spec.conditions))
for _, c := range spec.conditions {
entry := map[string]any{"type": c.kind, "status": c.status, "reason": c.reason, "message": c.message, "lastTransitionTime": "2026-11-01T10:00:00Z"}
if c.observedGeneration != 0 {
entry["observedGeneration"] = c.observedGeneration
}
conditions = append(conditions, entry)
}
_ = unstructured.SetNestedSlice(u.Object, conditions, "status", "conditions")
if spec.checks != nil {
checks := make([]any, 0, len(spec.checks))
for _, check := range spec.checks {
checks = append(checks, check)
}
_ = unstructured.SetNestedSlice(u.Object, checks, "status", "admissionChecks")
}
return u
}
func pending(reason, message string) condition {
return condition{kind: "QuotaReserved", status: "False", reason: reason, message: message}
}
var (
reserved = condition{kind: "QuotaReserved", status: "True", reason: "QuotaReserved", message: "Quota reserved in ClusterQueue gpu-team"}
admitted = condition{kind: "Admitted", status: "True", reason: "Admitted", message: "The workload is admitted"}
noReservation = condition{kind: "Admitted", status: "False", reason: "NoReservation", message: "The workload has no reservation"}
)
func boolPtr(value bool) *bool { return &value }
func classify(t *testing.T, spec workloadSpec) KueueClassification {
t.Helper()
got, ok := ClassifyKueueWorkload(buildWorkload(spec))
if !ok {
t.Fatal("Workload was not classified")
}
return got
}
func factCode(fact *rc.KueueAdmissionFact) rc.KueueAdmissionCode {
if fact == nil {
return ""
}
return fact.Code
}
func TestClassifyKueueWorkloadStates(t *testing.T) {
tests := []struct {
name string
spec workloadSpec
phase rc.KueuePhase
state rc.KueueAdmissionCode
inactive rc.KueueAdmissionCode
blocker rc.KueueAdmissionCode
disruption rc.KueueAdmissionCode
}{
{
name: "no queue name and no conditions",
spec: workloadSpec{noQueue: true},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoDecision,
blocker: rc.KueueAdmissionNoDecision,
},
{
name: "no conditions yet",
spec: workloadSpec{},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoDecision,
blocker: rc.KueueAdmissionNoDecision,
},
{
name: "stuck on quota (v0.18 coarse reason)",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyUnusedQuota), noReservation}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "stuck on quota served as v1beta1",
spec: workloadSpec{version: "v1beta1", conditions: []condition{pending("Pending", legacyUnusedQuota)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "stuck on quota (v0.20 granular reason)",
spec: workloadSpec{conditions: []condition{pending("WaitingForQuota", legacyUnusedQuota)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "every quota clause exceeds the maximum",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyExceedsQuota)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionExceedsQuota,
blocker: rc.KueueAdmissionExceedsQuota,
},
{
name: "mixed quota clauses could still fit later",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyMixedQuota)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "granular exceeds",
spec: workloadSpec{conditions: []condition{pending("ExceedsMaxQuota", "exceeds")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionExceedsQuota,
blocker: rc.KueueAdmissionExceedsQuota,
},
{
name: "a resource over the maximum rules out the flavor even when another resource is only short",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyCoRequisite)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionExceedsQuota,
blocker: rc.KueueAdmissionExceedsQuota,
},
{
name: "one PodSet that fits nowhere decides",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyTwoPodSets)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionExceedsQuota,
blocker: rc.KueueAdmissionExceedsQuota,
},
{
name: "granular topology placement failure",
spec: workloadSpec{conditions: []condition{pending("TopologyPlacementFailed", "cannot fit in topology")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionTopologyPlacementFailed,
blocker: rc.KueueAdmissionTopologyPlacementFailed,
},
{
name: "DRA resources Kueue could not resolve are not blamed on the Workload",
spec: workloadSpec{conditions: []condition{pending("DRAResourcesUnresolved", "failed to list ResourceSlices: timeout")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDRAResourcesUnresolved,
blocker: rc.KueueAdmissionDRAResourcesUnresolved,
},
{
name: "usage Kueue cannot record",
spec: workloadSpec{conditions: []condition{pending("UnrecordableResourceUsage", "total exceeds the recordable maximum")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInvalidWorkload,
blocker: rc.KueueAdmissionInvalidWorkload,
},
{
name: "pending evaluation blocked by retrying checks (v0.20 wording)",
spec: workloadSpec{conditions: []condition{pending("PendingEvaluation", "The workload has failed admission checks")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionChecksPending,
blocker: rc.KueueAdmissionChecksPending,
},
{
name: "v0.20 admitted reason for unready checks",
spec: workloadSpec{
conditions: []condition{reserved, {kind: "Admitted", status: "False", reason: "UnsatisfiedAdmissionChecks"}},
checks: []map[string]any{{"name": "provision", "state": "Ready"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionChecksPending,
blocker: rc.KueueAdmissionChecksPending,
},
{
name: "v0.19 preemption gating wrote QuotaReserved AdmissionGated with its own message",
spec: workloadSpec{conditions: []condition{
pending("AdmissionGated", "Workload requires preemption but preemption is gated"),
{kind: "BlockedOnPreemptionGates", status: "True", reason: "PreemptionGated", message: "Workload could free quota by preemption but a gate is closed"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionPreemptionGated,
blocker: rc.KueueAdmissionPreemptionGated,
},
{
name: "backoff outranks checks reset to Pending",
spec: workloadSpec{
conditions: []condition{
pending("Pending", "The workload has failed admission checks"),
{kind: "Evicted", status: "True", reason: "AdmissionCheck", message: "At least one admission check is false"},
{kind: "Requeued", status: "False", reason: "AdmissionCheck", message: "At least one admission check is false"},
},
checks: []map[string]any{{"name": "provision", "state": "Pending", "message": "Reset to Pending after eviction. Previously: Retry"}},
},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionRequeueBackoff,
blocker: rc.KueueAdmissionRequeueBackoff,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "every taint clause rules out its flavor, not only the last",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient unused quota for cpu in flavor a100, 1 more needed, untolerated taint {gpu true NoSchedule <nil>} in flavor a100, untolerated taint {spot true NoSchedule <nil>} in flavor spot")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "taint and unused quota in the same flavor rule it out",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient unused quota for cpu in flavor spot, 1 more needed, untolerated taint {spot true NoSchedule <nil>} in flavor spot")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "an unrecognized PodSet failure is not covered by another PodSet's quota wait",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set driver: insufficient unused quota for cpu in flavor default, 1 more needed; couldn't assign flavors to pod set workers: something Kueue adds later")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "a flavor without topology support is ruled out",
spec: workloadSpec{conditions: []condition{pending("Pending",
`couldn't assign flavors to pod set driver: insufficient unused quota for cpu in flavor default, 1 more needed; couldn't assign flavors to pod set workers: Flavor "default" does not support TopologyAwareScheduling`)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "an external admission gate after a lingering preemption gate",
spec: workloadSpec{
annotations: map[string]string{admissionGatedByAnnotation: "example.com/gate"},
conditions: []condition{
pending("AdmissionGated", "Admission is gated by: example.com/gate"),
{kind: "BlockedOnPreemptionGates", status: "True", reason: "PreemptionGated", message: legacyUnusedQuota},
},
},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionAdmissionGated,
blocker: rc.KueueAdmissionAdmissionGated,
},
{
name: "an old eviction does not hide a new reason Radar does not recognize",
spec: workloadSpec{conditions: []condition{
pending("Pending", "A reason a future Kueue writes"),
{kind: "Evicted", status: "True", reason: "Preempted", message: "Preempted to accommodate a workload"},
{kind: "Preempted", status: "True", reason: "InClusterQueue", message: "Preempted to accommodate a workload"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
disruption: rc.KueueAdmissionPreempted,
},
{
name: "drained by a stopped ClusterQueue, not requeued until it restarts",
spec: workloadSpec{conditions: []condition{
pending("Pending", "The ClusterQueue is stopped"),
{kind: "Evicted", status: "True", reason: "ClusterQueueStopped", message: "The ClusterQueue is stopped"},
{kind: "Requeued", status: "False", reason: "ClusterQueueStopped", message: "The ClusterQueue is stopped"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionClusterQueueInactive,
blocker: rc.KueueAdmissionClusterQueueInactive,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "drained by a stopped LocalQueue",
spec: workloadSpec{conditions: []condition{
pending("Pending", "The LocalQueue is stopped"),
{kind: "Evicted", status: "True", reason: "LocalQueueStopped", message: "The LocalQueue is stopped"},
{kind: "Requeued", status: "False", reason: "LocalQueueStopped", message: "The LocalQueue is stopped"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueInactive,
blocker: rc.KueueAdmissionLocalQueueInactive,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "preemptions in flight (v0.18 wording)",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyUnusedQuota+". Pending the preemption of 2 workload(s)")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionWaitingForPreemption,
blocker: rc.KueueAdmissionWaitingForPreemption,
},
{
name: "a cluster-side misconfiguration is not blamed on the Workload",
spec: workloadSpec{conditions: []condition{pending("Misconfigured", "invalid namespace selector: a bad operator")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "an elastic slice pinned to its original flavor cannot use the alternate",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: could not assign spot flavor since the original workload is assigned: default, insufficient quota for cpu in flavor default, previously considered podsets requests (0) + current podset request (8) > maximum capacity (4), insufficient unused quota for cpu in flavor spot, 2 more needed")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionExceedsQuota,
blocker: rc.KueueAdmissionExceedsQuota,
},
{
name: "a clause Radar does not understand keeps that flavor from counting as viable",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient unused quota for cpu in flavor spot, 2 more needed, some future restriction on flavor spot")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "skipped for another Workload in the same cycle (v0.18 wording)",
spec: workloadSpec{conditions: []condition{pending("Pending", "Workload no longer fits after processing another workload")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "overlapping preemption targets (v0.18 wording)",
spec: workloadSpec{conditions: []condition{pending("Pending", "Workload has overlapping preemption targets with another workload")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "fits after in-flight preemptions (v0.20 wording, granular reasons off)",
spec: workloadSpec{conditions: []condition{pending("Pending", "Workload has overlapping preemption targets with another workload, but will fit after these preemptions complete")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionWaitingForPreemption,
blocker: rc.KueueAdmissionWaitingForPreemption,
},
{
name: "a PodSet that fits nowhere outranks one that exceeds quota",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set launcher: untolerated taint {gpu true NoSchedule <nil>} in flavor a100; couldn't assign flavors to pod set workers: insufficient quota for cpu in flavor default, previously considered podsets requests (0) + current podset request (8) > maximum capacity (4)")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "preemption failed and will be retried (v0.18 wording)",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyUnusedQuota+". Preempting 1 workload(s) failed, will retry.")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionWaitingForPreemption,
blocker: rc.KueueAdmissionWaitingForPreemption,
},
{
name: "evicted while quota is still reserved: the eviction decides",
spec: workloadSpec{
conditions: []condition{reserved, {kind: "Evicted", status: "True", reason: "AdmissionCheck", message: "At least one admission check is false"}},
checks: []map[string]any{{"name": "provision", "state": "Retry", "message": "Retrying"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionEvicted,
blocker: rc.KueueAdmissionChecksPending,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "a resource the ClusterQueue doesn't offer outranks another resource's quota wait",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient unused quota for cpu in flavor default, 1 more needed, resource nvidia.com/gpu unavailable in ClusterQueue")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionResourceUnavailable,
blocker: rc.KueueAdmissionResourceUnavailable,
},
{
name: "a taint on one flavor and an unknown failure on another is not a confident no-match",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: some future restriction on flavor b, untolerated taint {spot true NoSchedule <nil>} in flavor a")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "a maximum on one flavor and an unknown failure on another is not a confident exceed",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient quota for cpu in flavor a, previously considered podsets requests (0) + current podset request (8) > maximum capacity (4), some future restriction on flavor b")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "a variant's disallowed flavors are out of the running, not unexplained",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient unused quota for cpu in flavor default, 1 more needed, skipping flavor spot due to WorkloadAllowedResourceFlavorAnnotation annotation")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "a flavor without the requested topology level is ruled out",
spec: workloadSpec{conditions: []condition{pending("Pending",
`couldn't assign flavors to pod set main: Flavor "default" does not contain the requested level`)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "v0.20 Deactivated reason left behind after reactivation",
spec: workloadSpec{active: boolPtr(true), conditions: []condition{pending("Deactivated", "The workload is deactivated")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoDecision,
blocker: rc.KueueAdmissionNoDecision,
},
{
name: "no longer on hold, waiting for quota (StatefulSet integration)",
spec: workloadSpec{conditions: []condition{pending("Pending", "Workload no longer on hold; waiting for quota reservation")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoDecision,
blocker: rc.KueueAdmissionNoDecision,
},
{
name: "granular NoMatchingFlavor for a resource the ClusterQueue doesn't offer",
spec: workloadSpec{conditions: []condition{pending("NoMatchingFlavor", legacyNoGPUResource)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionResourceUnavailable,
blocker: rc.KueueAdmissionResourceUnavailable,
},
{
name: "deferred mid-cycle (v0.20 wording, granular reasons off)",
spec: workloadSpec{conditions: []condition{pending("Pending", "Workload was evaluated mid-cycle and is deferred to the next scheduling cycle")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "backoff left behind after quota was reserved again",
spec: workloadSpec{
conditions: []condition{reserved, {kind: "Requeued", status: "False", reason: "PodsReadyTimeout"}},
checks: []map[string]any{{"name": "provision", "state": "Pending"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionChecksPending,
blocker: rc.KueueAdmissionChecksPending,
},
{
name: "a clause Radar doesn't understand fails the PodSet even when it names no flavor",
spec: workloadSpec{conditions: []condition{pending("Pending",
"couldn't assign flavors to pod set main: insufficient unused quota for cpu in flavor default, 1 more needed, resourceUsage cannot store the admitted total for cpu")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "v0.9 quota wording (in cohort)",
spec: workloadSpec{version: "v1beta1", conditions: []condition{pending("Pending", "couldn't assign flavors to pod set main: insufficient unused quota in cohort for cpu in flavor default, 1 more needed")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
},
{
name: "v0.9 Kueue deactivation (InactiveWorkload with its cause)",
spec: workloadSpec{version: "v1beta1", active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "InactiveWorkloadRequeuingLimitExceeded", message: "The workload is deactivated due to exceeding the maximum number of re-queuing retries"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivatedByKueue,
inactive: rc.KueueAdmissionDeactivatedByKueue,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "v0.9 deactivation by a person",
spec: workloadSpec{version: "v1beta1", active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "InactiveWorkload", message: "The workload is deactivated"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivated,
inactive: rc.KueueAdmissionDeactivated,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "a flavor that only serves topology-aware scheduling",
spec: workloadSpec{conditions: []condition{pending("Pending", `couldn't assign flavors to pod set main: Flavor "tas" supports only TopologyAwareScheduling`)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "untolerated taint in every flavor",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyTaintMismatch)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoMatchingFlavor,
blocker: rc.KueueAdmissionNoMatchingFlavor,
},
{
name: "resource the ClusterQueue has no quota for",
spec: workloadSpec{conditions: []condition{pending("Pending", legacyNoGPUResource)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionResourceUnavailable,
blocker: rc.KueueAdmissionResourceUnavailable,
},
{
name: "missing LocalQueue (v0.18)",
spec: workloadSpec{conditions: []condition{pending("Inadmissible", "LocalQueue gpu doesn't exist")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueMissing,
blocker: rc.KueueAdmissionLocalQueueMissing,
},
{
name: "missing LocalQueue (v0.20)",
spec: workloadSpec{conditions: []condition{pending("Misconfigured", "LocalQueue gpu doesn't exist")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueMissing,
blocker: rc.KueueAdmissionLocalQueueMissing,
},
{
name: "LocalQueue being deleted",
spec: workloadSpec{conditions: []condition{pending("Inadmissible", "LocalQueue gpu is terminating or missing")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueMissing,
blocker: rc.KueueAdmissionLocalQueueMissing,
},
{
name: "stopped LocalQueue reads the same as inactive",
spec: workloadSpec{conditions: []condition{pending("Suspended", "LocalQueue gpu is stopped")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueInactive,
blocker: rc.KueueAdmissionLocalQueueInactive,
},
{
name: "inactive LocalQueue",
spec: workloadSpec{conditions: []condition{pending("Inadmissible", "LocalQueue gpu is inactive")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueInactive,
blocker: rc.KueueAdmissionLocalQueueInactive,
},
{
name: "missing ClusterQueue (scheduler wording)",
spec: workloadSpec{conditions: []condition{pending("Pending", "ClusterQueue gpu-team not found")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionClusterQueueMissing,
blocker: rc.KueueAdmissionClusterQueueMissing,
},
{
name: "inactive ClusterQueue is a blocker, not a hold",
spec: workloadSpec{conditions: []condition{pending("Inadmissible", "ClusterQueue gpu-team is inactive")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionClusterQueueInactive,
blocker: rc.KueueAdmissionClusterQueueInactive,
},
{
name: "stopped ClusterQueue in the Workload controller's wording",
spec: workloadSpec{conditions: []condition{pending("Suspended", "ClusterQueue gpu-team is stopped")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionClusterQueueInactive,
blocker: rc.KueueAdmissionClusterQueueInactive,
},
{
name: "namespace not eligible (v0.18)",
spec: workloadSpec{conditions: []condition{pending("Pending", namespaceMismatch)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNamespaceNotEligible,
blocker: rc.KueueAdmissionNamespaceNotEligible,
},
{
name: "namespace not eligible (v0.20)",
spec: workloadSpec{conditions: []condition{pending("Misconfigured", "workload namespace doesn't match ClusterQueue selector")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNamespaceNotEligible,
blocker: rc.KueueAdmissionNamespaceNotEligible,
},
{
name: "the Workload's own requests fail validation",
spec: workloadSpec{conditions: []condition{pending("Misconfigured", "resources validation failed: spec.podSets[0]: Invalid value")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInvalidWorkload,
blocker: rc.KueueAdmissionInvalidWorkload,
},
{
name: "waiting for other Workloads to become PodsReady",
spec: workloadSpec{conditions: []condition{pending("Waiting", "waiting for all admitted workloads to be in PodsReady condition")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionWaitingForPodsReady,
blocker: rc.KueueAdmissionWaitingForPodsReady,
},
{
name: "pending evaluation",
spec: workloadSpec{conditions: []condition{pending("PendingEvaluation", "Workload is pending evaluation in the scheduling queue")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoDecision,
blocker: rc.KueueAdmissionNoDecision,
},
{
name: "admission gate cleared, not yet evaluated",
spec: workloadSpec{conditions: []condition{pending("Pending", "AdmissionGatedBy cleared, waiting for quota reservation")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionNoDecision,
blocker: rc.KueueAdmissionNoDecision,
},
{
name: "admission gated by an external controller",
spec: workloadSpec{
annotations: map[string]string{admissionGatedByAnnotation: "example.com/gate"},
conditions: []condition{pending("AdmissionGated", "Admission is gated by: example.com/gate")},
},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionAdmissionGated,
blocker: rc.KueueAdmissionAdmissionGated,
},
{
name: "AdmissionGated without the current annotation is not an admission gate",
spec: workloadSpec{conditions: []condition{pending("AdmissionGated", "Admission is gated by: example.com/gate")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "preemption gated in the current attempt",
spec: workloadSpec{conditions: []condition{
pending("Pending", legacyUnusedQuota),
{kind: "BlockedOnPreemptionGates", status: "True", reason: "PreemptionGated", message: legacyUnusedQuota},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionPreemptionGated,
blocker: rc.KueueAdmissionPreemptionGated,
},
{
name: "gate opened, a later attempt reports a different reason",
spec: workloadSpec{conditions: []condition{
pending("Pending", "LocalQueue gpu doesn't exist"),
{kind: "BlockedOnPreemptionGates", status: "True", reason: "PreemptionGated", message: legacyUnusedQuota},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionLocalQueueMissing,
blocker: rc.KueueAdmissionLocalQueueMissing,
},
{
name: "granular preemption-gated reason",
spec: workloadSpec{conditions: []condition{pending("PreemptionGated", "blocked")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionPreemptionGated,
blocker: rc.KueueAdmissionPreemptionGated,
},
{
name: "waiting for preempted Workloads to release quota",
spec: workloadSpec{conditions: []condition{pending("WaitingForPreemptedWorkloads", "waiting")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionWaitingForPreemption,
blocker: rc.KueueAdmissionWaitingForPreemption,
},
{
name: "unknown reason keeps native evidence",
spec: workloadSpec{conditions: []condition{pending("SomethingNew", "a future Kueue reason")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionUnrecognized,
blocker: rc.KueueAdmissionUnrecognized,
},
{
name: "quota reserved, waiting on admission checks",
spec: workloadSpec{
conditions: []condition{reserved, {kind: "Admitted", status: "False", reason: "UnsatisfiedChecks", message: "The workload has not all checks ready"}},
checks: []map[string]any{{"name": "provision", "state": "Pending", "message": "Waiting for capacity"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionChecksPending,
blocker: rc.KueueAdmissionChecksPending,
},
{
name: "quota reserved, a check retrying and no Admitted condition yet",
spec: workloadSpec{
conditions: []condition{reserved},
checks: []map[string]any{{"name": "policy", "state": "Ready"}, {"name": "provision", "state": "Retry", "message": "Retrying"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionChecksPending,
blocker: rc.KueueAdmissionChecksPending,
},
{
name: "quota reserved and nothing reported blocking",
spec: workloadSpec{conditions: []condition{reserved}},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionQuotaReserved,
},
{
name: "delayed topology assignment",
spec: workloadSpec{conditions: []condition{reserved, {kind: "Admitted", status: "False", reason: "PendingDelayedTopologyRequests"}}},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionTopologyAssignmentPending,
blocker: rc.KueueAdmissionTopologyAssignmentPending,
},
{
name: "rejected admission check without deactivation",
spec: workloadSpec{
conditions: []condition{reserved, {kind: "Admitted", status: "False", reason: "UnsatisfiedChecks"}},
checks: []map[string]any{{"name": "policy", "state": "Ready"}, {"name": "capacity", "state": "Rejected", "message": "Provisioning request cannot satisfy this workload"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionCheckRejected,
blocker: rc.KueueAdmissionCheckRejected,
},
{
name: "rejected check after Kueue decided to deactivate",
spec: workloadSpec{
conditions: []condition{reserved, {kind: "DeactivationTarget", status: "True", reason: "AdmissionCheck", message: "Admission check(s): [capacity], were in the Rejected state"}},
checks: []map[string]any{{"name": "capacity", "state": "Rejected"}},
},
phase: rc.KueuePhaseQuotaReserved,
state: rc.KueueAdmissionDeactivationPending,
inactive: rc.KueueAdmissionDeactivationPending,
},
{
name: "deactivated, not yet acknowledged",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{pending("Pending", legacyUnusedQuota)}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivated,
inactive: rc.KueueAdmissionDeactivated,
},
{
name: "deactivated by Kueue after a rejected check",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
pending("Pending", "The workload is deactivated due to Admission check(s): [capacity], were in the Rejected state"),
{kind: "Evicted", status: "True", reason: "DeactivatedDueToAdmissionCheck", message: "The workload is deactivated due to Admission check(s): [capacity], were in the Rejected state"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivatedByKueue,
inactive: rc.KueueAdmissionDeactivatedByKueue,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "deactivated by Kueue at the requeuing limit",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "DeactivatedDueToRequeuingLimitExceeded", message: "The workload is deactivated due to exceeding the maximum number of re-queuing retries"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivatedByKueue,
inactive: rc.KueueAdmissionDeactivatedByKueue,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "Kueue v0.18 deactivated an already-evicted Workload: the cause is not recorded",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
pending("Pending", "Exceeded the PodsReady timeout ml/train"),
{kind: "Evicted", status: "True", reason: "PodsReadyTimeout", message: "Exceeded the PodsReady timeout ml/train"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivated,
inactive: rc.KueueAdmissionDeactivated,
disruption: rc.KueueAdmissionPodsReadyTimeout,
},
{
name: "preempted, then deactivated by Kueue: the cause stays on the inactive fact",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
{kind: "Preempted", status: "True", reason: "InClusterQueue", message: "Preempted to accommodate a workload"},
{kind: "Evicted", status: "True", reason: "DeactivatedDueToAdmissionCheck", message: "The workload is deactivated due to a rejected check"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivatedByKueue,
inactive: rc.KueueAdmissionDeactivatedByKueue,
disruption: rc.KueueAdmissionPreempted,
},
{
name: "deactivated by a person and acknowledged",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "Deactivated", message: "The workload is deactivated"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivated,
inactive: rc.KueueAdmissionDeactivated,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "spec.active false while Kueue's own target is still set",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
{kind: "DeactivationTarget", status: "True", reason: "MaximumExecutionTimeExceeded", message: "exceeding the maximum execution time"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionDeactivatedByKueue,
inactive: rc.KueueAdmissionDeactivatedByKueue,
},
{
name: "right after an eviction QuotaReserved carries the eviction message",
spec: workloadSpec{conditions: []condition{
pending("Pending", "Preempted to accommodate a workload"),
{kind: "Evicted", status: "True", reason: "Preempted", message: "Preempted to accommodate a workload"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionPreempted,
blocker: rc.KueueAdmissionUnrecognized,
disruption: rc.KueueAdmissionPreempted,
},
{
name: "pending evaluation right after an eviction",
spec: workloadSpec{conditions: []condition{
pending("PendingEvaluation", "Workload is pending evaluation in the scheduling queue"),
{kind: "Evicted", status: "True", reason: "ClusterQueueStopped", message: "The ClusterQueue is stopped"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionEvicted,
blocker: rc.KueueAdmissionNoDecision,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "reactivated with the old QuotaReserved unchanged",
spec: workloadSpec{active: boolPtr(true), conditions: []condition{
pending("Pending", "The workload is deactivated"),
{kind: "Evicted", status: "True", reason: "Deactivated", message: "The workload is deactivated"},
{kind: "Requeued", status: "True", reason: "Reactivated", message: "The workload was reactivated"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionEvicted,
blocker: rc.KueueAdmissionUnrecognized,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "reactivated, old deactivation eviction still reported, now waiting on quota",
spec: workloadSpec{active: boolPtr(true), conditions: []condition{
pending("Pending", legacyUnusedQuota),
{kind: "Evicted", status: "True", reason: "DeactivatedDueToAdmissionCheck", message: "The workload is deactivated"},
{kind: "Requeued", status: "True", reason: "Reactivated", message: "The workload was reactivated"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
disruption: rc.KueueAdmissionEvicted,
},
{
name: "on hold",
spec: workloadSpec{conditions: []condition{pending("OnHold", "scaled to zero")}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionOnHold,
inactive: rc.KueueAdmissionOnHold,
},
{
name: "PodsReadyTimeout eviction waiting out backoff",
spec: workloadSpec{conditions: []condition{
pending("Pending", "Exceeded the PodsReady timeout ml/train"),
{kind: "Evicted", status: "True", reason: "PodsReadyTimeout", message: "Exceeded the PodsReady timeout ml/train"},
{kind: "Requeued", status: "False", reason: "PodsReadyTimeout", message: "Exceeded the PodsReady timeout ml/train"},
}},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionRequeueBackoff,
blocker: rc.KueueAdmissionRequeueBackoff,
disruption: rc.KueueAdmissionPodsReadyTimeout,
},
{
name: "PodsReadyTimeout eviction still releasing quota",
spec: workloadSpec{conditions: []condition{
reserved, admitted,
{kind: "Evicted", status: "True", reason: "PodsReadyTimeout", message: "Exceeded the PodsReady timeout ml/train"},
}},
phase: rc.KueuePhaseAdmitted,
state: rc.KueueAdmissionPodsReadyTimeout,
disruption: rc.KueueAdmissionPodsReadyTimeout,
},
{
name: "after eviction, checks reset to Pending do not decide while pending",
spec: workloadSpec{
conditions: []condition{
pending("Pending", legacyUnusedQuota),
{kind: "Evicted", status: "True", reason: "Preempted", message: "Preempted to accommodate a workload"},
{kind: "Preempted", status: "True", reason: "InClusterQueue", message: "Preempted to accommodate a workload"},
{kind: "Requeued", status: "True", reason: "Preempted"},
},
checks: []map[string]any{{"name": "provision", "state": "Pending", "message": "Reset to Pending after eviction. Previously: Ready"}},
},
phase: rc.KueuePhasePending,
state: rc.KueueAdmissionInsufficientQuota,
blocker: rc.KueueAdmissionInsufficientQuota,
disruption: rc.KueueAdmissionPreempted,
},
{
name: "old failure after recovery",
spec: workloadSpec{
conditions: []condition{
reserved, admitted,
{kind: "Evicted", status: "False", reason: "QuotaReserved", message: "Previously: Exceeded the PodsReady timeout"},
{kind: "Preempted", status: "False", reason: "QuotaReserved", message: "Previously: Preempted"},
{kind: "BlockedOnPreemptionGates", status: "False", reason: "QuotaReserved"},
{kind: "Requeued", status: "True", reason: "BackoffFinished"},
},
checks: []map[string]any{{"name": "provision", "state": "Ready", "message": "Previously retried"}},
},
phase: rc.KueuePhaseAdmitted,
state: rc.KueueAdmissionAdmitted,
},
{
name: "finished successfully",
spec: workloadSpec{conditions: []condition{reserved, admitted, {kind: "Finished", status: "True", reason: "Succeeded", message: "Job finished successfully"}}},
phase: rc.KueuePhaseFinished,
state: rc.KueueAdmissionFinished,
},
{
name: "finished while inactive keeps finished",
spec: workloadSpec{active: boolPtr(false), conditions: []condition{
{kind: "Finished", status: "True", reason: "Failed", message: "Job failed"},
{kind: "Evicted", status: "True", reason: "Deactivated", message: "The workload is deactivated"},
}},
phase: rc.KueuePhaseFinished,
state: rc.KueueAdmissionFinished,
disruption: rc.KueueAdmissionEvicted,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
got := classify(t, test.spec)
admission := got.Admission
if got.Phase != test.phase || admission.State != test.state || factCode(admission.Inactive) != test.inactive ||
factCode(admission.Blocker) != test.blocker || factCode(admission.Disruption) != test.disruption {
wire, _ := json.Marshal(admission)
t.Fatalf("phase=%s admission=%s; want phase=%s state=%s inactive=%q blocker=%q disruption=%q",
got.Phase, wire, test.phase, test.state, test.inactive, test.blocker, test.disruption)
}
})
}
}
func TestClassifyKueueWorkloadKeepsNativeEvidence(t *testing.T) {
got := classify(t, workloadSpec{conditions: []condition{pending("SomethingNew", "a future Kueue reason")}}).Admission
want := &rc.KueueAdmissionFact{
Code: rc.KueueAdmissionUnrecognized, Source: rc.KueueFactSourceCondition, Name: "QuotaReserved",
Reason: "SomethingNew", Message: "a future Kueue reason", Since: "2026-11-01T10:00:00Z",
}
if !reflect.DeepEqual(got.Blocker, want) {
t.Fatalf("blocker = %+v, want %+v", got.Blocker, want)
}
rejected := classify(t, workloadSpec{
conditions: []condition{reserved},
checks: []map[string]any{{"name": "capacity", "state": "Rejected", "message": "Provisioning request cannot satisfy this workload"}},
}).Admission
wantCheck := &rc.KueueAdmissionFact{
Code: rc.KueueAdmissionCheckRejected, Source: rc.KueueFactSourceAdmissionCheck, Name: "capacity",
Reason: "Rejected", Message: "Provisioning request cannot satisfy this workload",
}
if !reflect.DeepEqual(rejected.Blocker, wantCheck) || rejected.Inactive != nil {
t.Fatalf("rejected check = %+v inactive=%+v, want %+v and no inactive fact", rejected.Blocker, rejected.Inactive, wantCheck)
}
deactivated := classify(t, workloadSpec{active: boolPtr(false)}).Admission
if !reflect.DeepEqual(deactivated.Inactive, &rc.KueueAdmissionFact{Code: rc.KueueAdmissionDeactivated, Source: rc.KueueFactSourceSpec, Name: "active"}) || deactivated.Disruption != nil {
t.Fatalf("unacknowledged deactivation = %+v", deactivated)
}
pendingTarget := classify(t, workloadSpec{conditions: []condition{reserved, {kind: "DeactivationTarget", status: "True", reason: "RequeuingLimitExceeded", message: "exceeding the maximum number of re-queuing retries"}}}).Admission
if pendingTarget.Inactive == nil || pendingTarget.Inactive.Reason != "RequeuingLimitExceeded" {
t.Fatalf("deactivation cause lost: %+v", pendingTarget.Inactive)
}
noDecision := classify(t, workloadSpec{}).Admission
if !reflect.DeepEqual(noDecision.Blocker, &rc.KueueAdmissionFact{Code: rc.KueueAdmissionNoDecision}) {
t.Fatalf("no_decision without native evidence = %+v", noDecision.Blocker)
}
}
func TestClassifyKueueWorkloadStaleEvidence(t *testing.T) {
t.Run("stale negative condition is not a confirmed blocker", func(t *testing.T) {
quota := pending("Pending", legacyUnusedQuota)
quota.observedGeneration = 1
got := classify(t, workloadSpec{generation: 2, conditions: []condition{quota}}).Admission
if got.State != rc.KueueAdmissionInsufficientQuota || !got.Stale || got.Blocker == nil || !got.Blocker.Stale {
t.Fatalf("admission = %+v", got)
}
})
t.Run("stale positive condition is not a confirmed admission", func(t *testing.T) {
// Kueue rewrites spec.priority when a WorkloadPriorityClass changes and
// leaves Admitted untouched; the code survives, flagged as last reported.
quota, admit := reserved, admitted
quota.observedGeneration, admit.observedGeneration = 2, 2
got := classify(t, workloadSpec{generation: 3, conditions: []condition{quota, admit}}).Admission
if got.State != rc.KueueAdmissionAdmitted || !got.Stale || got.Attention != rc.KueueAttentionNone {
t.Fatalf("admission = %+v", got)
}
})
t.Run("a stale reservation makes the blocker it selected stale", func(t *testing.T) {
quota := reserved
quota.observedGeneration = 1
got := classify(t, workloadSpec{generation: 2, conditions: []condition{quota}, checks: []map[string]any{{"name": "provision", "state": "Pending"}}}).Admission
if got.State != rc.KueueAdmissionChecksPending || !got.Stale {
t.Fatalf("admission = %+v", got)
}
})
t.Run("a stale reservation does not discount a current backoff", func(t *testing.T) {
quota := pending("Pending", "Exceeded the PodsReady timeout")
quota.observedGeneration = 1
got := classify(t, workloadSpec{generation: 2, conditions: []condition{quota,
{kind: "Requeued", status: "False", reason: "PodsReadyTimeout", observedGeneration: 2}}}).Admission
if got.State != rc.KueueAdmissionRequeueBackoff || got.Stale {
t.Fatalf("admission = %+v", got)
}
})
t.Run("stale finish", func(t *testing.T) {
got := classify(t, workloadSpec{generation: 4, conditions: []condition{{kind: "Finished", status: "True", reason: "Succeeded", observedGeneration: 3}}}).Admission
if got.State != rc.KueueAdmissionFinished || !got.Stale {
t.Fatalf("admission = %+v", got)
}
})
t.Run("current spec decides even when a secondary axis is stale", func(t *testing.T) {
got := classify(t, workloadSpec{generation: 3, active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "Deactivated", observedGeneration: 2},
}}).Admission
if got.State != rc.KueueAdmissionDeactivated || got.Stale || got.Disruption == nil || !got.Disruption.Stale {
t.Fatalf("admission = %+v", got)
}
uncertain := classify(t, workloadSpec{generation: 3, active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "Preempted", observedGeneration: 2},
}}).Admission
if uncertain.State != rc.KueueAdmissionDeactivated || uncertain.Stale || uncertain.Attention != rc.KueueAttentionUnknown {
t.Fatalf("admission = %+v", uncertain)
}
})
t.Run("Kueue's own spec.active flip does not make its recorded cause stale", func(t *testing.T) {
got := classify(t, workloadSpec{generation: 2, active: boolPtr(false), conditions: []condition{
{kind: "DeactivationTarget", status: "True", reason: "AdmissionCheck", observedGeneration: 1},
}}).Admission
if got.State != rc.KueueAdmissionDeactivatedByKueue || got.Stale || got.Inactive == nil || !got.Inactive.Stale {
t.Fatalf("admission = %+v", got)
}
})
t.Run("current generation and unreported generation are not stale", func(t *testing.T) {
quota := pending("Pending", legacyUnusedQuota)
quota.observedGeneration = 2
if got := classify(t, workloadSpec{generation: 2, conditions: []condition{quota}}).Admission; got.Stale || got.Blocker.Stale {
t.Fatalf("current generation = %+v", got)
}
if got := classify(t, workloadSpec{generation: 2, conditions: []condition{pending("Pending", legacyUnusedQuota)}}).Admission; got.Stale {
t.Fatalf("unreported generation = %+v", got)
}
})
}
func TestDeactivationCauseFromBeforeAReactivationIsNotThisDeactivationsCause(t *testing.T) {
// Kueue deactivated it, someone reactivated it, then someone deactivated it
// again before any new reservation: Evicted still carries the old cause.
u := buildWorkload(workloadSpec{active: boolPtr(false), conditions: []condition{
{kind: "Evicted", status: "True", reason: "DeactivatedDueToAdmissionCheck", message: "deactivated due to a rejected check"},
{kind: "Requeued", status: "True", reason: "Reactivated", message: "The workload was reactivated"},
}})
conditions, _, _ := unstructured.NestedSlice(u.Object, "status", "conditions")
conditions[0].(map[string]any)["observedGeneration"] = int64(1)
conditions[1].(map[string]any)["observedGeneration"] = int64(2)
_ = unstructured.SetNestedSlice(u.Object, conditions, "status", "conditions")
u.SetGeneration(3)
got, _ := ClassifyKueueWorkload(u)
if got.Admission.State != rc.KueueAdmissionDeactivated || got.Admission.Attention != rc.KueueAttentionUnknown {
t.Fatalf("admission = %+v", got.Admission)
}
// Kueue deactivated it again after the reactivation and rewrote the
// already-True Evicted condition: same transition time, newer generation.
conditions[0].(map[string]any)["observedGeneration"] = int64(3)
_ = unstructured.SetNestedSlice(u.Object, conditions, "status", "conditions")
got, _ = ClassifyKueueWorkload(u)
if got.Admission.State != rc.KueueAdmissionDeactivatedByKueue {
t.Fatalf("rewritten cause = %+v", got.Admission)
}
}
func TestAwaitingAdmissionEncodesTheDocumentedPopulation(t *testing.T) {
for _, test := range []struct {
name string
u *unstructured.Unstructured
want bool
}{
{"waiting for quota", buildWorkload(workloadSpec{conditions: []condition{pending("Pending", legacyUnusedQuota)}}), true},
{"no queue name", buildWorkload(workloadSpec{noQueue: true}), false},
{"admitted", buildWorkload(workloadSpec{conditions: []condition{reserved, admitted}}), false},
{"deactivated", buildWorkload(workloadSpec{active: boolPtr(false)}), false},
{"concurrent-admission variant", loadFixture(t, "concurrent-variant.yaml"), false},
{"being deleted", func() *unstructured.Unstructured {
u := buildWorkload(workloadSpec{conditions: []condition{pending("Pending", legacyUnusedQuota)}})
now := metav1.Now()
u.SetDeletionTimestamp(&now)
return u
}(), false},
} {
t.Run(test.name, func(t *testing.T) {
got, _ := ClassifyKueueWorkload(test.u)
if got.AwaitingAdmission() != test.want {
t.Fatalf("AwaitingAdmission = %t, classification %+v", got.AwaitingAdmission(), got)
}
})
}
}
func TestClassifyKueueWorkloadAttention(t *testing.T) {
for _, test := range []struct {
spec workloadSpec
attention rc.KueueAttention
}{
{workloadSpec{conditions: []condition{reserved, admitted}}, rc.KueueAttentionNone},
{workloadSpec{conditions: []condition{pending("Pending", legacyUnusedQuota)}}, rc.KueueAttentionWaiting},
{workloadSpec{conditions: []condition{pending("Pending", legacyExceedsQuota)}}, rc.KueueAttentionUnknown},
{workloadSpec{conditions: []condition{pending("ExceedsMaxQuota", legacyExceedsQuota)}}, rc.KueueAttentionUnknown},
{workloadSpec{active: boolPtr(false)}, rc.KueueAttentionPaused},
{workloadSpec{conditions: []condition{pending("Inadmissible", "LocalQueue gpu doesn't exist")}}, rc.KueueAttentionActionNeeded},
{workloadSpec{conditions: []condition{pending("Inadmissible", "ClusterQueue gpu-team is inactive")}}, rc.KueueAttentionUnknown},
{workloadSpec{conditions: []condition{pending("SomethingNew", "x")}}, rc.KueueAttentionUnknown},
} {
if got := classify(t, test.spec).Admission; got.Attention != test.attention {
t.Errorf("state %s attention = %s, want %s", got.State, got.Attention, test.attention)
}
}
}
func TestEveryAdmissionCodeHasAnAttentionClass(t *testing.T) {
// unrecognized and the queue-inactive codes are deliberately unknown; any
// other code falling through to the default was forgotten.
deliberatelyUnknown := map[rc.KueueAdmissionCode]bool{
rc.KueueAdmissionUnrecognized: true, rc.KueueAdmissionLocalQueueInactive: true, rc.KueueAdmissionClusterQueueInactive: true,
rc.KueueAdmissionDRAResourcesUnresolved: true,
// The message covers only the flavors Kueue tried in its last attempt.
rc.KueueAdmissionExceedsQuota: true, rc.KueueAdmissionNoMatchingFlavor: true, rc.KueueAdmissionTopologyPlacementFailed: true,
}
for _, code := range allAdmissionCodes {
if got := attention(code); got == rc.KueueAttentionUnknown && !deliberatelyUnknown[code] {
t.Errorf("code %s has no attention class", code)
}
}
}
var allAdmissionCodes = []rc.KueueAdmissionCode{
rc.KueueAdmissionFinished, rc.KueueAdmissionAdmitted, rc.KueueAdmissionQuotaReserved,
rc.KueueAdmissionDeactivated, rc.KueueAdmissionDeactivatedByKueue, rc.KueueAdmissionDeactivationPending, rc.KueueAdmissionOnHold,
rc.KueueAdmissionNoDecision, rc.KueueAdmissionInsufficientQuota, rc.KueueAdmissionExceedsQuota, rc.KueueAdmissionNoMatchingFlavor, rc.KueueAdmissionResourceUnavailable,
rc.KueueAdmissionTopologyPlacementFailed, rc.KueueAdmissionWaitingForPreemption, rc.KueueAdmissionPreemptionGated,
rc.KueueAdmissionAdmissionGated, rc.KueueAdmissionNamespaceNotEligible, rc.KueueAdmissionLocalQueueMissing,
rc.KueueAdmissionLocalQueueInactive, rc.KueueAdmissionClusterQueueMissing, rc.KueueAdmissionClusterQueueInactive,
rc.KueueAdmissionCheckRejected, rc.KueueAdmissionChecksPending, rc.KueueAdmissionTopologyAssignmentPending,
rc.KueueAdmissionWaitingForPodsReady, rc.KueueAdmissionRequeueBackoff, rc.KueueAdmissionInvalidWorkload, rc.KueueAdmissionDRAResourcesUnresolved,
rc.KueueAdmissionUnrecognized, rc.KueueAdmissionPreempted, rc.KueueAdmissionPodsReadyTimeout, rc.KueueAdmissionEvicted,
}
func TestClassifyKueueWorkloadPreservesExistingDecision(t *testing.T) {
for _, test := range []struct {
name string
spec workloadSpec
decision rc.SchedulingDecision
}{
{name: "inactive ClusterQueue stays unsatisfied", spec: workloadSpec{conditions: []condition{pending("Inadmissible", "ClusterQueue gpu-team is inactive")}}, decision: rc.SchedulingDecisionUnsatisfied},
{name: "rejected check stays unsatisfied", spec: workloadSpec{conditions: []condition{reserved}, checks: []map[string]any{{"name": "capacity", "state": "Rejected"}}}, decision: rc.SchedulingDecisionUnsatisfied},
{name: "deactivated stays held", spec: workloadSpec{active: boolPtr(false)}, decision: rc.SchedulingDecisionHeld},
} {
t.Run(test.name, func(t *testing.T) {
u := buildWorkload(test.spec)
observation := onlyObservation(t, ForResource(u, rc.TierBasic))
classification, _ := ClassifyKueueWorkload(u)
if observation.Decision != test.decision {
t.Fatalf("decision = %s, want %s", observation.Decision, test.decision)
}
if observation.Kueue.Admission == nil || !reflect.DeepEqual(*observation.Kueue.Admission, classification.Admission) ||
observation.Kueue.Phase != classification.Phase || observation.Kueue.Outcome != classification.Outcome {
t.Fatalf("ForResource and ClassifyKueueWorkload disagree: %+v vs %+v", observation.Kueue, classification)
}
})
}
}
func TestClassifyKueueWorkloadRejectsOtherObjects(t *testing.T) {
foreign := buildWorkload(workloadSpec{})
foreign.SetAPIVersion("example.io/v1beta2")
if _, ok := ClassifyKueueWorkload(foreign); ok {
t.Fatal("Workload kind in another group was classified")
}
if _, ok := ClassifyKueueWorkload(&corev1.Pod{}); ok {
t.Fatal("typed object was classified")
}
}
func TestPrimaryConditionFollowsTheCurrentAttempt(t *testing.T) {
externallyGated := buildWorkload(workloadSpec{
annotations: map[string]string{admissionGatedByAnnotation: "example.com/gate"},
conditions: []condition{
pending("AdmissionGated", "Admission is gated by: example.com/gate"),
{kind: "BlockedOnPreemptionGates", status: "True", reason: "PreemptionGated", message: legacyUnusedQuota},
},
})
observation := onlyObservation(t, ForResource(externallyGated, rc.TierBasic))
if observation.PrimaryCondition == nil || observation.PrimaryCondition.Type != "QuotaReserved" || observation.Decision != rc.SchedulingDecisionHeld {
t.Fatalf("external gate hidden by a lingering preemption gate: decision=%s primary=%+v", observation.Decision, observation.PrimaryCondition)
}
lingering := buildWorkload(workloadSpec{conditions: []condition{
pending("Inadmissible", "LocalQueue gpu doesn't exist"),
{kind: "BlockedOnPreemptionGates", status: "True", reason: "PreemptionGated", message: legacyUnusedQuota},
}})
if primary := onlyObservation(t, ForResource(lingering, rc.TierBasic)).PrimaryCondition; primary == nil || primary.Type != "QuotaReserved" {
t.Fatalf("lingering preemption gate became the primary condition: %+v", primary)
}
withChecks := buildWorkload(workloadSpec{conditions: []condition{
pending("Pending", legacyUnusedQuota),
{kind: "Admitted", status: "False", reason: "NoReservationUnsatisfiedChecks", message: "The workload has no reservation and not all checks ready"},
}})
if primary := onlyObservation(t, ForResource(withChecks, rc.TierBasic)).PrimaryCondition; primary == nil || primary.Type != "QuotaReserved" {
t.Fatalf("generic NoReservationUnsatisfiedChecks hid the quota reason: %+v", primary)
}
}
// The shared vector lists every code once; the k8s-ui presentation test reads
// the same file, so a code added here without a UI label fails there.
func TestSharedCodeVectorMatchesDeclaredCodes(t *testing.T) {
raw, err := os.ReadFile("testdata/kueue_admission_codes.json")
if err != nil {
t.Fatal(err)
}
var vector struct {
Codes []string `json:"codes"`
Attention []string `json:"attention"`
}
if err := json.Unmarshal(raw, &vector); err != nil {
t.Fatal(err)
}
declared := map[string][]string{}
file, err := parser.ParseFile(token.NewFileSet(), "../resourcecontext/kueue_admission.go", nil, 0)
if err != nil {
t.Fatal(err)
}
for _, decl := range file.Decls {
gen, ok := decl.(*ast.GenDecl)
if !ok || gen.Tok != token.CONST {
continue
}
for _, spec := range gen.Specs {
value := spec.(*ast.ValueSpec)
ident, ok := value.Type.(*ast.Ident)
if !ok || len(value.Values) != 1 {
continue
}
literal, ok := value.Values[0].(*ast.BasicLit)
if !ok {
continue
}
unquoted, _ := strconv.Unquote(literal.Value)
declared[ident.Name] = append(declared[ident.Name], unquoted)
}
}
sameSet := func(name string, got, want []string) {
sort.Strings(got)
sort.Strings(want)
if !reflect.DeepEqual(got, want) {
t.Errorf("%s: shared vector %v, declared %v", name, want, got)
}
}
sameSet("KueueAdmissionCode", declared["KueueAdmissionCode"], vector.Codes)
sameSet("KueueAttention", declared["KueueAttention"], vector.Attention)
listed := make([]string, 0, len(allAdmissionCodes))
for _, code := range allAdmissionCodes {
listed = append(listed, string(code))
}
sameSet("allAdmissionCodes", listed, vector.Codes)
}
func TestPendingChecksCiteTheCheck(t *testing.T) {
got := classify(t, workloadSpec{
conditions: []condition{reserved, {kind: "Admitted", status: "False", reason: "UnsatisfiedChecks", message: "The workload has not all checks ready"}},
checks: []map[string]any{{"name": "policy", "state": "Ready"}, {"name": "provision", "state": "Pending", "message": "Waiting for capacity"}},
}).Admission
if got.Blocker == nil || got.Blocker.Source != rc.KueueFactSourceAdmissionCheck || got.Blocker.Name != "provision" || got.Blocker.Message != "Waiting for capacity" {
t.Fatalf("blocker = %+v", got.Blocker)
}
}
// decision is the controller-neutral scheduling view and keeps its own
// meaning; these pairs are where it and the classification deliberately
// differ, so consumers know to read state and attention instead.
func TestDecisionAndStateDisagreeOnlyWhereDocumented(t *testing.T) {
for _, test := range []struct {
name string
spec workloadSpec
decision rc.SchedulingDecision
state rc.KueueAdmissionCode
attention rc.KueueAttention
}{
{"inactive queue", workloadSpec{conditions: []condition{pending("Inadmissible", "ClusterQueue gpu-team is inactive")}}, rc.SchedulingDecisionUnsatisfied, rc.KueueAdmissionClusterQueueInactive, rc.KueueAttentionUnknown},
{"external admission gate", workloadSpec{annotations: map[string]string{admissionGatedByAnnotation: "example.com/gate"}, conditions: []condition{pending("AdmissionGated", "Admission is gated by: example.com/gate")}}, rc.SchedulingDecisionHeld, rc.KueueAdmissionAdmissionGated, rc.KueueAttentionWaiting},
{"Kueue is deactivating", workloadSpec{conditions: []condition{reserved, {kind: "DeactivationTarget", status: "True", reason: "AdmissionCheck"}}}, rc.SchedulingDecisionUnknown, rc.KueueAdmissionDeactivationPending, rc.KueueAttentionActionNeeded},
} {
t.Run(test.name, func(t *testing.T) {
u := buildWorkload(test.spec)
observation := onlyObservation(t, ForResource(u, rc.TierBasic))
admission := observation.Kueue.Admission
if observation.Decision != test.decision || admission.State != test.state || admission.Attention != test.attention {
t.Fatalf("decision=%s state=%s attention=%s", observation.Decision, admission.State, admission.Attention)
}
})
}
}
func TestTruncatedFlavorMessageProvesOnlyItsCompleteSegments(t *testing.T) {
var clauses []string
for i := 0; len(strings.Join(clauses, ", ")) < 33*1024; i++ {
clauses = append(clauses, fmt.Sprintf("insufficient unused quota for example.com/r%03d in flavor default, 1 more needed", i))
}
full := "couldn't assign flavors to pod set main: " + strings.Join(clauses, ", ") +
", insufficient quota for cpu in flavor default, previously considered podsets requests (0) + current podset request (8) > maximum capacity (4)"
// Kueue's TruncateConditionMessage: cut to 32 KiB, ending in " ...".
truncated := full[:32*1024-len(" ...")] + " ..."
got := classify(t, workloadSpec{conditions: []condition{pending("Pending", truncated)}}).Admission
if got.State != rc.KueueAdmissionUnrecognized {
t.Fatalf("truncated message read as %s", got.State)
}
// A complete earlier PodSet that fits nowhere is still proof.
proven := "couldn't assign flavors to pod set launcher: insufficient quota for cpu in flavor a, previously considered podsets requests (0) + current podset request (8) > maximum capacity (4); couldn't assign flavors to pod set " + truncated[len("couldn't assign flavors to pod set "):]
proven = proven[:32*1024-len(" ...")] + " ..."
if got := classify(t, workloadSpec{conditions: []condition{pending("Pending", proven)}}).Admission; got.State != rc.KueueAdmissionExceedsQuota {
t.Fatalf("complete segment before the cut read as %s", got.State)
}
}
func TestConcurrentAdmissionParentDefersToItsVariants(t *testing.T) {
u := buildWorkload(workloadSpec{conditions: []condition{pending("PendingEvaluation", "Workload is pending evaluation in the scheduling queue")}})
u.SetLabels(map[string]string{concurrentAdmissionParentLabel: "true"})
got, _ := ClassifyKueueWorkload(u)
if !got.Parent || got.Admission.State != rc.KueueAdmissionNoDecision || got.Admission.Attention != rc.KueueAttentionUnknown || !got.AwaitingAdmission() {
t.Fatalf("parent = %+v", got)
}
}