pkg/k8score/drain_plan_test.go
Rama main · réplica de lectura del repositorio original.
package k8score
import (
"context"
"fmt"
"strings"
"sync"
"testing"
"time"
corev1 "k8s.io/api/core/v1"
policyv1 "k8s.io/api/policy/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/kubernetes/fake"
k8stesting "k8s.io/client-go/testing"
)
func drainTestPod(name string, mutate ...func(*corev1.Pod)) corev1.Pod {
yes := true
pod := corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: "shop",
Labels: map[string]string{"app": "web"},
OwnerReferences: []metav1.OwnerReference{{
Kind: "ReplicaSet", Name: "web-abc", Controller: &yes,
}},
},
Spec: corev1.PodSpec{NodeName: "worker-1"},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
}
for _, m := range mutate {
m(&pod)
}
return pod
}
func withOwner(kind string, controller bool) func(*corev1.Pod) {
return func(p *corev1.Pod) {
c := controller
p.OwnerReferences = []metav1.OwnerReference{{Kind: kind, Name: "owner", Controller: &c}}
}
}
func withoutOwner(p *corev1.Pod) { p.OwnerReferences = nil }
func withEmptyDir(p *corev1.Pod) {
p.Spec.Volumes = []corev1.Volume{{Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}}
}
func withMirror(p *corev1.Pod) { p.Annotations = map[string]string{corev1.MirrorPodAnnotationKey: "x"} }
func withDeletionTimestamp(p *corev1.Pod) {
now := metav1.Now()
p.DeletionTimestamp = &now
}
func withPhase(ph corev1.PodPhase) func(*corev1.Pod) {
return func(p *corev1.Pod) { p.Status.Phase = ph }
}
func withReady(ready bool) func(*corev1.Pod) {
return func(p *corev1.Pod) {
st := corev1.ConditionFalse
if ready {
st = corev1.ConditionTrue
}
p.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: st}}
}
}
func pdb(ns, name string, selector *metav1.LabelSelector, allowed int32) policyv1.PodDisruptionBudget {
return policyv1.PodDisruptionBudget{
ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Generation: 1},
Spec: policyv1.PodDisruptionBudgetSpec{Selector: selector},
Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: allowed, ObservedGeneration: 1},
}
}
func withPolicy(policy policyv1.UnhealthyPodEvictionPolicyType) func(*policyv1.PodDisruptionBudget) {
return func(p *policyv1.PodDisruptionBudget) { p.Spec.UnhealthyPodEvictionPolicy = &policy }
}
func withHealth(current, desired int32) func(*policyv1.PodDisruptionBudget) {
return func(p *policyv1.PodDisruptionBudget) {
p.Status.CurrentHealthy, p.Status.DesiredHealthy = current, desired
}
}
func stale(p *policyv1.PodDisruptionBudget) { p.Generation = 2 }
func pdbWith(base policyv1.PodDisruptionBudget, mutate ...func(*policyv1.PodDisruptionBudget)) policyv1.PodDisruptionBudget {
for _, m := range mutate {
m(&base)
}
return base
}
func TestClassifyPodForDrain(t *testing.T) {
webSelector := &metav1.LabelSelector{MatchLabels: map[string]string{"app": "web"}}
defaults := DrainOptions{IgnoreDaemonSets: true}
tests := []struct {
name string
pod corev1.Pod
opts DrainOptions
pdbs []policyv1.PodDisruptionBudget
want DrainOutcome
}{
{name: "managed running pod is evicted", pod: drainTestPod("a"), opts: defaults, want: DrainOutcomeEvict},
{name: "succeeded pod is skipped", pod: drainTestPod("a", withPhase(corev1.PodSucceeded)), opts: defaults, want: DrainOutcomeSkip},
{name: "failed pod is skipped", pod: drainTestPod("a", withPhase(corev1.PodFailed)), opts: defaults, want: DrainOutcomeSkip},
{name: "mirror pod is skipped even with force", pod: drainTestPod("a", withMirror), opts: DrainOptions{IgnoreDaemonSets: true, Force: true}, want: DrainOutcomeSkip},
{name: "DaemonSet pod is skipped when ignored", pod: drainTestPod("a", withOwner("DaemonSet", true)), opts: defaults, want: DrainOutcomeSkip},
{name: "DaemonSet pod is never evicted, even when not ignored", pod: drainTestPod("a", withOwner("DaemonSet", true)), opts: DrainOptions{}, want: DrainOutcomeSkip},
{name: "terminating pod is evicted without consulting PDBs", pod: drainTestPod("a", withDeletionTimestamp), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "web", webSelector, 0)}, want: DrainOutcomeEvict},
{name: "terminating DaemonSet pod is still skipped", pod: drainTestPod("a", withOwner("DaemonSet", true), withDeletionTimestamp), opts: defaults, want: DrainOutcomeSkip},
{name: "non-controller DaemonSet reference does not make a DaemonSet pod", pod: drainTestPod("a", withOwner("DaemonSet", false)), opts: defaults, want: DrainOutcomeSkip}, // unmanaged, no force
{name: "unmanaged pod is skipped without force", pod: drainTestPod("a", withoutOwner), opts: defaults, want: DrainOutcomeSkip},
{name: "unmanaged pod is evicted with force", pod: drainTestPod("a", withoutOwner), opts: DrainOptions{IgnoreDaemonSets: true, Force: true}, want: DrainOutcomeEvict},
{name: "owner reference without controller flag counts as unmanaged", pod: drainTestPod("a", withOwner("ReplicaSet", false)), opts: defaults, want: DrainOutcomeSkip},
{name: "emptyDir pod is skipped unless allowed", pod: drainTestPod("a", withEmptyDir), opts: defaults, want: DrainOutcomeSkip},
{name: "emptyDir pod is evicted when allowed", pod: drainTestPod("a", withEmptyDir), opts: DrainOptions{IgnoreDaemonSets: true, DeleteEmptyDirData: true}, want: DrainOutcomeEvict},
{name: "exhausted PDB may block", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "web", webSelector, 0)}, want: DrainOutcomeMayBlock},
{name: "PDB with budget left does not block", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "web", webSelector, 1)}, want: DrainOutcomeEvict},
{name: "PDB in another namespace is ignored", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("other", "web", webSelector, 0)}, want: DrainOutcomeEvict},
{name: "PDB with non-matching selector is ignored", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "db", &metav1.LabelSelector{MatchLabels: map[string]string{"app": "db"}}, 0)}, want: DrainOutcomeEvict},
{name: "PDB with nil selector matches nothing", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "none", nil, 0)}, want: DrainOutcomeEvict},
{name: "PDB with empty selector matches every pod in the namespace", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "all", &metav1.LabelSelector{}, 0)}, want: DrainOutcomeMayBlock},
{name: "skip decisions win over PDB", pod: drainTestPod("a", withEmptyDir), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "web", webSelector, 0)}, want: DrainOutcomeSkip},
{name: "pending pod bypasses PDBs", pod: drainTestPod("a", withPhase(corev1.PodPending)), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "web", webSelector, 0)}, want: DrainOutcomeEvict},
{name: "two PDBs covering the pod block the eviction even with budget left", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdb("shop", "web", webSelector, 1), pdb("shop", "all", &metav1.LabelSelector{}, 1)}, want: DrainOutcomeMayBlock},
{name: "unreconciled PDB status blocks", pod: drainTestPod("a"), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdbWith(pdb("shop", "web", webSelector, 1), stale)}, want: DrainOutcomeMayBlock},
{name: "unready pod under AlwaysAllow is evicted despite an exhausted budget", pod: drainTestPod("a", withReady(false)), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdbWith(pdb("shop", "web", webSelector, 0), withPolicy(policyv1.AlwaysAllow))}, want: DrainOutcomeEvict},
{name: "unready pod under IfHealthyBudget with a healthy budget is evicted", pod: drainTestPod("a", withReady(false)), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdbWith(pdb("shop", "web", webSelector, 0), withHealth(2, 2))}, want: DrainOutcomeEvict},
{name: "unready pod under IfHealthyBudget with an unhealthy budget may block", pod: drainTestPod("a", withReady(false)), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdbWith(pdb("shop", "web", webSelector, 0), withHealth(1, 2))}, want: DrainOutcomeMayBlock},
{name: "ready pod under AlwaysAllow with an exhausted budget still may block", pod: drainTestPod("a", withReady(true)), opts: defaults, pdbs: []policyv1.PodDisruptionBudget{pdbWith(pdb("shop", "web", webSelector, 0), withPolicy(policyv1.AlwaysAllow))}, want: DrainOutcomeMayBlock},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := ClassifyPodForDrain(tt.pod, tt.opts, tt.pdbs)
if got.Outcome != tt.want {
t.Fatalf("outcome = %q (%s), want %q", got.Outcome, got.Reason, tt.want)
}
if got.Reason == "" {
t.Fatalf("every decision must carry an operator-facing reason")
}
if got.Namespace != tt.pod.Namespace || got.Name != tt.pod.Name {
t.Fatalf("decision must identify the pod: %+v", got)
}
if tt.want == DrainOutcomeMayBlock && got.PDB == "" {
t.Fatalf("may-block decision must name the PDB: %+v", got)
}
if got.EmptyDir != hasLocalStorage(tt.pod) {
t.Fatalf("emptyDir flag must reflect the pod's volumes regardless of outcome: %+v", got)
}
if got.Terminating != (tt.pod.DeletionTimestamp != nil) {
t.Fatalf("terminating flag must reflect the pod's deletion timestamp: %+v", got)
}
})
}
}
func TestPlanNodeDrainIsReadOnlyAndCountsOutcomes(t *testing.T) {
client := fake.NewSimpleClientset(
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker-1"}},
func() *corev1.Pod { p := drainTestPod("web-1"); return &p }(),
func() *corev1.Pod { p := drainTestPod("web-2", withEmptyDir); return &p }(),
func() *corev1.Pod { p := drainTestPod("agent", withOwner("DaemonSet", true)); return &p }(),
func() *corev1.Pod { p := drainTestPod("elsewhere"); p.Spec.NodeName = "worker-2"; return &p }(),
&policyv1.PodDisruptionBudget{
ObjectMeta: metav1.ObjectMeta{Name: "web", Namespace: "shop"},
Spec: policyv1.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "web"}}},
Status: policyv1.PodDisruptionBudgetStatus{DisruptionsAllowed: 0},
},
)
// the fake clientset does not implement field selectors; filter like the API server would
client.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
list := &corev1.PodList{}
all, _ := client.Tracker().List(schema.GroupVersionResource{Version: "v1", Resource: "pods"}, schema.GroupVersionKind{Version: "v1", Kind: "Pod"}, "")
for _, p := range all.(*corev1.PodList).Items {
if p.Spec.NodeName == "worker-1" {
list.Items = append(list.Items, p)
}
}
return true, list, nil
})
plan, err := PlanNodeDrain(context.Background(), client, "worker-1", DrainOptions{IgnoreDaemonSets: true})
if err != nil {
t.Fatalf("PlanNodeDrain: %v", err)
}
for _, a := range client.Actions() {
switch a.GetVerb() {
case "get", "list":
default:
t.Fatalf("plan must not mutate the cluster, saw %s %s", a.GetVerb(), a.GetResource().Resource)
}
}
if !plan.Estimate || plan.Node != "worker-1" || plan.GeneratedAt.IsZero() {
t.Fatalf("plan header is incomplete: %+v", plan)
}
if !plan.PDBsEvaluated {
t.Fatalf("PDBs were listable and must be reported as evaluated")
}
for _, d := range plan.Pods {
if (d.Outcome != DrainOutcomeSkip) != d.PDBChecked {
t.Fatalf("pdbChecked must be true exactly for decisions that reached the budget check: %+v", d)
}
}
if got := plan.Summary; got.Evict != 0 || got.Skip != 2 || got.MayBlock != 1 {
t.Fatalf("summary = %+v, want evict=0 skip=2 mayBlock=1", got)
}
if len(plan.Pods) != 3 {
t.Fatalf("plan must list only pods on the node, got %d", len(plan.Pods))
}
}
func TestPlanNodeDrainReportsUnevaluatedPDBs(t *testing.T) {
client := fake.NewSimpleClientset(
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker-1"}},
func() *corev1.Pod { p := drainTestPod("web-1"); return &p }(),
)
client.PrependReactor("list", "poddisruptionbudgets", func(k8stesting.Action) (bool, runtime.Object, error) {
return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "policy", Resource: "poddisruptionbudgets"}, "", nil)
})
plan, err := PlanNodeDrain(context.Background(), client, "worker-1", DrainOptions{IgnoreDaemonSets: true})
if err != nil {
t.Fatalf("a forbidden PDB list must not fail the plan: %v", err)
}
if plan.PDBsEvaluated || plan.PDBError == "" {
t.Fatalf("plan must say PDBs were not evaluated instead of reporting zero: %+v", plan)
}
if plan.Summary.MayBlock != 0 || plan.Pods[0].Outcome != DrainOutcomeEvict || plan.Pods[0].PDBChecked {
t.Fatalf("pod must be classified without PDB knowledge and flagged: %+v", plan.Pods[0])
}
}
func TestPlanNodeDrainUnknownNode(t *testing.T) {
client := fake.NewSimpleClientset()
_, err := PlanNodeDrain(context.Background(), client, "ghost", DrainOptions{})
if !apierrors.IsNotFound(err) {
t.Fatalf("want NotFound for an unknown node, got %v", err)
}
}
func TestDrainNodeReportsSkippedPodsWithoutPDBClaims(t *testing.T) {
client := fake.NewSimpleClientset(
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker-1"}},
func() *corev1.Pod { p := drainTestPod("bare", withoutOwner); return &p }(),
)
res, err := DrainNode(context.Background(), client, "worker-1", DrainOptions{IgnoreDaemonSets: true, Timeout: time.Second})
if err != nil {
t.Fatalf("DrainNode: %v", err)
}
if len(res.SkippedPods) != 1 || res.SkippedPods[0].Outcome != DrainOutcomeSkip || res.SkippedPods[0].PDBChecked {
t.Fatalf("skipped pods must be reported with pdbChecked=false (PDBs are not consulted during execution): %+v", res.SkippedPods)
}
}
func TestPlanNodeDrainListsPDBsOnlyWhereADecisionNeedsThem(t *testing.T) {
client := fake.NewSimpleClientset(
&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker-1"}},
func() *corev1.Pod {
p := drainTestPod("agent", withOwner("DaemonSet", true))
p.Namespace = "infra"
return &p
}(),
func() *corev1.Pod { p := drainTestPod("web-1"); return &p }(),
)
client.PrependReactor("list", "poddisruptionbudgets", func(action k8stesting.Action) (bool, runtime.Object, error) {
if action.GetNamespace() == "infra" {
t.Fatalf("PDBs must not be listed for a namespace whose pods are all skipped")
}
return false, nil, nil
})
plan, err := PlanNodeDrain(context.Background(), client, "worker-1", DrainOptions{IgnoreDaemonSets: true})
if err != nil {
t.Fatalf("PlanNodeDrain: %v", err)
}
if !plan.PDBsEvaluated {
t.Fatalf("plan must count as evaluated when every needed namespace was listed: %+v", plan)
}
for _, d := range plan.Pods {
wantChecked := d.Name == "web-1"
if d.PDBChecked != wantChecked {
t.Fatalf("pdbChecked for %s = %v, want %v", d.Name, d.PDBChecked, wantChecked)
}
}
}
// Six PDB-blocked pods sleep in their 429 retry loops; with only five concurrency
// slots, a pod with no budget at all must still get its eviction attempted promptly.
// The free pod's reactor is gated on five distinct blocked pods having been attempted
// first, so this test cannot pass by racing ahead of the blocked retry loops.
func TestDrainNodeEvictsFreePodWhilePDBBlockedPodsRetry(t *testing.T) {
objs := []runtime.Object{&corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker-1"}}}
for i := 0; i < 6; i++ {
p := drainTestPod(fmt.Sprintf("blocked-%d", i))
objs = append(objs, &p)
}
free := drainTestPod("free")
objs = append(objs, &free)
client := fake.NewSimpleClientset(objs...)
// Reactors run under the fake clientset's lock, so the gate must never block:
// the free pod is refused with the same 429 until five distinct blocked pods
// have been attempted, then its next retry is allowed through.
var mu sync.Mutex
blockedSeen := map[string]bool{}
gateOpen := false
var freeEvictedAfter time.Duration
start := time.Now()
pdbRefused := apierrors.NewTooManyRequests("Cannot evict pod as it would violate the pod's disruption budget.", 0)
client.PrependReactor("create", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) {
if action.GetSubresource() != "eviction" {
return false, nil, nil
}
ev, ok := action.(k8stesting.CreateAction).GetObject().(*policyv1.Eviction)
if !ok {
t.Errorf("eviction create carried unexpected object")
return true, nil, nil
}
mu.Lock()
defer mu.Unlock()
if ev.Name == "free" {
if !gateOpen {
return true, nil, pdbRefused
}
if freeEvictedAfter == 0 {
freeEvictedAfter = time.Since(start)
}
return true, nil, nil
}
blockedSeen[ev.Name] = true
if len(blockedSeen) >= 5 {
gateOpen = true
}
return true, nil, pdbRefused
})
const timeout = 3 * time.Second
res, err := DrainNode(context.Background(), client, "worker-1", DrainOptions{IgnoreDaemonSets: true, Timeout: timeout})
if err != nil {
t.Fatalf("DrainNode: %v", err)
}
if len(res.EvictedPods) != 1 || res.EvictedPods[0] != "shop/free" {
t.Fatalf("free pod must be evicted, got evicted=%v errors=%v", res.EvictedPods, res.Errors)
}
// Half the deadline separates "evicted while the blocked pods retried" from
// "evicted only after the deadline freed their slots".
if freeEvictedAfter <= 0 || freeEvictedAfter > timeout/2 {
t.Fatalf("free pod was evicted after %v; slots are being held across PDB backoff sleeps", freeEvictedAfter)
}
if len(res.Errors) != 6 {
t.Fatalf("all six blocked pods must report errors, got %v", res.Errors)
}
for _, e := range res.Errors {
if !strings.Contains(e, "timed out waiting for PDB to allow eviction") {
t.Fatalf("blocked pod error must name the PDB timeout, got %q", e)
}
}
}