pkg/audit/storage_test.go
Rama main · réplica de lectura del repositorio original.
package audit
import (
"slices"
"strings"
"testing"
"time"
appsv1 "k8s.io/api/apps/v1"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
storagev1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
func storageInput() *CheckInput {
return &CheckInput{
PVCConsumerNamespaces: []string{"app"},
PVDeletionEventsComplete: true,
Pods: []*corev1.Pod{}, Deployments: []*appsv1.Deployment{}, ReplicaSets: []*appsv1.ReplicaSet{},
StatefulSets: []*appsv1.StatefulSet{}, DaemonSets: []*appsv1.DaemonSet{}, Jobs: []*batchv1.Job{}, CronJobs: []*batchv1.CronJob{},
PersistentVolumeClaims: []*corev1.PersistentVolumeClaim{{
ObjectMeta: metav1.ObjectMeta{Name: "data-db-0", Namespace: "app", CreationTimestamp: metav1.NewTime(time.Now().Add(-48 * time.Hour))},
Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("local"), VolumeName: "disk", Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("8Gi")}}},
Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound},
}},
PersistentVolumes: []*corev1.PersistentVolume{{ObjectMeta: metav1.ObjectMeta{Name: "disk", UID: "disk-uid", CreationTimestamp: metav1.NewTime(time.Now().Add(-72 * time.Hour))}, Spec: corev1.PersistentVolumeSpec{Capacity: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("8Gi")}, StorageClassName: "local", PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, ClaimRef: &corev1.ObjectReference{Namespace: "app", Name: "data-db-0"}}}},
StorageClasses: []*storagev1.StorageClass{{ObjectMeta: metav1.ObjectMeta{Name: "local"}, VolumeBindingMode: ptr(storagev1.VolumeBindingWaitForFirstConsumer)}},
Events: []*corev1.Event{},
}
}
func pvcPodSpec(name string) corev1.PodSpec {
return corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}}}}}
}
func TestPVCNoConsumer(t *testing.T) {
for _, tc := range []struct {
name string
change func(*CheckInput)
want bool
}{
{"no consumer", func(*CheckInput) {}, true},
{"Pod", func(i *CheckInput) {
i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")})
}, false},
{"terminal Pod", func(i *CheckInput) {
i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0"), Status: corev1.PodStatus{Phase: corev1.PodSucceeded}})
}, false},
{"generic ephemeral Pod", func(i *CheckInput) {
i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "data-db"}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "0", VolumeSource: corev1.VolumeSource{Ephemeral: &corev1.EphemeralVolumeSource{}}}}}})
}, false},
{"StatefulSet direct claim", func(i *CheckInput) {
i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}})
}, false},
{"Deployment at zero", func(i *CheckInput) {
i.Deployments = append(i.Deployments, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.DeploymentSpec{Replicas: ptr(int32(0)), Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}})
}, false},
{"ReplicaSet", func(i *CheckInput) {
i.ReplicaSets = append(i.ReplicaSets, &appsv1.ReplicaSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.ReplicaSetSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}})
}, false},
{"DaemonSet", func(i *CheckInput) {
i.DaemonSets = append(i.DaemonSets, &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}})
}, false},
{"StatefulSet generated at zero", func(i *CheckInput) {
i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr(int32(0)), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}})
}, false},
{"StatefulSet retained ordinal", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Name = "data-db-12"
i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr(int32(0)), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}})
}, false},
{"not a generated ordinal", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Name = "data-db-backup"
i.StatefulSets = append(i.StatefulSets, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}})
}, true},
{"Job including complete", func(i *CheckInput) {
i.Jobs = append(i.Jobs, &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}, Status: batchv1.JobStatus{Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}}})
}, false},
{"CronJob", func(i *CheckInput) {
i.CronJobs = append(i.CronJobs, &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}}})
}, false},
{"other namespace", func(i *CheckInput) {
i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "other"}, Spec: pvcPodSpec("data-db-0")})
}, true},
{"unreadable Pods", func(i *CheckInput) { i.Pods = nil }, false},
{"unknown coverage", func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, false},
{"CNPG controller owner", func(i *CheckInput) {
i.PersistentVolumeClaims[0].OwnerReferences = []metav1.OwnerReference{{APIVersion: "postgresql.cnpg.io/v1", Kind: "Cluster", Name: "hibernated", Controller: ptr(true)}}
}, false},
{"non-controller owner", func(i *CheckInput) {
i.PersistentVolumeClaims[0].OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "metadata"}}
}, true},
{"unreadable PVs", func(i *CheckInput) { i.PersistentVolumes = nil }, true},
{"PVs outside scope", func(i *CheckInput) { i.PersistentVolumes = []*corev1.PersistentVolume{} }, true},
{"Pending", func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending }, false},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
tc.change(i)
r := RunChecks(i)
if got := findingNames(r.Findings, "pvcNoConsumer")[i.PersistentVolumeClaims[0].Name]; got != tc.want {
t.Fatalf("finding=%v want=%v: %+v", got, tc.want, r.Findings)
}
if tc.want {
for _, f := range r.Findings {
if f.CheckID == "pvcNoConsumer" {
for _, s := range []string{"Requested 8Gi", `storage class "local"`, "Bound PV disk", "PVC age", "reclaim policy"} {
if !strings.Contains(f.Message, s) {
t.Errorf("missing %q in %q", s, f.Message)
}
}
}
}
}
if tc.name == "unreadable PVs" || tc.name == "PVs outside scope" {
// The claim is still evaluated; only releasedPV is skipped, and
// only when the inventory is unavailable rather than out of scope.
if !strings.Contains(r.Findings[0].Message, "reclaim policy not visible") || r.CheckCounts["pvcNoConsumer"].Evaluated != 1 {
t.Fatalf("claim finding lost its retention caveat: %+v", r)
}
if slices.Contains(r.MissingInputs, "persistentvolumes") != (tc.name == "unreadable PVs") {
t.Fatalf("PV missing input does not track skipped releasedPV: %v", r.MissingInputs)
}
}
if tc.name == "unknown coverage" && (r.CheckCounts["pvcNoConsumer"].Evaluated != 0 || !slices.Contains(r.MissingInputs, "pvc-consumers")) {
t.Fatalf("unknown counted as passing: %+v", r)
}
})
}
}
func TestPVCUnusedCondition(t *testing.T) {
now := time.Date(2026, time.October, 12, 12, 0, 0, 0, time.UTC)
recent := metav1.NewTime(now.Add(-2 * time.Hour))
old := metav1.NewTime(now.Add(-10 * 24 * time.Hour))
evidence := "Kubernetes reports it unused by any Pod since 2026-10-02T12:00:00Z (10d ago)."
for _, tc := range []struct {
name string
status corev1.ConditionStatus
since metav1.Time
change func(*CheckInput)
evaluated, passed int
finding bool
evidence string
missingConsumers bool
}{
{name: "False without observed consumer", status: corev1.ConditionFalse, since: recent, evaluated: 1, passed: 1},
{name: "False with unreadable Pods", status: corev1.ConditionFalse, change: func(i *CheckInput) { i.Pods = nil }, evaluated: 1, passed: 1},
{name: "False with unknown coverage", status: corev1.ConditionFalse, change: func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, evaluated: 1, passed: 1},
{name: "True recently unused", status: corev1.ConditionTrue, since: recent},
{name: "True at age floor", status: corev1.ConditionTrue, since: metav1.NewTime(now.Add(-24 * time.Hour))},
{name: "True just past age floor", status: corev1.ConditionTrue, since: metav1.NewTime(now.Add(-24*time.Hour - time.Second)), evaluated: 1, finding: true, evidence: "Kubernetes reports it unused by any Pod since 2026-10-11T11:59:59Z (1d ago)."},
{name: "True future transition", status: corev1.ConditionTrue, since: metav1.NewTime(now.Add(time.Hour))},
{name: "True with unknown coverage during grace", status: corev1.ConditionTrue, since: recent, change: func(i *CheckInput) { i.PVCConsumerNamespaces = nil }},
{name: "True unused for ten days", status: corev1.ConditionTrue, since: old, evaluated: 1, finding: true, evidence: evidence},
{name: "True uses transition instead of creation", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) {
i.PersistentVolumeClaims[0].CreationTimestamp = recent
}, evaluated: 1, finding: true, evidence: evidence},
{name: "True zero transition", status: corev1.ConditionTrue, evaluated: 1, finding: true},
{name: "True zero transition on young claim", status: corev1.ConditionTrue, change: func(i *CheckInput) {
i.PersistentVolumeClaims[0].CreationTimestamp = recent
}},
{name: "True with zero replica StatefulSet template", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) {
i.StatefulSets = []*appsv1.StatefulSet{{ObjectMeta: metav1.ObjectMeta{Namespace: "app", Name: "db"}, Spec: appsv1.StatefulSetSpec{Replicas: ptr(int32(0)), VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "data"}}}}}}
}, evaluated: 1, passed: 1},
{name: "True with observed Pod", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) {
i.Pods = []*corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")}}
}, evaluated: 1, passed: 1},
{name: "True with unknown coverage", status: corev1.ConditionTrue, since: old, change: func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, missingConsumers: true},
{name: "absent", evaluated: 1, finding: true},
{name: "Unknown", status: corev1.ConditionUnknown, since: recent, evaluated: 1, finding: true},
{name: "unrelated False condition", change: func(i *CheckInput) {
i.PersistentVolumeClaims[0].Status.Conditions = []corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionFalse}}
}, evaluated: 1, finding: true},
{name: "unrelated condition before Unused", status: corev1.ConditionFalse, change: func(i *CheckInput) {
pvc := i.PersistentVolumeClaims[0]
pvc.Status.Conditions = append([]corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimResizing, Status: corev1.ConditionTrue}}, pvc.Status.Conditions...)
}, evaluated: 1, passed: 1},
{name: "Pending ignores False", status: corev1.ConditionFalse, change: func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending }, evaluated: 1, finding: true},
{name: "Pending ignores recent True", status: corev1.ConditionTrue, since: recent, change: func(i *CheckInput) { i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending }, evaluated: 1, finding: true},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
pvc := i.PersistentVolumeClaims[0]
pvc.CreationTimestamp = metav1.NewTime(now.Add(-30 * 24 * time.Hour))
if tc.status != "" {
pvc.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimUnused, Status: tc.status, LastTransitionTime: tc.since}}
}
if tc.change != nil {
tc.change(i)
}
tr := newEvalTracker()
r := buildResults(checkStorage(tr, i, now), tr, nil)
checkID := "pvcNoConsumer"
if pvc.Status.Phase == corev1.ClaimPending {
checkID = "pvcLongPending"
}
if got := findingNames(r.Findings, checkID)[pvc.Name]; got != tc.finding {
t.Fatalf("finding=%v want=%v: %+v", got, tc.finding, r.Findings)
}
if count := r.CheckCounts[checkID]; count.Evaluated != tc.evaluated || count.Passed != tc.passed {
t.Fatalf("counts=%+v want evaluated=%d passed=%d", count, tc.evaluated, tc.passed)
}
if slices.Contains(r.MissingInputs, "pvc-consumers") != tc.missingConsumers {
t.Fatalf("missing consumer input=%v want=%v", r.MissingInputs, tc.missingConsumers)
}
for _, f := range r.Findings {
if f.CheckID != checkID {
continue
}
if tc.evidence != "" {
if !strings.Contains(f.Message, tc.evidence) {
t.Errorf("missing evidence %q in %q", tc.evidence, f.Message)
}
} else if strings.Contains(f.Message, "reports it unused") {
t.Errorf("unexpected Unused evidence in %q", f.Message)
}
for _, fact := range []string{"Requested 8Gi", `storage class "local"`, "PVC age"} {
if !strings.Contains(f.Message, fact) {
t.Errorf("missing fact %q in %q", fact, f.Message)
}
}
}
})
}
}
func TestPVCFindingMessages(t *testing.T) {
now := time.Date(2026, time.October, 12, 12, 0, 0, 0, time.UTC)
for _, tc := range []struct {
name string
phase corev1.PersistentVolumeClaimPhase
unused bool
checkID string
message string
}{
{"Bound", corev1.ClaimBound, false, "pvcNoConsumer", `Requested 8Gi; storage class "local"; PVC age 2d. Bound PV disk; reclaim policy Retain.`},
{"Bound with Unused condition", corev1.ClaimBound, true, "pvcNoConsumer", `Kubernetes reports it unused by any Pod since 2026-10-10T12:00:00Z (2d ago). Requested 8Gi; storage class "local"; PVC age 2d. Bound PV disk; reclaim policy Retain.`},
{"Pending", corev1.ClaimPending, false, "pvcLongPending", `Pending since creation, 2d ago. StorageClass uses WaitForFirstConsumer. Requested 8Gi; storage class "local"; PVC age 2d.`},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
pvc := i.PersistentVolumeClaims[0]
pvc.CreationTimestamp = metav1.NewTime(now.Add(-48 * time.Hour))
pvc.Status.Phase = tc.phase
if tc.unused {
pvc.Status.Conditions = []corev1.PersistentVolumeClaimCondition{{Type: corev1.PersistentVolumeClaimUnused, Status: corev1.ConditionTrue, LastTransitionTime: pvc.CreationTimestamp}}
}
fs := checkStorage(newEvalTracker(), i, now)
if len(fs) != 1 || fs[0].CheckID != tc.checkID {
t.Fatalf("expected one %s finding: %+v", tc.checkID, fs)
}
if fs[0].Message != tc.message {
t.Errorf("message=%q want=%q", fs[0].Message, tc.message)
}
for _, caveat := range []string{"No consumer observed among", "may still reference this claim"} {
if strings.Contains(fs[0].Message, caveat) {
t.Errorf("unexpected caveat %q in %q", caveat, fs[0].Message)
}
}
})
}
}
func TestPVCCloneConsumers(t *testing.T) {
now := time.Now()
for _, phase := range []corev1.PersistentVolumeClaimPhase{corev1.ClaimBound, corev1.ClaimPending} {
for _, tc := range []struct {
name string
change func(*corev1.PersistentVolumeClaim)
consumed bool
}{
{"Pending dataSource", func(*corev1.PersistentVolumeClaim) {}, true},
{"explicit core group", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource.APIGroup = ptr("") }, true},
{"owned destination", func(p *corev1.PersistentVolumeClaim) {
p.OwnerReferences = []metav1.OwnerReference{{Kind: "Job", Name: "clone", Controller: ptr(true)}}
p.CreationTimestamp = metav1.NewTime(now.Add(-48 * time.Hour))
}, true},
{"deleting destination", func(p *corev1.PersistentVolumeClaim) {
p.DeletionTimestamp = ptr(metav1.NewTime(now))
p.CreationTimestamp = metav1.NewTime(now.Add(-48 * time.Hour))
}, true},
{"Bound destination", func(p *corev1.PersistentVolumeClaim) { p.Status.Phase = corev1.ClaimBound }, false},
{"dataSourceRef", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0"}
}, true},
{"dataSourceRef explicit core and same namespace", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{APIGroup: ptr(""), Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("app")}
}, true},
{"dataSourceRef empty namespace", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("")}
}, true},
{"dataSourceRef targets another namespace", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("other")}
}, false},
{"destination in other namespace", func(p *corev1.PersistentVolumeClaim) { p.Namespace = "other" }, false},
{"cross namespace destination referencing source", func(p *corev1.PersistentVolumeClaim) {
p.Namespace = "other"
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0", Namespace: ptr("app")}
}, true},
{"other source name", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource.Name = "other" }, false},
{"non-core PVC", func(p *corev1.PersistentVolumeClaim) { p.Spec.DataSource.APIGroup = ptr("storage.example.com") }, false},
{"VolumeSnapshot", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = &corev1.TypedLocalObjectReference{APIGroup: ptr("snapshot.storage.k8s.io"), Kind: "VolumeSnapshot", Name: "data-db-0"}
}, false},
{"dataSourceRef VolumeSnapshot", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{APIGroup: ptr("snapshot.storage.k8s.io"), Kind: "VolumeSnapshot", Name: "data-db-0"}
}, false},
{"dataSourceRef non-core PVC", func(p *corev1.PersistentVolumeClaim) {
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{APIGroup: ptr("storage.example.com"), Kind: "PersistentVolumeClaim", Name: "data-db-0"}
}, false},
{"Bound dataSourceRef", func(p *corev1.PersistentVolumeClaim) {
p.Status.Phase = corev1.ClaimBound
p.Spec.DataSource = nil
p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0"}
}, false},
} {
t.Run(string(phase)+"/"+tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims[0].Status.Phase = phase
clone := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{Name: "clone", Namespace: "app", CreationTimestamp: metav1.NewTime(now.Add(-time.Minute))},
Spec: corev1.PersistentVolumeClaimSpec{DataSource: &corev1.TypedLocalObjectReference{Kind: "PersistentVolumeClaim", Name: "data-db-0"}},
Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending},
}
tc.change(clone)
i.PersistentVolumeClaims = append(i.PersistentVolumeClaims, clone)
tr := newEvalTracker()
r := buildResults(checkStorage(tr, i, now), tr, nil)
checkID := "pvcNoConsumer"
if phase == corev1.ClaimPending {
checkID = "pvcLongPending"
}
if findingNames(r.Findings, checkID)["data-db-0"] == tc.consumed {
t.Fatalf("source consumed=%v: %+v", tc.consumed, r.Findings)
}
wantPassed := 0
if tc.consumed {
wantPassed = 1
}
if count := r.CheckCounts[checkID]; count.Evaluated != 1 || count.Passed != wantPassed || len(r.MissingInputs) != 0 {
t.Fatalf("clone consumer counts: %+v missing=%v", count, r.MissingInputs)
}
})
}
}
}
func TestLongPendingPVC(t *testing.T) {
for _, tc := range []struct {
name string
age time.Duration
consumer, immediate, unknownClass, unknownConsumers bool
want bool
}{
{"WFFC no consumer", 48 * time.Hour, false, false, false, false, true},
{"WFFC with consumer", 48 * time.Hour, true, false, false, false, false},
{"WFFC with Pod", 48 * time.Hour, true, false, false, false, false},
{"Immediate with consumer", 48 * time.Hour, true, true, false, false, false},
{"young", time.Hour, false, false, false, false, false},
{"exact threshold", 24 * time.Hour, false, false, false, false, false},
{"unknown class with consumer", 48 * time.Hour, true, false, true, false, false},
{"unknown class no consumer", 48 * time.Hour, false, false, true, false, true},
{"unknown consumers", 48 * time.Hour, false, false, false, true, false},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
now := time.Now()
i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending
i.PersistentVolumeClaims[0].CreationTimestamp = metav1.NewTime(now.Add(-tc.age))
if tc.consumer {
if tc.name == "WFFC with Pod" {
i.Pods = append(i.Pods, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")})
} else {
i.CronJobs = append(i.CronJobs, &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: pvcPodSpec("data-db-0")}}}}})
}
}
if tc.immediate {
i.StorageClasses[0].VolumeBindingMode = ptr(storagev1.VolumeBindingImmediate)
}
if tc.unknownClass {
i.StorageClasses = nil
}
if tc.unknownConsumers {
i.PVCConsumerNamespaces = nil
}
tr := newEvalTracker()
fs := checkStorage(tr, i, now)
if got := findingNames(fs, "pvcLongPending")["data-db-0"]; got != tc.want {
t.Fatalf("finding=%v want=%v: %+v", got, tc.want, fs)
}
if tc.want {
for _, fact := range []string{"Requested 8Gi", `storage class "local"`, "PVC age 2d"} {
if !strings.Contains(fs[0].Message, fact) {
t.Errorf("missing fact %q in %q", fact, fs[0].Message)
}
}
bindingEvidence := "StorageClass uses WaitForFirstConsumer."
if tc.unknownClass {
bindingEvidence = "StorageClass binding mode not visible."
}
if !strings.Contains(fs[0].Message, bindingEvidence) {
t.Errorf("missing binding evidence %q in %q", bindingEvidence, fs[0].Message)
}
if !strings.Contains(fs[0].Message, "Pending since creation, 2d ago") || strings.Contains(fs[0].Message, "bound PV") || strings.Contains(fs[0].Message, "reclaim policy") || strings.Contains(fs[0].Message, "snapshot does not establish") {
t.Fatalf("inaccurate Pending detail: %+v", fs[0])
}
}
})
}
}
func TestPVCStorageEligibility(t *testing.T) {
now := time.Now()
for _, phase := range []corev1.PersistentVolumeClaimPhase{corev1.ClaimBound, corev1.ClaimPending} {
for _, age := range []time.Duration{time.Hour, 24 * time.Hour, 25 * time.Hour} {
t.Run(string(phase)+"/"+age.String(), func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims[0].Status.Phase = phase
i.PersistentVolumeClaims[0].CreationTimestamp = metav1.NewTime(now.Add(-age))
fs := checkStorage(newEvalTracker(), i, now)
if (len(fs) > 0) != (age > 24*time.Hour) {
t.Fatalf("age eligibility: %+v", fs)
}
i.PersistentVolumeClaims[0].OwnerReferences = []metav1.OwnerReference{{APIVersion: "postgresql.cnpg.io/v1", Kind: "Cluster", Name: "hibernated", Controller: ptr(true)}}
tr := newEvalTracker()
if fs := checkStorage(tr, i, now); len(fs) != 0 || tr.counts["pvcNoConsumer"]["app"] != 0 || tr.counts["pvcLongPending"]["app"] != 0 {
t.Fatalf("controller-owned claim evaluated: %+v", fs)
}
})
}
}
}
// Missing inputs name only gaps that left a subject unevaluated; gaps that
// leave a finding in place are caveats in its message.
func TestStorageMissingInputsOnlyForSkippedSubjects(t *testing.T) {
storageKeys := []string{"persistentvolumeclaims", "persistentvolumes", "storageclasses", "pvc-consumers", "pvc-binding-mode", "pv-deletion-events", "pv-release-time"}
for _, tc := range []struct {
name string
change func(*CheckInput)
missing []string
findings int
caveat string
}{
{"healthy", func(i *CheckInput) {
i.Pods = []*corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, Spec: pvcPodSpec("data-db-0")}}
i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound
}, nil, 0, ""},
{"complete evidence", func(*CheckInput) {}, nil, 2, ""},
{"unreadable claims", func(i *CheckInput) { i.PersistentVolumeClaims = nil }, []string{"persistentvolumeclaims"}, 1, ""},
{"unknown consumers", func(i *CheckInput) { i.PVCConsumerNamespaces = nil }, []string{"pvc-consumers"}, 1, ""},
{"unreadable PVs without claims", func(i *CheckInput) {
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
i.PersistentVolumes = nil
}, []string{"persistentvolumes"}, 0, ""},
{"PVs outside scope", func(i *CheckInput) { i.PersistentVolumes = []*corev1.PersistentVolume{} }, nil, 1, "reclaim policy not visible"},
{"unreadable classes", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending
i.StorageClasses = nil
}, nil, 2, "StorageClass binding mode not visible"},
{"unreadable events after grace", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.Events = nil
}, nil, 2, "Deletion warning events not fully visible"},
{"unreadable events in grace", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(time.Now().Add(-time.Minute)))
i.Events = nil
}, []string{"pv-deletion-events"}, 1, ""},
{"unknown release time", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = nil
}, []string{"pv-release-time"}, 1, ""},
{"Failed volume with unreadable events", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.Phase = corev1.VolumeFailed
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(time.Now().Add(-time.Minute)))
i.Events = nil
}, nil, 2, "Deletion warning events not fully visible"},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(time.Now().Add(-72 * time.Hour)))
tc.change(i)
r := RunChecks(i)
var got []string
for _, key := range storageKeys {
if slices.Contains(r.MissingInputs, key) {
got = append(got, key)
}
}
if !slices.Equal(got, tc.missing) {
t.Fatalf("missing inputs %v, want %v", got, tc.missing)
}
var storageFindings []Finding
for _, f := range r.Findings {
if f.CheckID == "pvcNoConsumer" || f.CheckID == "pvcLongPending" || f.CheckID == "releasedPV" {
storageFindings = append(storageFindings, f)
}
}
if len(storageFindings) != tc.findings {
t.Fatalf("findings %+v, want %d", storageFindings, tc.findings)
}
if tc.caveat != "" && !slices.ContainsFunc(storageFindings, func(f Finding) bool { return strings.Contains(f.Message, tc.caveat) }) {
t.Fatalf("evidence gap %q not stated in findings: %+v", tc.caveat, storageFindings)
}
if tc.caveat == "" {
for _, f := range storageFindings {
if strings.Contains(f.Message, "not visible") || strings.Contains(f.Message, "not fully visible") {
t.Fatalf("unexpected evidence caveat: %+v", f)
}
}
}
})
}
}
// Bound volumes pass; Available and Pending volumes have no claim to have
// outlived, so they are neither passes nor findings; Failed volumes never pass.
func TestReleasedPVCounts(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound
for name, phase := range map[string]corev1.PersistentVolumePhase{"available": corev1.VolumeAvailable, "pending": corev1.VolumePending, "failed": corev1.VolumeFailed} {
i.PersistentVolumes = append(i.PersistentVolumes, &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name)}, Spec: corev1.PersistentVolumeSpec{PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimDelete}, Status: corev1.PersistentVolumeStatus{Phase: phase}})
}
r := RunChecks(i)
if count := r.CheckCounts["releasedPV"]; count.Evaluated != 2 || count.Passed != 1 {
t.Fatalf("releasedPV counts: %+v", count)
}
if got := findingNames(r.Findings, "releasedPV"); len(got) != 1 || !got["failed"] {
t.Fatalf("releasedPV findings: %+v", r.Findings)
}
}
// Deleting volumes belong to stuckTerminating, like deleting claims: releasedPV
// neither passes a deleting Bound volume nor double-reports a deleting
// Released or Failed one.
func TestReleasedPVLeavesDeletingVolumesToStuckTerminating(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
deleted := metav1.NewTime(time.Now().Add(-2 * time.Hour))
released := metav1.NewTime(time.Now().Add(-48 * time.Hour))
i.PersistentVolumes = nil
for name, phase := range map[string]corev1.PersistentVolumePhase{"bound": corev1.VolumeBound, "released": corev1.VolumeReleased, "failed": corev1.VolumeFailed} {
i.PersistentVolumes = append(i.PersistentVolumes, &corev1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(name), DeletionTimestamp: &deleted, Finalizers: []string{"kubernetes.io/pv-protection"}},
Spec: corev1.PersistentVolumeSpec{PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain},
Status: corev1.PersistentVolumeStatus{Phase: phase, LastPhaseTransitionTime: &released},
})
}
if slices.ContainsFunc(i.PersistentVolumes, PVNeedsReclaimEvents) {
t.Fatal("deleting volumes must not request reclaim events")
}
r := RunChecks(i)
if count, ok := r.CheckCounts["releasedPV"]; ok && (count.Evaluated != 0 || count.Passed != 0) {
t.Fatalf("releasedPV counts: %+v", count)
}
if got := findingNames(r.Findings, "releasedPV"); len(got) != 0 {
t.Fatalf("releasedPV findings: %+v", got)
}
if got := findingNames(r.Findings, "stuckTerminating"); len(got) != 3 {
t.Fatalf("stuckTerminating findings: %+v", got)
}
if slices.Contains(r.MissingInputs, "persistentvolumes") || slices.Contains(r.MissingInputs, "pv-deletion-events") || slices.Contains(r.MissingInputs, "pv-release-time") {
t.Fatalf("missing inputs: %v", r.MissingInputs)
}
}
func TestFailedPV(t *testing.T) {
now := time.Now()
failedAt := now.Add(-5 * time.Minute)
for _, tc := range []struct {
name string
change func(*CheckInput)
text []string
absent []string
caveat bool
}{
{"Delete with warning", func(i *CheckInput) {
i.Events = []*corev1.Event{deleteFailure(failedAt, "host_path deleter only supports /tmp/.+")}
}, []string{"Failed PV: reclaiming the volume failed after its claim was deleted (reclaim policy Delete).", "Latest reclaim warning (5m ago): host_path deleter only supports /tmp/.+.", "Failed for 5m."}, []string{"Status message"}, false},
{"status message without events", func(*CheckInput) {}, []string{"Status message: status failure."}, []string{"Latest reclaim warning"}, false},
{"partial event coverage", func(i *CheckInput) { i.PVDeletionEventsComplete = false }, []string{"Status message: status failure."}, nil, true},
{"unreadable events", func(i *CheckInput) { i.Events = nil }, []string{"Failed PV"}, nil, true},
{"visible warning with partial coverage", func(i *CheckInput) {
i.PVDeletionEventsComplete = false
i.Events = []*corev1.Event{deleteFailure(failedAt, "visible failure")}
}, []string{"Latest reclaim warning (5m ago): visible failure."}, nil, true},
{"no message anywhere", func(i *CheckInput) {
i.PersistentVolumes[0].Status.Message = ""
i.PVDeletionEventsComplete = false
}, []string{"Failed PV", "Deletion warning events not fully visible.", "8Gi"}, []string{"Status message", "Latest reclaim warning"}, true},
{"warning before the Failed transition", func(i *CheckInput) {
i.Events = []*corev1.Event{deleteFailure(failedAt.Add(-time.Hour), "previous release failure")}
}, []string{"Status message: status failure."}, []string{"previous release failure"}, false},
{"Recycle", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRecycle
e := deleteFailure(failedAt, "recycler pod failed")
e.Reason = "VolumeFailedRecycle"
i.Events = []*corev1.Event{e}
}, []string{"(reclaim policy Recycle)", "Latest reclaim warning (5m ago): recycler pod failed."}, nil, false},
{"policy changed to Retain after the failure", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain
}, []string{"(reclaim policy Retain)", "Status message: status failure."}, nil, false},
{"transition unavailable", func(i *CheckInput) {
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = nil
}, []string{"; PV age "}, []string{"Failed for"}, false},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
pv := i.PersistentVolumes[0]
pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
pv.Status = corev1.PersistentVolumeStatus{Phase: corev1.VolumeFailed, Message: "status failure", LastPhaseTransitionTime: ptr(metav1.NewTime(failedAt))}
tc.change(i)
tr := newEvalTracker()
r := buildResults(checkStorage(tr, i, now), tr, nil)
// The phase establishes the failure, so incomplete events never
// leave the volume unevaluated.
if count := r.CheckCounts["releasedPV"]; count.Evaluated != 1 || count.Passed != 0 || len(r.MissingInputs) != 0 {
t.Fatalf("Failed PV counted as passed or skipped: %+v missing=%v", count, r.MissingInputs)
}
if len(r.Findings) != 1 || r.Findings[0].CheckID != "releasedPV" || r.Findings[0].Kind != "PersistentVolume" {
t.Fatalf("Failed PV not reported: %+v", r.Findings)
}
message := r.Findings[0].Message
for _, text := range append(tc.text, "Capacity 8Gi", `storage class "local"`, "former claim app/data-db-0") {
if !strings.Contains(message, text) {
t.Errorf("message lacks %q: %s", text, message)
}
}
for _, text := range tc.absent {
if strings.Contains(message, text) {
t.Errorf("message has %q: %s", text, message)
}
}
if strings.Contains(message, "Deletion warning events not fully visible") != tc.caveat {
t.Errorf("event coverage caveat want %v: %s", tc.caveat, message)
}
})
}
}
func TestReleasedPVIgnoresRecycleWarnings(t *testing.T) {
now := time.Now()
i := storageInput()
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute)))
e := deleteFailure(now, "recycler pod failed")
e.Reason = "VolumeFailedRecycle"
i.Events = []*corev1.Event{e}
if got := latestPVReclaimWarnings(i.PersistentVolumes, i.Events, now); len(got) != 0 {
t.Fatalf("recycle warning read as a deletion failure: %+v", got)
}
}
func TestReleasedPV(t *testing.T) {
now := time.Now()
for _, tc := range []struct {
name string
change func(*CheckInput)
want, text string
}{
{"Retain", func(*CheckInput) {}, "disk", "kept after its claim was deleted"},
{"Bound", func(i *CheckInput) { i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound }, "", ""},
{"Delete no event", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
}, "disk", "deletion has not completed"},
{"Delete recent event", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Minute), "latest failure")}
}, "disk", "deletion is failing: latest failure"},
{"Delete old event", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.Events = []*corev1.Event{deleteFailure(now.Add(-48*time.Hour), "stale failure")}
}, "disk", "Latest deletion warning (2d ago): stale failure"},
{"Delete old UID", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
e := deleteFailure(now, "other object")
e.InvolvedObject.UID = "old-uid"
i.Events = []*corev1.Event{e}
}, "disk", "deletion has not completed"},
{"Delete unreadable events", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.Events = nil
}, "disk", "deletion has not completed"},
{"Retain unreadable events", func(i *CheckInput) { i.Events = nil }, "disk", "kept after its claim was deleted"},
{"transition time", func(i *CheckInput) {
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Hour)))
}, "", ""},
{"Retain recent release", func(i *CheckInput) {
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-23 * time.Hour)))
}, "", ""},
{"Delete recent release", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute)))
}, "", ""},
{"Delete recent release with warning", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute)))
i.Events = []*corev1.Event{deleteFailure(now, "immediate failure")}
}, "disk", "deletion is failing: immediate failure"},
{"Delete exact threshold", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Hour)))
}, "", ""},
{"Delete old release", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-2 * time.Hour)))
}, "disk", "Released for 2h"},
{"latest event series", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
e := deleteFailure(now.Add(-48*time.Hour), "series failure")
e.Series = &corev1.EventSeries{LastObservedTime: metav1.NewMicroTime(now.Add(-time.Minute))}
i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Hour), "earlier failure"), e}
}, "disk", "series failure"},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-72 * time.Hour)))
tc.change(i)
tr := newEvalTracker()
fs := checkStorage(tr, i, now)
if got := findingNames(fs, "releasedPV")["disk"]; got != (tc.want != "") {
t.Fatalf("finding=%v want=%q: %+v", got, tc.want, fs)
}
for _, f := range fs {
if f.CheckID == "releasedPV" && (!strings.Contains(f.Message, tc.text) || !strings.Contains(f.Message, "8Gi") || !strings.Contains(f.Message, "app/data-db-0")) {
t.Errorf("missing evidence: %+v", f)
}
}
if tr.counts["releasedPV"][""] != 1 {
t.Fatal("readable PV not evaluated")
}
if tc.name == "Delete unreadable events" && (len(tr.missingInputs) != 0 || !strings.Contains(fs[len(fs)-1].Message, "Deletion warning events not fully visible")) {
t.Fatalf("flagged volume's evidence gap reported as a skip, or not stated: %v %+v", tr.missingInputs, fs)
}
if tc.name == "Delete no event" && !strings.Contains(fs[len(fs)-1].Message, "Released for 3d") {
t.Fatal("release duration not labelled")
}
})
}
}
func TestReleasedPVWithoutReleaseTime(t *testing.T) {
now := time.Now()
for _, transition := range []struct {
name string
at *metav1.Time
}{
{"nil", nil},
{"zero", &metav1.Time{}},
} {
for _, tc := range []struct {
name string
change func(*CheckInput)
findings int
text string
}{
{"Delete no warning", func(*CheckInput) {}, 0, ""},
{"Delete stale warning", func(i *CheckInput) {
i.Events = []*corev1.Event{deleteFailure(now.Add(-48*time.Hour), "stale failure")}
}, 0, ""},
{"Delete unreadable events", func(i *CheckInput) { i.Events = nil }, 0, ""},
{"Delete partial events", func(i *CheckInput) { i.PVDeletionEventsComplete = false }, 0, ""},
{"Delete recent warning", func(i *CheckInput) {
i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Minute), "current failure")}
}, 1, "deletion is failing: current failure."},
{"Delete recent warning with partial events", func(i *CheckInput) {
i.PVDeletionEventsComplete = false
i.Events = []*corev1.Event{deleteFailure(now.Add(-time.Minute), "current failure")}
}, 1, "deletion is failing: current failure."},
{"Retain", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain
}, 1, "release time unknown"},
{"Retain young PV", func(i *CheckInput) {
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain
i.PersistentVolumes[0].CreationTimestamp = metav1.NewTime(now.Add(-time.Minute))
}, 1, "release time unknown"},
} {
t.Run(transition.name+"/"+tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = transition.at
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
tc.change(i)
tr := newEvalTracker()
r := buildResults(checkStorage(tr, i, now), tr, nil)
if count := r.CheckCounts["releasedPV"]; count.Evaluated != tc.findings || count.Passed != 0 || len(r.Findings) != tc.findings {
t.Fatalf("unknown release counted as passing or age-based finding: %+v findings=%+v", count, r.Findings)
}
var wantMissing []string
if tc.findings == 0 {
wantMissing = []string{"pv-release-time"}
}
if !slices.Equal(r.MissingInputs, wantMissing) {
t.Fatalf("missing inputs %v, want %v", r.MissingInputs, wantMissing)
}
for _, f := range r.Findings {
if !strings.Contains(f.Message, tc.text) || !strings.Contains(f.Message, "PV age ") || strings.Contains(f.Message, "Released for") || strings.Contains(f.Message, "deletion has not completed") {
t.Fatalf("unknown release duration misstated: %+v", f)
}
if strings.Contains(f.Message, "Deletion warning events not fully visible") == i.PVDeletionEventsComplete {
t.Fatalf("event coverage caveat wrong: %+v", f)
}
}
})
}
}
}
func TestPVDeletionEventIdentityAndTimes(t *testing.T) {
now := time.Now()
for _, tc := range []struct {
name string
change func(*corev1.Event)
want bool
}{
{"current UID", func(*corev1.Event) {}, true},
{"old UID", func(e *corev1.Event) { e.InvolvedObject.UID = "previous" }, false},
{"wrong name", func(e *corev1.Event) { e.InvolvedObject.Name = "other" }, false},
{"namespaced ref", func(e *corev1.Event) { e.InvolvedObject.Namespace = "app" }, false},
{"CRD ref", func(e *corev1.Event) { e.InvolvedObject.APIVersion = "storage.example.com/v1" }, false},
{"wrong kind", func(e *corev1.Event) { e.InvolvedObject.Kind = "PersistentVolumeClaim" }, false},
{"Normal", func(e *corev1.Event) { e.Type = corev1.EventTypeNormal }, false},
{"wrong reason", func(e *corev1.Event) { e.Reason = "ProvisioningFailed" }, false},
{"future timestamp", func(e *corev1.Event) { e.LastTimestamp = metav1.NewTime(now.Add(time.Hour)) }, false},
{"event time", func(e *corev1.Event) { e.LastTimestamp = metav1.Time{}; e.EventTime = metav1.NewMicroTime(now) }, true},
{"creation time", func(e *corev1.Event) { e.LastTimestamp = metav1.Time{}; e.CreationTimestamp = metav1.NewTime(now) }, true},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
e := deleteFailure(now, "warning")
tc.change(e)
got := latestPVReclaimWarnings(i.PersistentVolumes, []*corev1.Event{e}, now)
if (got["disk-uid"].event != nil) != tc.want {
t.Fatalf("event identity/timestamp: %+v", got)
}
})
}
}
func deleteFailure(at time.Time, message string) *corev1.Event {
return &corev1.Event{ObjectMeta: metav1.ObjectMeta{Namespace: "app"}, InvolvedObject: corev1.ObjectReference{APIVersion: "v1", Kind: "PersistentVolume", Name: "disk", UID: "disk-uid"}, Type: corev1.EventTypeWarning, Reason: "VolumeFailedDelete", Message: message, LastTimestamp: metav1.NewTime(at)}
}
func TestRecentReleasedPVEventCoverage(t *testing.T) {
now := time.Now()
for _, tc := range []struct {
name string
change func(*CheckInput)
evaluated, passed, findings int
missing bool
}{
{"complete events", func(*CheckInput) {}, 1, 1, 0, false},
{"unreadable events", func(i *CheckInput) { i.Events = nil }, 0, 0, 0, true},
{"partial events", func(i *CheckInput) { i.PVDeletionEventsComplete = false }, 0, 0, 0, true},
{"partial events with warning", func(i *CheckInput) {
i.PVDeletionEventsComplete = false
i.Events = []*corev1.Event{deleteFailure(now, "visible failure")}
}, 1, 0, 1, false},
{"unreadable events after grace", func(i *CheckInput) {
i.Events = nil
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-2 * time.Hour)))
}, 1, 0, 1, false},
{"Retain does not need events", func(i *CheckInput) {
i.Events = nil
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain
}, 1, 1, 0, false},
{"Bound does not need events", func(i *CheckInput) {
i.Events = nil
i.PersistentVolumes[0].Status.Phase = corev1.VolumeBound
}, 1, 1, 0, false},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute)))
tc.change(i)
tr := newEvalTracker()
r := buildResults(checkStorage(tr, i, now), tr, nil)
count := r.CheckCounts["releasedPV"]
if count.Evaluated != tc.evaluated || count.Passed != tc.passed || len(r.Findings) != tc.findings || slices.Contains(r.MissingInputs, "pv-deletion-events") != tc.missing {
t.Fatalf("evaluated=%d passed=%d findings=%+v missing=%v", count.Evaluated, count.Passed, r.Findings, r.MissingInputs)
}
complete := i.Events != nil && i.PVDeletionEventsComplete
for _, f := range r.Findings {
if strings.Contains(f.Message, "Deletion warning events not fully visible") == complete {
t.Fatalf("event coverage caveat wrong for complete=%v: %+v", complete, f)
}
}
})
}
}
func TestPVDeletionWarningsCurrentRelease(t *testing.T) {
now := time.Now()
for _, tc := range []struct {
name string
change func(*CheckInput)
message string
}{
{"previous release warning", func(*CheckInput) {}, ""},
{"previous release warning after grace", func(i *CheckInput) {
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-2 * time.Hour)))
}, "deletion has not completed"},
{"warning at transition", func(i *CheckInput) {
i.Events[0].LastTimestamp = *i.PersistentVolumes[0].Status.LastPhaseTransitionTime
}, "deletion is failing"},
{"series continued in current release", func(i *CheckInput) {
i.Events[0].Series = &corev1.EventSeries{LastObservedTime: metav1.NewMicroTime(now)}
}, "deletion is failing"},
{"transition unavailable", func(i *CheckInput) {
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = nil
}, "deletion is failing"},
{"transition zero", func(i *CheckInput) {
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = &metav1.Time{}
}, "deletion is failing"},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims = []*corev1.PersistentVolumeClaim{}
i.PersistentVolumes[0].Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete
i.PersistentVolumes[0].Status.Phase = corev1.VolumeReleased
i.PersistentVolumes[0].Status.LastPhaseTransitionTime = ptr(metav1.NewTime(now.Add(-time.Minute)))
i.Events = []*corev1.Event{deleteFailure(now.Add(-3*time.Hour), "previous release failure")}
tc.change(i)
fs := checkStorage(newEvalTracker(), i, now)
if tc.message == "" {
if len(fs) != 0 {
t.Fatalf("previous release warning reused: %+v", fs)
}
} else if len(fs) != 1 || !strings.Contains(fs[0].Message, tc.message) {
t.Fatalf("current release outcome: %+v", fs)
}
if tc.name == "previous release warning after grace" && strings.Contains(fs[0].Message, "previous release failure") {
t.Fatalf("stale warning included as evidence: %+v", fs[0])
}
})
}
}
func TestLongPendingPVCStorageClassEvidence(t *testing.T) {
now := time.Now()
for _, tc := range []struct {
name string
change func(*CheckInput)
message string
hidden bool
}{
{"unreadable classes", func(i *CheckInput) { i.StorageClasses = nil }, "binding mode not visible", true},
{"unreadable default class", func(i *CheckInput) {
i.StorageClasses = nil
i.PersistentVolumeClaims[0].Spec.StorageClassName = nil
}, "binding mode not visible", true},
{"missing named class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = ptr("missing") }, `StorageClass "missing" not found`, false},
{"empty class inventory", func(i *CheckInput) { i.StorageClasses = []*storagev1.StorageClass{} }, `StorageClass "local" not found`, false},
{"no default class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = nil }, "No default StorageClass applies", false},
{"no classes or default", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Spec.StorageClassName = nil
i.StorageClasses = []*storagev1.StorageClass{}
}, "No default StorageClass applies", false},
{"default WFFC class", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Spec.StorageClassName = nil
i.StorageClasses[0].Annotations = map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}
}, "Default StorageClass \"local\" uses WaitForFirstConsumer", false},
{"newest default class", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Spec.StorageClassName = nil
i.StorageClasses[0].Annotations = map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}
i.StorageClasses[0].CreationTimestamp = metav1.NewTime(now.Add(-time.Hour))
i.StorageClasses = append([]*storagev1.StorageClass{{ObjectMeta: metav1.ObjectMeta{Name: "new", CreationTimestamp: metav1.NewTime(now), Annotations: map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}}}}, i.StorageClasses...)
}, "Default StorageClass \"new\" applies", false},
{"default timestamp tie", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Spec.StorageClassName = nil
i.StorageClasses[0].Annotations = map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}
i.StorageClasses = append(i.StorageClasses, &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "alphabetical", Annotations: map[string]string{"storageclass.kubernetes.io/is-default-class": "true"}}})
}, "Default StorageClass \"alphabetical\" applies", false},
{"explicit no class", func(i *CheckInput) { i.PersistentVolumeClaims[0].Spec.StorageClassName = ptr("") }, "storage class \"\"", false},
{"explicit no class with unreadable classes", func(i *CheckInput) {
i.PersistentVolumeClaims[0].Spec.StorageClassName = ptr("")
i.StorageClasses = nil
}, "storage class \"\"", false},
} {
t.Run(tc.name, func(t *testing.T) {
i := storageInput()
i.PersistentVolumeClaims[0].Status.Phase = corev1.ClaimPending
tc.change(i)
tr := newEvalTracker()
fs := checkStorage(tr, i, now)
if len(fs) != 1 || !strings.Contains(fs[0].Message, tc.message) {
t.Fatalf("class evidence: %+v", fs)
}
// The claim is evaluated either way, so class visibility is a
// caveat in the finding, never a missing input.
if len(tr.missingInputs) != 0 {
t.Fatalf("evaluated claim reported as skipped: %+v", tr.missingInputs)
}
if !tc.hidden && strings.Contains(fs[0].Message, "binding mode not visible") {
t.Fatalf("readable class evidence described as invisible: %+v", fs[0])
}
})
}
}