internal/issues/pods.go
Rama main · réplica de lectura del repositorio original.
package issues
import (
"strconv"
"time"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/sophotechlabs/spinoza/internal/unstr"
)
const detectorStartup = "pod-startup"
const (
phaseFailed = "Failed"
phaseSucceeded = "Succeeded"
)
const titleCrashLoop = "CrashLoopBackOff"
var containerStatusFields = []string{
"initContainerStatuses",
"containerStatuses",
"ephemeralContainerStatuses",
}
var imageReasons = map[string]bool{
"ImagePullBackOff": true,
"ErrImagePull": true,
"InvalidImageName": true,
"RegistryUnavailable": true,
"ImageInspectError": true,
}
var configReasons = map[string]bool{
"CreateContainerConfigError": true,
"CreateContainerError": true,
"RunContainerError": true,
"StartError": true,
}
type symptom struct {
severity int
title string
detail string
action string
}
func podFindings(snap *snapshot) []finding {
out := []finding{}
for _, item := range snap.of("", kindPod) {
found, ok := podSymptom(item.obj)
if !ok {
continue
}
moved := rolloutOf(snap, item)
out = append(out, finding{
detector: detectorStartup,
severity: found.severity,
title: found.title,
detail: found.detail,
action: found.action,
change: moved.what,
changedAt: moved.at,
kind: kindPod,
subject: item,
since: podSince(item.obj),
})
}
return out
}
func podSymptom(pod *unstructured.Unstructured) (symptom, bool) {
if pod.GetDeletionTimestamp() != nil {
return symptom{}, false
}
phase := unstr.String(pod, "status", "phase")
if phase == phaseSucceeded {
return symptom{}, false
}
worstFault, found := worstContainerSymptom(pod)
if found {
return worstFault, true
}
if phase == phaseFailed {
return failedPodSymptom(pod), true
}
return unschedulableSymptom(pod)
}
func worstContainerSymptom(pod *unstructured.Unstructured) (symptom, bool) {
best := symptom{}
seen := false
for _, field := range containerStatusFields {
for _, raw := range unstr.Slice(pod, "status", field) {
entry, ok := raw.(map[string]any)
if !ok {
continue
}
found, hit := containerSymptomOf(entry)
if !hit {
continue
}
if !seen || found.severity > best.severity {
best = found
seen = true
}
}
}
return best, seen
}
func containerSymptomOf(entry map[string]any) (symptom, bool) {
name := unstr.At(entry, "name")
state, ok := entry["state"].(map[string]any)
if !ok {
return symptom{}, false
}
if waiting, isWaiting := state["waiting"].(map[string]any); isWaiting {
return waitingSymptom(name, waiting, entry)
}
if terminated, isTerminated := state["terminated"].(map[string]any); isTerminated {
return terminatedSymptom(name, terminated, entry)
}
return symptom{}, false
}
func waitingSymptom(name string, waiting, entry map[string]any) (symptom, bool) {
reason := unstr.At(waiting, "reason")
message := unstr.At(waiting, "message")
switch {
case reason == titleCrashLoop:
return crashSymptom(name, entry), true
case imageReasons[reason]:
return symptom{
severity: severityFatal,
title: reason,
detail: containerLine(name, message, reason),
action: "check the image name, tag and pull secret",
}, true
case configReasons[reason]:
return symptom{
severity: severityFatal,
title: reason,
detail: containerLine(name, message, reason),
action: "check the referenced ConfigMap, Secret and command",
}, true
}
return symptom{}, false
}
func terminatedSymptom(name string, terminated, entry map[string]any) (symptom, bool) {
reason := unstr.At(terminated, "reason")
if reason == "OOMKilled" {
return symptom{
severity: severityFatal,
title: reason,
detail: "container " + name + " was killed for exceeding its memory limit",
action: "raise the memory limit, or make it hold less",
}, true
}
if !crashing(terminated, entry) {
return symptom{}, false
}
return crashSymptom(name, entry), true
}
func crashing(terminated, entry map[string]any) bool {
code, ok := terminated["exitCode"].(int64)
if !ok {
return false
}
if code == 0 {
return false
}
restarts, counted := entry["restartCount"].(int64)
if !counted {
return false
}
return restarts > 0
}
func crashSymptom(name string, entry map[string]any) symptom {
return symptom{
severity: severityFatal,
title: titleCrashLoop,
detail: crashDetail(name, entry),
action: "read the container's logs",
}
}
func crashDetail(name string, entry map[string]any) string {
detail := "container " + name + " keeps exiting"
last, ok := entry["lastState"].(map[string]any)
if !ok {
return detail
}
terminated, isTerminated := last["terminated"].(map[string]any)
if !isTerminated {
return detail
}
code, hasCode := terminated["exitCode"].(int64)
if hasCode {
detail += " with exit code " + strconv.FormatInt(code, 10)
}
reason := unstr.At(terminated, "reason")
if reason != "" && reason != "Error" {
detail += " (" + reason + ")"
}
restarts, hasRestarts := entry["restartCount"].(int64)
if hasRestarts && restarts > 0 {
detail += ", restarted " + strconv.FormatInt(restarts, 10) + " times"
}
return detail
}
func containerLine(name, message, reason string) string {
if message != "" {
return "container " + name + ": " + message
}
return "container " + name + ": " + reason
}
func failedPodSymptom(pod *unstructured.Unstructured) symptom {
reason := unstr.String(pod, "status", "reason")
message := unstr.String(pod, "status", "message")
title := reason
if title == "" {
title = "PodFailed"
}
detail := message
if detail == "" {
detail = "the pod reached the Failed phase"
}
return symptom{
severity: severityFatal,
title: title,
detail: detail,
action: "delete it, or fix what its controller creates",
}
}
func unschedulableSymptom(pod *unstructured.Unstructured) (symptom, bool) {
for _, raw := range unstr.Slice(pod, "status", "conditions") {
entry, ok := raw.(map[string]any)
if !ok {
continue
}
if unstr.At(entry, "type") != "PodScheduled" || unstr.At(entry, "status") == "True" {
continue
}
message := unstr.At(entry, "message")
if message == "" {
message = "no node accepted the pod"
}
return symptom{
severity: severityFatal,
title: "Unschedulable",
detail: message,
action: "free capacity, relax the affinity or tolerate the taint",
}, true
}
return symptom{}, false
}
func podSince(pod *unstructured.Unstructured) time.Time {
broke := lastExit(pod)
if moved := newestCondition(pod); moved.After(broke) {
broke = moved
}
if !broke.IsZero() {
return broke
}
return podBoundAt(pod)
}
func podBoundAt(pod *unstructured.Unstructured) time.Time {
at, err := time.Parse(time.RFC3339, unstr.String(pod, "status", "startTime"))
if err == nil {
return at
}
return pod.GetCreationTimestamp().Time
}
func lastExit(pod *unstructured.Unstructured) time.Time {
newest := time.Time{}
for _, field := range containerStatusFields {
for _, raw := range unstr.Slice(pod, "status", field) {
entry, ok := raw.(map[string]any)
if !ok {
continue
}
last, hasLast := entry["lastState"].(map[string]any)
if !hasLast {
continue
}
terminated, isTerminated := last["terminated"].(map[string]any)
if !isTerminated {
continue
}
at, err := time.Parse(time.RFC3339, unstr.At(terminated, "finishedAt"))
if err == nil && at.After(newest) {
newest = at
}
}
}
return newest
}