pkg/helmhistory/helmhistory.go
Rama main · réplica de lectura del repositorio original.
package helmhistory
import (
"fmt"
"regexp"
"sort"
"strings"
"time"
"github.com/skyhook-io/radar/pkg/timeutil"
)
const (
KindReleaseFailed OperationKind = "release_failed"
KindUpgradeFailed OperationKind = "upgrade_failed"
KindUpgradeRolledBack OperationKind = "upgrade_rolled_back"
KindRollback OperationKind = "rollback"
KindPending OperationKind = "pending"
StatusFailed OperationStatus = "failed"
StatusRolledBack OperationStatus = "rolled_back"
StatusCompleted OperationStatus = "completed"
StatusStuck OperationStatus = "stuck_pending"
ConfidenceHigh Confidence = "high"
ConfidenceMedium Confidence = "medium"
ConfidenceLow Confidence = "low"
SourceStatus Source = "helm_status"
SourceHistory Source = "helm_history"
)
const defaultPendingStuckAfter = 10 * time.Minute
var (
rollbackToPattern = regexp.MustCompile(`(?i)^rollback to ([0-9]+)(?:\b|$)`)
upgradeFailurePattern = regexp.MustCompile(`(?i)^upgrade(?:\s+"[^"]+")?\s+failed:`)
)
type OperationKind string
type OperationStatus string
type Confidence string
type Source string
type Revision struct {
Revision int
Status string
Chart string
AppVersion string
Description string
Updated time.Time
Deleted time.Time
}
type Operation struct {
Kind OperationKind `json:"kind"`
Status OperationStatus `json:"status"`
Source Source `json:"source"`
Confidence Confidence `json:"confidence"`
Message string `json:"message"`
RawMessage string `json:"rawMessage,omitempty"`
Evidence string `json:"evidence,omitempty"`
FailureDescription string `json:"failureDescription,omitempty"`
Revision int `json:"revision,omitempty"`
FailedRevision int `json:"failedRevision,omitempty"`
RollbackRevision int `json:"rollbackRevision,omitempty"`
TargetRevision int `json:"targetRevision,omitempty"`
PendingStatus string `json:"pendingStatus,omitempty"`
Updated time.Time `json:"updated,omitempty"`
}
type Options struct {
PendingStuckAfter time.Duration
MaxOperations int
Now time.Time
}
type Analysis struct {
LastOperation *Operation `json:"lastOperation,omitempty"`
Operations []Operation `json:"operations,omitempty"`
}
func Analyze(releaseName string, currentRevision int, revisions []Revision, opts Options) Analysis {
if len(revisions) == 0 {
return Analysis{}
}
pendingStuckAfter := opts.PendingStuckAfter
if pendingStuckAfter <= 0 {
pendingStuckAfter = defaultPendingStuckAfter
}
now := opts.Now
if now.IsZero() {
now = time.Now()
}
ordered := append([]Revision(nil), revisions...)
sort.Slice(ordered, func(i, j int) bool {
return ordered[i].Revision < ordered[j].Revision
})
var ops []Operation
for i := 0; i < len(ordered); i++ {
rev := ordered[i]
status := normalizeStatus(rev.Status)
if status == "failed" {
if i+1 < len(ordered) {
next := ordered[i+1]
if next.Revision == rev.Revision+1 && isCompletedRevisionStatus(next.Status) {
if target, ok := rollbackTarget(next.Description); ok {
op := rolledBackOperation(rev, next, target)
ops = append(ops, op)
i++
continue
}
}
}
ops = append(ops, failedOperation(releaseName, rev, SourceHistory))
continue
}
if isCompletedRevisionStatus(status) {
if target, ok := rollbackTarget(rev.Description); ok {
ops = append(ops, rollbackOperation(rev, target))
}
}
}
current, hasCurrent := findCurrent(ordered, currentRevision)
resolvedCurrentRevision := currentRevision
if hasCurrent {
resolvedCurrentRevision = current.Revision
}
var live *Operation
if hasCurrent {
status := normalizeStatus(current.Status)
started := current.Updated
if status == "uninstalling" {
started = current.Deleted
}
switch {
case status == "failed":
op := failedOperation(releaseName, current, SourceStatus)
live = &op
case isPending(status) && !started.IsZero() && now.Sub(started) >= pendingStuckAfter:
op := pendingOperation(current, now.Sub(started))
live = &op
}
}
if live != nil && live.Status == StatusFailed {
ops = withoutDuplicateLiveFailure(ops, *live)
}
sort.Slice(ops, func(i, j int) bool {
return operationRevision(ops[i]) > operationRevision(ops[j])
})
if opts.MaxOperations > 0 && len(ops) > opts.MaxOperations {
ops = ops[:opts.MaxOperations]
}
var last *Operation
if live != nil {
last = live
} else {
for i := range ops {
if operationRevision(ops[i]) == resolvedCurrentRevision && (ops[i].Kind == KindUpgradeRolledBack || ops[i].Kind == KindRollback) {
op := ops[i]
last = &op
break
}
}
}
return Analysis{LastOperation: last, Operations: ops}
}
func rolledBackOperation(failed, rollback Revision, target int) Operation {
return Operation{
Kind: KindUpgradeRolledBack,
Status: StatusRolledBack,
Source: SourceHistory,
Confidence: ConfidenceMedium,
Message: fmt.Sprintf("Upgrade failed at rev %d; Helm rolled back to rev %d as rev %d.", failed.Revision, target, rollback.Revision),
Evidence: fmt.Sprintf("failed revision %d followed by rollback revision %d", failed.Revision, rollback.Revision),
FailureDescription: strings.TrimSpace(failed.Description),
FailedRevision: failed.Revision,
RollbackRevision: rollback.Revision,
TargetRevision: target,
Updated: rollback.Updated,
}
}
func failedOperation(releaseName string, rev Revision, source Source) Operation {
kind := KindReleaseFailed
message := fmt.Sprintf("Release failed at rev %d.", rev.Revision)
if isUpgradeFailureDescription(rev.Description) {
kind = KindUpgradeFailed
message = fmt.Sprintf("Upgrade failed at rev %d.", rev.Revision)
}
if rev.Description != "" {
message = message + " " + rev.Description
}
rawMessage := ""
if IsReadinessTimeoutMessage(message) {
rawMessage = strings.TrimSpace(rev.Description)
message = readinessTimeoutMessage(releaseName)
}
evidence := "Helm history revision status is failed"
if source == SourceStatus {
evidence = "latest Helm revision status is failed"
}
return Operation{
Kind: kind,
Status: StatusFailed,
Source: source,
Confidence: ConfidenceHigh,
Message: message,
RawMessage: rawMessage,
Evidence: evidence,
Revision: rev.Revision,
Updated: rev.Updated,
}
}
func IsReadinessTimeoutMessage(msg string) bool {
lower := strings.ToLower(msg)
if !strings.Contains(lower, "context deadline exceeded") || !strings.Contains(lower, "status: inprogress") {
return false
}
return strings.Contains(lower, "not ready") || strings.Contains(lower, "available: 0/")
}
func readinessTimeoutMessage(releaseName string) string {
name := strings.TrimSpace(releaseName)
if name == "" {
return "Helm release did not become ready before Helm timed out."
}
return fmt.Sprintf("Helm release %q did not become ready before Helm timed out.", name)
}
func rollbackOperation(rev Revision, target int) Operation {
return Operation{
Kind: KindRollback,
Status: StatusCompleted,
Source: SourceHistory,
Confidence: ConfidenceMedium,
Message: fmt.Sprintf("Helm rolled back to rev %d as rev %d.", target, rev.Revision),
Evidence: fmt.Sprintf("revision %d description indicates rollback", rev.Revision),
Revision: rev.Revision,
TargetRevision: target,
Updated: rev.Updated,
}
}
func pendingOperation(rev Revision, age time.Duration) Operation {
evidence := "latest Helm revision is still pending"
updated := rev.Updated
if normalizeStatus(rev.Status) == "uninstalling" {
evidence = "latest Helm revision is still uninstalling, timed from Helm Info.Deleted"
updated = rev.Deleted
}
return Operation{
Kind: KindPending,
Status: StatusStuck,
Source: SourceStatus,
Confidence: ConfidenceHigh,
Message: fmt.Sprintf("Release has been %s for %s.", rev.Status, timeutil.FormatAgeShort(age)),
Evidence: evidence,
Revision: rev.Revision,
PendingStatus: normalizeStatus(rev.Status),
Updated: updated,
}
}
func rollbackTarget(description string) (int, bool) {
matches := rollbackToPattern.FindStringSubmatch(strings.TrimSpace(description))
if len(matches) != 2 {
return 0, false
}
var target int
if _, err := fmt.Sscanf(matches[1], "%d", &target); err != nil || target <= 0 {
return 0, false
}
return target, true
}
func isUpgradeFailureDescription(description string) bool {
description = strings.TrimSpace(description)
if description == "" {
return false
}
return upgradeFailurePattern.MatchString(description)
}
func findCurrent(ordered []Revision, currentRevision int) (Revision, bool) {
for i := len(ordered) - 1; i >= 0; i-- {
if currentRevision <= 0 || ordered[i].Revision == currentRevision {
return ordered[i], true
}
}
return Revision{}, false
}
func isPending(status string) bool {
status = normalizeStatus(status)
return status == "pending-install" || status == "pending-upgrade" || status == "pending-rollback" || status == "uninstalling"
}
func isCompletedRevisionStatus(status string) bool {
status = normalizeStatus(status)
return status == "deployed" || status == "superseded"
}
func normalizeStatus(status string) string {
return strings.ToLower(strings.TrimSpace(status))
}
func withoutDuplicateLiveFailure(ops []Operation, live Operation) []Operation {
out := ops[:0]
for _, op := range ops {
if op.Status == StatusFailed && op.Revision == live.Revision {
continue
}
out = append(out, op)
}
return out
}
func operationRevision(op Operation) int {
if op.RollbackRevision > 0 {
return op.RollbackRevision
}
if op.Revision > 0 {
return op.Revision
}
return op.FailedRevision
}