notifications/alerts.go
Rama main · réplica de lectura del repositorio original.
package notifications
import (
"context"
"fmt"
"time"
"github.com/coroot/coroot/db"
"github.com/coroot/coroot/model"
"github.com/coroot/coroot/timeseries"
"github.com/coroot/coroot/utils"
"k8s.io/klog"
)
type AlertNotifier struct {
db *db.DB
}
func NewAlertNotifier(database *db.DB) *AlertNotifier {
n := &AlertNotifier{db: database}
go func() {
for range time.Tick(retryInterval) {
n.sendAlerts()
}
}()
return n
}
func (n *AlertNotifier) Enqueue(project *db.Project, app *model.Application, alert *model.Alert, rule *model.AlertingRule, now timeseries.Time) {
category := model.ApplicationCategoryApplication
if app != nil {
category = app.Category
} else if rule.NotificationCategory != "" {
category = rule.NotificationCategory
}
categorySettings := project.GetApplicationCategories()[category]
if categorySettings == nil {
return
}
notificationSettings := categorySettings.NotificationSettings.Alerts
if !notificationSettings.Enabled {
return
}
if slack := notificationSettings.Slack; slack != nil && slack.Enabled {
n.enqueue(now, project, alert, rule, db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeSlack, SlackChannel: slack.Channel})
}
if teams := notificationSettings.Teams; teams != nil && teams.Enabled {
n.enqueue(now, project, alert, rule, db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeTeams, TeamsChannel: teams.Channel})
}
if pagerduty := notificationSettings.Pagerduty; pagerduty != nil && pagerduty.Enabled {
n.enqueue(now, project, alert, rule, db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypePagerduty})
}
if opsgenie := notificationSettings.Opsgenie; opsgenie != nil && opsgenie.Enabled {
n.enqueue(now, project, alert, rule, db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeOpsgenie})
}
if webhook := notificationSettings.Webhook; webhook != nil && webhook.Enabled {
n.enqueue(now, project, alert, rule, db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeWebhook})
}
n.sendAlerts()
}
func (n *AlertNotifier) sendAlerts() {
ps, err := n.db.GetProjects()
if err != nil {
klog.Errorln(err)
return
}
if len(ps) == 0 {
return
}
projects := map[db.ProjectId]*db.Project{}
for _, p := range ps {
projects[p.Id] = p
}
type destinationKey struct {
projectId db.ProjectId
destination db.IncidentNotificationDestination
}
failedDestinations := map[destinationKey]bool{}
notifications, err := n.db.GetNotSentAlertNotifications(timeseries.Now().Add(-retryWindow))
if err != nil {
klog.Errorln(err)
return
}
for _, notification := range notifications {
dKey := destinationKey{projectId: notification.ProjectId, destination: notification.Destination}
if failedDestinations[dKey] {
continue
}
project := projects[notification.ProjectId]
if project == nil {
continue
}
integrations := project.Settings.Integrations
var sendErr error
client := getClient(notification.Destination, integrations, NotificationTypeAlert)
if client != nil {
if notification.Destination.IntegrationType == db.IntegrationTypeSlack {
if prevNotifications, err := n.db.GetPreviousAlertNotifications(notification); err != nil {
klog.Errorln(err)
} else {
for _, pn := range prevNotifications {
if pn.ExternalKey != "" {
notification.ExternalKey = pn.ExternalKey
}
}
}
}
ctx, cancel := context.WithTimeout(context.Background(), sendTimeout)
sendErr = client.SendAlert(ctx, integrations.BaseUrl, ¬ification)
cancel()
}
if sendErr != nil {
klog.Errorf("failed to send alert to %s: %s", notification.Destination.IntegrationType, sendErr)
failedDestinations[dKey] = true
} else {
notification.SentAt = timeseries.Now()
if err = n.db.UpdateAlertNotification(notification); err != nil {
klog.Errorln(err)
}
}
}
}
func (n *AlertNotifier) enqueue(now timeseries.Time, project *db.Project, alert *model.Alert, rule *model.AlertingRule, destination db.IncidentNotificationDestination) {
notification := db.AlertNotification{
ProjectId: project.Id,
AlertId: alert.Id,
RuleId: alert.RuleId,
ApplicationId: alert.ApplicationId,
Destination: destination,
Timestamp: now,
Status: alert.Severity,
}
details := &db.AlertNotificationDetails{
ProjectName: project.Name,
RuleName: rule.Name,
Severity: alert.Severity.String(),
Summary: alert.Summary,
Details: filterAlertDetails(alert.Details),
}
if alert.ResolvedAt > 0 {
details.Duration = utils.FormatDurationShort(alert.ResolvedAt.Sub(alert.OpenedAt), 2)
}
switch destination.IntegrationType {
case db.IntegrationTypeSlack, db.IntegrationTypeTeams, db.IntegrationTypeWebhook:
if alert.ResolvedAt > 0 {
n.onResolve("", notification, details)
} else {
n.onOpen("", notification, details)
}
case db.IntegrationTypePagerduty, db.IntegrationTypeOpsgenie:
openKey, err := n.getOpenAlert(notification)
if err != nil {
klog.Errorln(err)
return
}
externalKey := fmt.Sprintf("%s:%s:%s", project.Id, alert.Id, alert.Severity.String())
if alert.ResolvedAt > 0 {
if openKey != "" {
n.onResolve(openKey, notification, details)
}
} else {
n.onOpen(externalKey, notification, details)
}
default:
klog.Errorln("unknown destination:", destination)
}
}
func (n *AlertNotifier) onOpen(externalKey string, notification db.AlertNotification, details *db.AlertNotificationDetails) {
notification.ExternalKey = externalKey
notification.Details = details
n.db.PutAlertNotification(notification)
}
func (n *AlertNotifier) onResolve(externalKey string, notification db.AlertNotification, details *db.AlertNotificationDetails) {
notification.Status = model.OK
notification.ExternalKey = externalKey
notification.Details = details
n.db.PutAlertNotification(notification)
}
func (n *AlertNotifier) getOpenAlert(notification db.AlertNotification) (string, error) {
prevNotifications, err := n.db.GetPreviousAlertNotifications(notification)
if err != nil {
return "", err
}
var openKey string
for _, prev := range prevNotifications {
if prev.Status == model.OK {
if openKey == prev.ExternalKey {
openKey = ""
}
} else {
openKey = prev.ExternalKey
}
}
return openKey, nil
}
func filterAlertDetails(details []model.AlertDetail) []model.AlertDetail {
var filtered []model.AlertDetail
for _, d := range details {
if d.Name == "PromQL" || d.Name == "PromQLChart" {
continue
}
filtered = append(filtered, d)
}
return filtered
}
func alertUrl(baseUrl string, n *db.AlertNotification) string {
return fmt.Sprintf("%s/p/%s/alerts?alert=%s", baseUrl, n.ProjectId, n.AlertId)
}
func EnqueueResolvedAlerts(database *db.DB, project *db.Project, alerts []*model.Alert, rule *model.AlertingRule) {
now := timeseries.Now()
for _, alert := range alerts {
alert.ResolvedAt = now
category := alert.ApplicationCategory
if category == "" {
if rule.NotificationCategory != "" {
category = rule.NotificationCategory
} else {
category = model.ApplicationCategoryApplication
}
}
categorySettings := project.GetApplicationCategories()[category]
if categorySettings == nil {
continue
}
notificationSettings := categorySettings.NotificationSettings.Alerts
if !notificationSettings.Enabled {
continue
}
enqueueResolvedAlert(database, now, project, alert, rule, notificationSettings)
}
}
func enqueueResolvedAlert(database *db.DB, now timeseries.Time, project *db.Project, alert *model.Alert, rule *model.AlertingRule, settings db.ApplicationCategoryAlertNotificationSettings) {
details := &db.AlertNotificationDetails{
ProjectName: project.Name,
RuleName: rule.Name,
Severity: alert.Severity.String(),
Summary: alert.Summary,
Details: filterAlertDetails(alert.Details),
Duration: utils.FormatDurationShort(now.Sub(alert.OpenedAt), 2),
ResolvedBy: alert.ResolvedBy,
}
if slack := settings.Slack; slack != nil && slack.Enabled {
notification := db.AlertNotification{
ProjectId: project.Id,
AlertId: alert.Id,
RuleId: alert.RuleId,
ApplicationId: alert.ApplicationId,
Destination: db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeSlack, SlackChannel: slack.Channel},
Timestamp: now,
Status: model.OK,
Details: details,
}
database.PutAlertNotification(notification)
}
if teams := settings.Teams; teams != nil && teams.Enabled {
notification := db.AlertNotification{
ProjectId: project.Id,
AlertId: alert.Id,
RuleId: alert.RuleId,
ApplicationId: alert.ApplicationId,
Destination: db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeTeams, TeamsChannel: teams.Channel},
Timestamp: now,
Status: model.OK,
Details: details,
}
database.PutAlertNotification(notification)
}
if pagerduty := settings.Pagerduty; pagerduty != nil && pagerduty.Enabled {
externalKey := fmt.Sprintf("%s:%s:%s", project.Id, alert.Id, alert.Severity.String())
notification := db.AlertNotification{
ProjectId: project.Id,
AlertId: alert.Id,
RuleId: alert.RuleId,
ApplicationId: alert.ApplicationId,
Destination: db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypePagerduty},
Timestamp: now,
Status: model.OK,
ExternalKey: externalKey,
Details: details,
}
database.PutAlertNotification(notification)
}
if opsgenie := settings.Opsgenie; opsgenie != nil && opsgenie.Enabled {
externalKey := fmt.Sprintf("%s:%s:%s", project.Id, alert.Id, alert.Severity.String())
notification := db.AlertNotification{
ProjectId: project.Id,
AlertId: alert.Id,
RuleId: alert.RuleId,
ApplicationId: alert.ApplicationId,
Destination: db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeOpsgenie},
Timestamp: now,
Status: model.OK,
ExternalKey: externalKey,
Details: details,
}
database.PutAlertNotification(notification)
}
if webhook := settings.Webhook; webhook != nil && webhook.Enabled {
notification := db.AlertNotification{
ProjectId: project.Id,
AlertId: alert.Id,
RuleId: alert.RuleId,
ApplicationId: alert.ApplicationId,
Destination: db.IncidentNotificationDestination{IntegrationType: db.IntegrationTypeWebhook},
Timestamp: now,
Status: model.OK,
Details: details,
}
database.PutAlertNotification(notification)
}
}