constructor/queries.go

Rama main · réplica de lectura del repositorio original.

package constructor

import (
	"fmt"
	"net"
	"slices"
	"strconv"
	"strings"
	"time"

	"github.com/coroot/coroot/db"
	"github.com/coroot/coroot/model"
	"github.com/coroot/coroot/timeseries"
	"github.com/coroot/coroot/utils"
	promModel "github.com/prometheus/common/model"
	"golang.org/x/exp/maps"
	"k8s.io/klog"
)

const (
	qApplicationCustomSLI = "application_custom_sli"

	qRecordingRuleApplicationLogMessages         = "rr_application_log_messages"
	qRecordingRuleApplicationTCPSuccessful       = "rr_connection_tcp_successful"
	qRecordingRuleApplicationTCPActive           = "rr_connection_tcp_active"
	qRecordingRuleApplicationTCPFailed           = "rr_connection_tcp_failed"
	qRecordingRuleApplicationTCPConnectionTime   = "rr_connection_tcp_connection_time"
	qRecordingRuleApplicationTCPBytesSent        = "rr_connection_tcp_bytes_sent"
	qRecordingRuleApplicationTCPBytesReceived    = "rr_connection_tcp_bytes_received"
	qRecordingRuleApplicationTCPRetransmissions  = "rr_connection_tcp_retransmissions"
	qRecordingRuleApplicationNetLatency          = "rr_connection_net_latency"
	qRecordingRuleApplicationL7Requests          = "rr_connection_l7_requests"
	qRecordingRuleApplicationL7Latency           = "rr_connection_l7_latency"
	qRecordingRuleApplicationTraffic             = "rr_application_traffic"
	qRecordingRuleApplicationL7Histogram         = "rr_application_l7_histogram"
	qRecordingRuleApplicationL7InboundRequests   = "rr_application_l7_inbound_requests"
	qRecordingRuleApplicationL7InboundHistogram  = "rr_application_l7_inbound_histogram"
	qRecordingRuleApplicationLLMInboundHistogram = "rr_application_llm_inbound_histogram"
	qRecordingRuleApplicationCategories          = "rr_application_categories"
	qRecordingRuleApplicationSLO                 = "rr_application_slo"
	qRecordingRuleApplicationExternalEndpoint    = "rr_connection_external_endpoints"
	qRecordingRuleApplicationDNSRequests         = "rr_application_dns_requests"
	qRecordingRuleApplicationDNSLatency          = "rr_application_dns_latency"
)

var applicationAnnotations = maps.Keys(model.ApplicationAnnotationLabels)

var qConnectionAggregations = []string{
	qRecordingRuleApplicationTCPSuccessful,
	qRecordingRuleApplicationTCPActive,
	qRecordingRuleApplicationTCPFailed,
	qRecordingRuleApplicationTCPConnectionTime,
	qRecordingRuleApplicationTCPBytesSent,
	qRecordingRuleApplicationTCPBytesReceived,
	qRecordingRuleApplicationTCPRetransmissions,
	qRecordingRuleApplicationNetLatency,
	qRecordingRuleApplicationL7Requests,
	qRecordingRuleApplicationL7Latency,
	qRecordingRuleApplicationTraffic,
	qRecordingRuleApplicationExternalEndpoint,
	qRecordingRuleApplicationDNSRequests,
	qRecordingRuleApplicationDNSLatency,
}

var (
	possibleNamespaceLabels  = []string{"namespace", "ns", "kubernetes_namespace", "kubernetes_ns", "k8s_namespace", "k8s_ns"}
	possiblePodLabels        = []string{"pod", "pod_name", "kubernetes_pod", "k8s_pod"}
	possibleDBInstanceLabels = []string{"address", "instance", "rds_instance_id", "ec_instance_id"}
)

type Query struct {
	Name   string
	Query  string
	Labels *utils.StringSet

	InstanceToInstance bool
	FillFunc           timeseries.FillFunc
}

func (q Query) WithFillFunc(f timeseries.FillFunc) Query {
	q.FillFunc = f
	return q
}

func Q(name, query string, labels ...string) Query {
	ls := utils.NewStringSet(model.LabelMachineId, model.LabelSystemUuid, model.LabelContainerId, model.LabelDestination, model.LabelDestinationIP, model.LabelActualDestination)
	ls.Add(labels...)
	return Query{Name: name, Query: query, Labels: ls}
}

func qItoI(name, query string, labels ...string) Query {
	q := Q(name, query, append(labels, "app_id")...)
	q.InstanceToInstance = true
	return q
}

func qPod(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"uid"}, labels)...)
}

func qRDS(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"rds_instance_id"}, labels)...)
}

func qCloudSQL(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"cloudsql_instance_id"}, labels)...)
}

func qMemorystore(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"memorystore_instance_id"}, labels)...)
}

func qOCIDB(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"oci_db_id"}, labels)...)
}

func qOCICache(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"oci_cache_id"}, labels)...)
}

func qDB(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat(possibleDBInstanceLabels, possibleNamespaceLabels, possiblePodLabels, labels)...)
}

func qJVM(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"jvm"}, labels)...)
}

func qDotNet(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"application"}, labels)...)
}

func qFargateContainer(name, query string, labels ...string) Query {
	return Q(name, query, slices.Concat([]string{"kubernetes_io_hostname", "namespace", "pod", "container"}, labels)...)
}

func l7Req(metric string) string {
	return fmt.Sprintf(`sum by(app_id, destination, actual_destination, status) (rate(%s{app_id!=""}[$RANGE])) or rate(%s{app_id=""}[$RANGE])`, metric, metric)
}

func l7ReqWithMethod(metric string) string {
	return fmt.Sprintf(`sum by(app_id, destination, actual_destination, status, method) (rate(%s{app_id!=""}[$RANGE])) or rate(%s{app_id=""}[$RANGE])`, metric, metric)
}

func l7Latency(metric string) string {
	return fmt.Sprintf(`sum by(app_id, destination, actual_destination) (rate(%s{app_id!="", type!="llm"}[$RANGE])) or rate(%s{app_id="", type!="llm"}[$RANGE])`, metric, metric)
}

func l7Histogram(metric string) string {
	return fmt.Sprintf(`sum by(app_id, destination, actual_destination, le) (rate(%s{app_id!="", type!="llm"}[$RANGE])) or rate(%s{app_id="", type!="llm"}[$RANGE])`, metric, metric)
}

func l7InboundReq(metric string) string {
	return fmt.Sprintf(`sum by(app_id, status) (rate(%s{app_id!=""}[$RANGE])) or rate(%s{app_id=""}[$RANGE])`, metric, metric)
}

func l7InboundHistogram(metric string) string {
	return fmt.Sprintf(`sum by(app_id, le) (rate(%s{app_id!=""}[$RANGE])) or rate(%s{app_id=""}[$RANGE])`, metric, metric)
}

var QUERIES = []Query{
	Q("node_agent_info", `node_agent_info`, "version"),

	Q("up", `up`, "job", "instance"),

	Q("node_info", `node_info`, "hostname", "kernel_version"),
	Q("node_cloud_info", `node_cloud_info`, "provider", "region", "availability_zone", "instance_type", "instance_life_cycle"),
	Q("node_uptime_seconds", `node_uptime_seconds`),
	Q("node_cpu_cores", `node_resources_cpu_logical_cores`),
	Q("node_cpu_usage_percent", `sum(rate(node_resources_cpu_usage_seconds_total{mode!="idle"}[$RANGE])) without(mode) /sum(rate(node_resources_cpu_usage_seconds_total[$RANGE])) without(mode)*100`),
	Q("node_cpu_usage_by_mode", `rate(node_resources_cpu_usage_seconds_total{mode!="idle"}[$RANGE]) / ignoring(mode) group_left sum(rate(node_resources_cpu_usage_seconds_total[$RANGE])) without(mode)*100`, "mode"),
	Q("node_memory_total_bytes", `node_resources_memory_total_bytes`),
	Q("node_memory_available_bytes", `node_resources_memory_available_bytes`),
	Q("node_memory_free_bytes", `node_resources_memory_free_bytes`),
	Q("node_memory_cached_bytes", `node_resources_memory_cached_bytes`),
	Q("node_disk_read_time", `rate(node_resources_disk_read_time_seconds_total[$RANGE])`, "device"),
	Q("node_disk_write_time", `rate(node_resources_disk_write_time_seconds_total[$RANGE])`, "device"),
	Q("node_disk_reads", `rate(node_resources_disk_reads_total[$RANGE])`, "device"),
	Q("node_disk_writes", `rate(node_resources_disk_writes_total[$RANGE])`, "device"),
	Q("node_disk_read_bytes", `rate(node_resources_disk_read_bytes_total[$RANGE])`, "device"),
	Q("node_disk_written_bytes", `rate(node_resources_disk_written_bytes_total[$RANGE])`, "device"),
	Q("node_disk_io_time", `rate(node_resources_disk_io_time_seconds_total[$RANGE])`, "device"),
	Q("node_net_up", `node_net_interface_up`, "interface"),
	Q("node_net_ip", `node_net_interface_ip`, "interface", "ip"),
	Q("node_net_rx_bytes", `rate(node_net_received_bytes_total[$RANGE])`, "interface"),
	Q("node_net_tx_bytes", `rate(node_net_transmitted_bytes_total[$RANGE])`, "interface"),
	Q("node_gpu_info", `node_gpu_info`, "gpu_uuid", "name", "driver_version"),
	Q("node_gpu_memory_total_bytes", `node_resources_gpu_memory_total_bytes`, "gpu_uuid"),
	Q("node_gpu_memory_used_bytes", `node_resources_gpu_memory_used_bytes`, "gpu_uuid"),
	Q("node_gpu_memory_utilization_percent_avg", `node_resources_gpu_memory_utilization_percent_avg`, "gpu_uuid"),
	Q("node_gpu_memory_utilization_percent_peak", `node_resources_gpu_memory_utilization_percent_peak`, "gpu_uuid"),
	Q("node_gpu_utilization_percent_avg", `node_resources_gpu_utilization_percent_avg`, "gpu_uuid"),
	Q("node_gpu_utilization_percent_peak", `node_resources_gpu_utilization_percent_peak`, "gpu_uuid"),
	Q("node_gpu_temperature_celsius", `node_resources_gpu_temperature_celsius`, "gpu_uuid"),
	Q("node_gpu_power_usage_watts", `node_resources_gpu_power_usage_watts`, "gpu_uuid"),

	Q("ip_to_fqdn", `sum by(fqdn, ip) (ip_to_fqdn)`, "ip", "fqdn"),

	Q("fargate_node_machine_cpu_cores", `machine_cpu_cores{eks_amazonaws_com_compute_type="fargate"}`, "eks_amazonaws_com_compute_type", "kubernetes_io_hostname", "topology_kubernetes_io_region", "topology_kubernetes_io_zone"),
	Q("fargate_node_machine_memory_bytes", `machine_memory_bytes{eks_amazonaws_com_compute_type="fargate"}`, "eks_amazonaws_com_compute_type", "kubernetes_io_hostname", "topology_kubernetes_io_region", "topology_kubernetes_io_zone"),

	qFargateContainer("fargate_container_spec_cpu_limit_cores", `container_spec_cpu_quota{eks_amazonaws_com_compute_type="fargate"}/container_spec_cpu_period{eks_amazonaws_com_compute_type="fargate"}`),
	qFargateContainer("fargate_container_cpu_usage_seconds", `rate(container_cpu_usage_seconds_total{eks_amazonaws_com_compute_type="fargate"}[$RANGE])`),
	qFargateContainer("fargate_container_cpu_cfs_throttled_seconds", `rate(container_cpu_cfs_throttled_seconds_total{eks_amazonaws_com_compute_type="fargate"}[$RANGE])`),
	qFargateContainer("fargate_container_spec_memory_limit_bytes", `container_spec_memory_limit_bytes{eks_amazonaws_com_compute_type="fargate"}`),
	qFargateContainer("fargate_container_memory_rss", `container_memory_rss{eks_amazonaws_com_compute_type="fargate"}`),
	qFargateContainer("fargate_container_memory_cache", `container_memory_cache{eks_amazonaws_com_compute_type="fargate"}`),
	qFargateContainer("fargate_container_oom_events_total", `container_oom_events_total{eks_amazonaws_com_compute_type="fargate"}`, "job", "instance"),

	Q("kube_node_info", `kube_node_info`, "node", "kernel_version"),
	Q("kube_service_info", `kube_service_info`, "namespace", "service", "cluster_ip"),
	Q("kube_service_port", `kube_service_port`, "namespace", "service", "port", "node_port"),
	Q("kube_service_spec_type", `kube_service_spec_type`, "namespace", "service", "type"),
	Q("kube_endpoint_address", `kube_endpoint_address`, "namespace", "endpoint", "ip", "port_number"),
	Q("kube_service_status_load_balancer_ingress", `kube_service_status_load_balancer_ingress`, "namespace", "service", "ip"),
	Q("kube_deployment_spec_replicas", `kube_deployment_spec_replicas`, "namespace", "deployment"),
	Q("kube_daemonset_status_desired_number_scheduled", `kube_daemonset_status_desired_number_scheduled`, "namespace", "daemonset"),
	Q("kube_statefulset_replicas", `kube_statefulset_replicas`, "namespace", "statefulset"),
	Q("kube_deployment_annotations", `kube_deployment_annotations`, append(applicationAnnotations, "namespace", "deployment")...),
	Q("kube_statefulset_annotations", `kube_statefulset_annotations`, append(applicationAnnotations, "namespace", "statefulset")...),
	Q("kube_daemonset_annotations", `kube_daemonset_annotations`, append(applicationAnnotations, "namespace", "daemonset")...),
	Q("kube_cronjob_annotations", `kube_cronjob_annotations`, append(applicationAnnotations, "namespace", "cronjob")...),
	Q("kube_job_owner", `kube_job_owner`, "namespace", "job_name", "owner_kind", "owner_name", "owner_is_controller"),

	qPod("kube_pod_info", `kube_pod_info`, "namespace", "pod", "created_by_name", "created_by_kind", "node", "pod_ip", "host_ip"),
	qPod("kube_pod_owner", `kube_pod_owner{owner_is_controller="false"}`, "owner_kind", "owner_name"),
	qPod("kube_pod_annotations", `kube_pod_annotations`, applicationAnnotations...),
	qPod("kube_pod_labels", `kube_pod_labels`,
		"label_postgres_operator_crunchydata_com_cluster", "label_postgres_operator_crunchydata_com_role",
		"label_cluster_name", "label_team", "label_application", "label_spilo_role",
		"label_role",
		"label_cnpg_io_cluster",
		"label_stackgres_io_cluster_name",
		"label_app_kubernetes_io_managed_by",
		"label_app_kubernetes_io_instance",
		"label_helm_sh_chart",
		"label_app_kubernetes_io_name",
		"label_app_kubernetes_io_component", "label_app_kubernetes_io_part_of",
		"label_valkey_io_cluster",
		"label_project_runner_gitlab_com_name",
		"label_serving_kserve_io_inferenceservice", "label_component",
		"label_nvidia_com_dynamo_graph_deployment_name", "label_nvidia_com_dynamo_component_type",
		"label_disaggregatedset_x_k8s_io_name", "label_disaggregatedset_x_k8s_io_role",
		"label_leaderworkerset_sigs_k8s_io_name", "label_leaderworkerset_sigs_k8s_io_worker_index",
	),
	qPod("kube_pod_status_phase", `kube_pod_status_phase > 0`, "phase"),
	qPod("kube_pod_status_ready", `kube_pod_status_ready{condition="true"}`),
	qPod("kube_pod_status_scheduled", `kube_pod_status_scheduled{condition="true"} > 0`),
	qPod("kube_pod_init_container_info", `kube_pod_init_container_info`, "namespace", "pod", "container"),
	qPod("kube_pod_container_resource_requests", `kube_pod_container_resource_requests`, "namespace", "pod", "container", "resource"),
	qPod("kube_pod_container_status_ready", `kube_pod_container_status_ready > 0`, "namespace", "pod", "container"),
	qPod("kube_pod_container_status_running", `kube_pod_container_status_running > 0`, "namespace", "pod", "container"),
	qPod("kube_pod_container_status_waiting", `kube_pod_container_status_waiting > 0`, "namespace", "pod", "container"),
	qPod("kube_pod_container_status_waiting_reason", `kube_pod_container_status_waiting_reason > 0`, "namespace", "pod", "container", "reason"),
	qPod("kube_pod_container_status_terminated", `kube_pod_container_status_terminated > 0`, "namespace", "pod", "container"),
	qPod("kube_pod_container_status_terminated_reason", `kube_pod_container_status_terminated_reason > 0`, "namespace", "pod", "container", "reason"),
	qPod("kube_pod_container_status_last_terminated_reason", `kube_pod_container_status_last_terminated_reason`, "namespace", "pod", "container", "reason"),

	Q("container_info", `container_info`, "image", "systemd_triggered_by", "systemd_type"),
	Q("container_application_type", `container_application_type`, "application_type"),
	Q("container_cpu_limit", `container_resources_cpu_limit_cores`),
	Q("container_cpu_usage", `rate(container_resources_cpu_usage_seconds_total[$RANGE])`),
	Q("container_cpu_delay", `rate(container_resources_cpu_delay_seconds_total[$RANGE])`),
	Q("container_throttled_time", `rate(container_resources_cpu_throttled_seconds_total[$RANGE])`),
	Q("container_memory_limit", `container_resources_memory_limit_bytes`),
	Q("container_memory_rss", `container_resources_memory_rss_bytes`),
	Q("container_memory_cache", `container_resources_memory_cache_bytes`),
	Q("container_memory_pressure", `rate(container_resources_memory_pressure_waiting_seconds_total[$RANGE])`, "kind"),
	Q("container_io_pressure", `rate(container_resources_io_pressure_waiting_seconds_total[$RANGE])`, "kind"),
	Q("container_oom_kills_total", `container_oom_kills_total % 10000000`, "job", "instance"),
	Q("container_restarts", `container_restarts_total % 10000000`, "job", "instance"),
	Q("container_volume_size", `container_resources_disk_size_bytes`, "mount_point", "volume", "device"),
	Q("container_volume_used", `container_resources_disk_used_bytes`, "mount_point", "volume", "device"),
	Q("container_gpu_usage_percent", `container_resources_gpu_usage_percent`, "gpu_uuid"),
	Q("container_gpu_memory_usage_percent", `container_resources_gpu_memory_usage_percent`, "gpu_uuid"),
	Q("container_net_tcp_listen_info", `container_net_tcp_listen_info`, "listen_addr", "proxy"),

	qItoI("container_net_latency", `avg by(app_id, destination_ip) (container_net_latency_seconds{app_id!=""}) or container_net_latency_seconds{app_id=""}`),
	qItoI("container_net_tcp_successful_connects", `sum by(app_id, destination, actual_destination) (rate(container_net_tcp_successful_connects_total{app_id!=""}[$RANGE])) or rate(container_net_tcp_successful_connects_total{app_id=""}[$RANGE])`),
	qItoI("container_net_tcp_failed_connects", `sum by(app_id, destination, actual_destination) (rate(container_net_tcp_failed_connects_total{app_id!=""}[$RANGE])) or rate(container_net_tcp_failed_connects_total{app_id=""}[$RANGE])`),
	qItoI("container_net_tcp_active_connections", `sum by(app_id, destination, actual_destination) (container_net_tcp_active_connections{app_id!=""}) or container_net_tcp_active_connections{app_id=""}`),
	qItoI("container_net_tcp_connection_time_seconds", `sum by(app_id, destination, actual_destination) (rate(container_net_tcp_connection_time_seconds_total{app_id!=""}[$RANGE])) or rate(container_net_tcp_connection_time_seconds_total{app_id=""}[$RANGE])`),
	qItoI("container_net_tcp_bytes_sent", `sum by(app_id, destination, actual_destination, az, region) (rate(container_net_tcp_bytes_sent_total{app_id!=""}[$RANGE])) or rate(container_net_tcp_bytes_sent_total{app_id=""}[$RANGE])`, "region", "az"),
	qItoI("container_net_tcp_bytes_received", `sum by(app_id, destination, actual_destination, az, region) (rate(container_net_tcp_bytes_received_total{app_id!=""}[$RANGE])) or rate(container_net_tcp_bytes_received_total{app_id=""}[$RANGE])`, "region", "az"),
	qItoI("container_net_tcp_retransmits", `sum by(app_id, destination, actual_destination) (rate(container_net_tcp_retransmits_total{app_id!=""}[$RANGE])) or rate(container_net_tcp_retransmits_total{app_id=""}[$RANGE])`),

	Q("container_log_messages", `container_log_messages_total % 10000000`, "level", "pattern_hash", "sample", "job", "instance"),

	qItoI("container_http_requests_count", l7Req("container_http_requests_total"), "status"),
	qItoI("container_http_requests_latency_total", l7Latency("container_http_requests_duration_seconds_total_sum")),
	qItoI("container_http_requests_histogram", l7Histogram("container_http_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_postgres_queries_count", l7Req("container_postgres_queries_total"), "status"),
	qItoI("container_postgres_queries_latency_total", l7Latency("container_postgres_queries_duration_seconds_total_sum")),
	qItoI("container_postgres_queries_histogram", l7Histogram("container_postgres_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_redis_queries_count", l7Req("container_redis_queries_total"), "status"),
	qItoI("container_redis_queries_latency_total", l7Latency("container_redis_queries_duration_seconds_total_sum")),
	qItoI("container_redis_queries_histogram", l7Histogram("container_redis_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_memcached_queries_count", l7Req("container_memcached_queries_total"), "status"),
	qItoI("container_memcached_queries_latency_total", l7Latency("container_memcached_queries_duration_seconds_total_sum")),
	qItoI("container_memcached_queries_histogram", l7Histogram("container_memcached_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_mysql_queries_count", l7Req("container_mysql_queries_total"), "status"),
	qItoI("container_mysql_queries_latency_total", l7Latency("container_mysql_queries_duration_seconds_total_sum")),
	qItoI("container_mysql_queries_histogram", l7Histogram("container_mysql_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_mongo_queries_count", l7Req("container_mongo_queries_total"), "status"),
	qItoI("container_mongo_queries_latency_total", l7Latency("container_mongo_queries_duration_seconds_total_sum")),
	qItoI("container_mongo_queries_histogram", l7Histogram("container_mongo_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_kafka_requests_count", l7Req("container_kafka_requests_total"), "status"),
	qItoI("container_kafka_requests_latency_total", l7Latency("container_kafka_requests_duration_seconds_total_sum")),
	qItoI("container_kafka_requests_histogram", l7Histogram("container_kafka_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_cassandra_queries_count", l7Req("container_cassandra_queries_total"), "status"),
	qItoI("container_cassandra_queries_latency_total", l7Latency("container_cassandra_queries_duration_seconds_total_sum")),
	qItoI("container_cassandra_queries_histogram", l7Histogram("container_cassandra_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_clickhouse_queries_count", l7Req("container_clickhouse_queries_total"), "status"),
	qItoI("container_clickhouse_queries_latency_total", l7Latency("container_clickhouse_queries_duration_seconds_total_sum")),
	qItoI("container_clickhouse_queries_histogram", l7Histogram("container_clickhouse_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_zookeeper_requests_count", l7Req("container_zookeeper_requests_total"), "status"),
	qItoI("container_zookeeper_requests_latency_total", l7Latency("container_zookeeper_requests_duration_seconds_total_sum")),
	qItoI("container_zookeeper_requests_histogram", l7Histogram("container_zookeeper_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_foundationdb_requests_count", l7Req("container_foundationdb_requests_total"), "status"),
	qItoI("container_foundationdb_requests_latency_total", l7Latency("container_foundationdb_requests_duration_seconds_total_sum")),
	qItoI("container_foundationdb_requests_histogram", l7Histogram("container_foundationdb_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_rabbitmq_messages", l7ReqWithMethod("container_rabbitmq_messages_total"), "status", "method"),
	qItoI("container_nats_messages", l7ReqWithMethod("container_nats_messages_total"), "status", "method"),

	qItoI("container_http_inbound_requests_count", l7InboundReq("container_http_inbound_requests_total"), "status"),
	qItoI("container_http_inbound_requests_histogram", l7InboundHistogram("container_http_inbound_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_llm_inbound_requests_count", l7InboundReq("container_llm_inbound_requests_total"), "status"),
	qItoI("container_llm_inbound_requests_histogram", l7InboundHistogram("container_llm_inbound_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_postgres_inbound_queries_count", l7InboundReq("container_postgres_inbound_queries_total"), "status"),
	qItoI("container_postgres_inbound_queries_histogram", l7InboundHistogram("container_postgres_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_redis_inbound_queries_count", l7InboundReq("container_redis_inbound_queries_total"), "status"),
	qItoI("container_redis_inbound_queries_histogram", l7InboundHistogram("container_redis_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_memcached_inbound_queries_count", l7InboundReq("container_memcached_inbound_queries_total"), "status"),
	qItoI("container_memcached_inbound_queries_histogram", l7InboundHistogram("container_memcached_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_mysql_inbound_queries_count", l7InboundReq("container_mysql_inbound_queries_total"), "status"),
	qItoI("container_mysql_inbound_queries_histogram", l7InboundHistogram("container_mysql_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_mongo_inbound_queries_count", l7InboundReq("container_mongo_inbound_queries_total"), "status"),
	qItoI("container_mongo_inbound_queries_histogram", l7InboundHistogram("container_mongo_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_kafka_inbound_requests_count", l7InboundReq("container_kafka_inbound_requests_total"), "status"),
	qItoI("container_kafka_inbound_requests_histogram", l7InboundHistogram("container_kafka_inbound_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_cassandra_inbound_queries_count", l7InboundReq("container_cassandra_inbound_queries_total"), "status"),
	qItoI("container_cassandra_inbound_queries_histogram", l7InboundHistogram("container_cassandra_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_clickhouse_inbound_queries_count", l7InboundReq("container_clickhouse_inbound_queries_total"), "status"),
	qItoI("container_clickhouse_inbound_queries_histogram", l7InboundHistogram("container_clickhouse_inbound_queries_duration_seconds_total_bucket"), "le"),
	qItoI("container_zookeeper_inbound_requests_count", l7InboundReq("container_zookeeper_inbound_requests_total"), "status"),
	qItoI("container_zookeeper_inbound_requests_histogram", l7InboundHistogram("container_zookeeper_inbound_requests_duration_seconds_total_bucket"), "le"),
	qItoI("container_foundationdb_inbound_requests_count", l7InboundReq("container_foundationdb_inbound_requests_total"), "status"),
	qItoI("container_foundationdb_inbound_requests_histogram", l7InboundHistogram("container_foundationdb_inbound_requests_duration_seconds_total_bucket"), "le"),

	Q("l7_requests_by_dest", "sum by(actual_destination, status) (rate(container_mongo_queries_total[$RANGE]) or rate(container_mysql_queries_total[$RANGE]))", "status"),
	Q("l7_total_latency_by_dest", "sum by(actual_destination) (rate(container_mongo_queries_duration_seconds_total_sum[$RANGE]) or rate(container_mysql_queries_duration_seconds_total_sum[$RANGE]))"),

	qItoI("container_dns_requests_total", `sum by(app_id, request_type, domain, status) (rate(container_dns_requests_total{app_id!=""}[$RANGE])) or rate(container_dns_requests_total{app_id=""}[$RANGE])`, "request_type", "domain", "status"),
	qItoI("container_dns_requests_latency", `sum by(app_id, le) (rate(container_dns_requests_duration_seconds_total_bucket{app_id!=""}[$RANGE])) or rate(container_dns_requests_duration_seconds_total_bucket{app_id=""}[$RANGE]) `, "le"),

	Q("aws_discovery_error", `aws_discovery_error`, "error"),
	Q("gcp_discovery_error", `gcp_discovery_error`, "error"),
	Q("oci_discovery_error", `oci_discovery_error`, "error"),
	qRDS("aws_rds_info", `aws_rds_info`, "cluster_id", "ipv4", "port", "engine", "engine_version", "instance_type", "storage_type", "region", "availability_zone", "multi_az"),
	qRDS("aws_rds_status", `aws_rds_status`, "status"),
	qRDS("aws_rds_cpu_cores", `aws_rds_cpu_cores`),
	qRDS("aws_rds_cpu_usage_percent", `aws_rds_cpu_usage_percent`, "mode"),
	qRDS("aws_rds_memory_total_bytes", `aws_rds_memory_total_bytes`),
	qRDS("aws_rds_memory_cached_bytes", `aws_rds_memory_cached_bytes`),
	qRDS("aws_rds_memory_free_bytes", `aws_rds_memory_free_bytes`),
	qRDS("aws_rds_storage_provisioned_iops", `aws_rds_storage_provisioned_iops`),
	qRDS("aws_rds_allocated_storage_gibibytes", `aws_rds_allocated_storage_gibibytes`),
	qRDS("aws_rds_fs_total_bytes", `aws_rds_fs_total_bytes{mount_point="/rdsdbdata"}`),
	qRDS("aws_rds_fs_used_bytes", `aws_rds_fs_used_bytes{mount_point="/rdsdbdata"}`),
	qRDS("aws_rds_io_util_percent", `aws_rds_io_util_percent`, "device"),
	qRDS("aws_rds_io_ops_per_second", `aws_rds_io_ops_per_second`, "device", "operation"),
	qRDS("aws_rds_io_await_seconds", `aws_rds_io_await_seconds`, "device"),
	qRDS("aws_rds_net_rx_bytes_per_second", `aws_rds_net_rx_bytes_per_second`, "interface"),
	qRDS("aws_rds_net_tx_bytes_per_second", `aws_rds_net_tx_bytes_per_second`, "interface"),
	qRDS("aws_rds_log_messages_total", `aws_rds_log_messages_total % 10000000`, "level", "pattern_hash", "sample", "job", "instance"),

	Q("aws_elasticache_info", `aws_elasticache_info`, "ec_instance_id", "cluster_id", "ipv4", "port", "engine", "engine_version", "instance_type", "region", "availability_zone"),
	Q("aws_elasticache_status", `aws_elasticache_status`, "ec_instance_id", "status"),

	qCloudSQL("gcp_cloudsql_info", `gcp_cloudsql_info`, "project", "region", "zone", "ipv4", "port", "engine", "engine_version", "tier", "availability_type", "connection_name", "instance_type", "primary_instance"),
	qCloudSQL("gcp_cloudsql_status", `gcp_cloudsql_status`, "status"),
	// the GCP metrics are loaded in this order: the totals first, then the metrics derived from them
	qCloudSQL("gcp_cloudsql_cpu_usage_percent", `gcp_cloudsql_cpu_usage_percent`),
	qCloudSQL("gcp_cloudsql_cpu_cores", `gcp_cloudsql_cpu_cores`),
	qCloudSQL("gcp_cloudsql_cpu_usage_cores", `gcp_cloudsql_cpu_usage_cores`),
	qCloudSQL("gcp_cloudsql_memory_total_bytes", `gcp_cloudsql_memory_total_bytes`),
	qCloudSQL("gcp_cloudsql_memory_components_percent", `gcp_cloudsql_memory_components_percent`, "component"),
	qCloudSQL("gcp_cloudsql_memory_used_bytes", `gcp_cloudsql_memory_used_bytes`),
	qCloudSQL("gcp_cloudsql_disk_total_bytes", `gcp_cloudsql_disk_total_bytes`),
	qCloudSQL("gcp_cloudsql_disk_used_bytes", `gcp_cloudsql_disk_used_bytes`),
	qCloudSQL("gcp_cloudsql_network_bytes_per_second", `gcp_cloudsql_network_bytes_per_second`, "direction"),
	qCloudSQL("gcp_cloudsql_io_ops_per_second", `gcp_cloudsql_io_ops_per_second`, "operation"),
	qCloudSQL("gcp_cloudsql_io_bytes_per_second", `gcp_cloudsql_io_bytes_per_second`, "operation"),
	qCloudSQL("gcp_cloudsql_log_messages_total", `gcp_cloudsql_log_messages_total % 10000000`, "level", "pattern_hash", "sample", "job", "instance"),
	qMemorystore("gcp_memorystore_info", `gcp_memorystore_info`, "project", "region", "zone", "ipv4", "port", "engine", "engine_version", "tier", "memory_size_gb", "instance"),
	qMemorystore("gcp_memorystore_status", `gcp_memorystore_status`, "status"),
	qMemorystore("gcp_memorystore_cpu_cores", `gcp_memorystore_cpu_cores`),
	qMemorystore("gcp_memorystore_cpu_usage_percent", `gcp_memorystore_cpu_usage_percent`),
	qMemorystore("gcp_memorystore_cpu_usage_cores", `gcp_memorystore_cpu_usage_cores`),
	qMemorystore("gcp_memorystore_memory_total_bytes", `gcp_memorystore_memory_total_bytes`),
	qMemorystore("gcp_memorystore_memory_used_bytes", `gcp_memorystore_memory_used_bytes`),
	qMemorystore("gcp_memorystore_network_bytes_per_second", `gcp_memorystore_network_bytes_per_second`, "direction"),
	// OCI: the totals come before the metrics derived from them
	qOCIDB("oci_db_info", `oci_db_info`, "name", "compartment", "region", "availability_domain", "ipv4", "port", "engine", "engine_version", "shape", "high_availability", "primary"),
	qOCIDB("oci_db_status", `oci_db_status`, "status"),
	qOCIDB("oci_db_cpu_cores", `oci_db_cpu_cores`),
	qOCIDB("oci_db_cpu_usage_percent", `oci_db_cpu_usage_percent`),
	qOCIDB("oci_db_cpu_usage_cores", `oci_db_cpu_usage_cores`),
	qOCIDB("oci_db_memory_total_bytes", `oci_db_memory_total_bytes`),
	qOCIDB("oci_db_memory_used_bytes", `oci_db_memory_used_bytes`),
	qOCIDB("oci_db_memory_usage_percent", `oci_db_memory_usage_percent`),
	qOCIDB("oci_db_disk_total_bytes", `oci_db_disk_total_bytes`),
	qOCIDB("oci_db_disk_used_bytes", `oci_db_disk_used_bytes`),
	qOCIDB("oci_db_network_bytes_per_second", `oci_db_network_bytes_per_second`, "direction"),
	qOCIDB("oci_db_io_ops_per_second", `oci_db_io_ops_per_second`, "operation"),
	qOCIDB("oci_db_io_bytes_per_second", `oci_db_io_bytes_per_second`, "operation"),
	qOCIDB("oci_db_io_latency_seconds", `oci_db_io_latency_seconds`, "operation"), // after the ops it is multiplied by
	qOCIDB("oci_db_log_messages_total", `oci_db_log_messages_total % 10000000`, "level", "pattern_hash", "sample", "job", "instance"),
	qOCICache("oci_cache_info", `oci_cache_info`, "name", "compartment", "region", "ipv4", "port", "engine", "engine_version", "node_count", "node_memory_gb"),
	qOCICache("oci_cache_status", `oci_cache_status`, "status"),
	qOCICache("oci_cache_memory_total_bytes", `oci_cache_memory_total_bytes`),
	qOCICache("oci_cache_cpu_usage_percent", `oci_cache_cpu_usage_percent`),
	qOCICache("oci_cache_memory_used_bytes", `oci_cache_memory_used_bytes`),
	qOCICache("oci_cache_network_bytes_per_second", `oci_cache_network_bytes_per_second`, "direction"),
	qOCICache("oci_cache_log_messages_total", `oci_cache_log_messages_total % 10000000`, "level", "pattern_hash", "sample", "job", "instance"),

	qDB("pg_up", `pg_up`),
	qDB("pg_scrape_error", `pg_scrape_error`, "error", "warning"),
	qDB("pg_info", `pg_info`, "server_version"),
	qDB("pg_setting", `pg_setting`, "name", "unit"),
	qDB("pg_connections", `pg_connections{db!="postgres"}`, "db", "user", "state", "query", "wait_event_type"),
	qDB("pg_lock_awaiting_queries", `pg_lock_awaiting_queries`, "db", "user", "blocking_query"),
	qDB("pg_latency_seconds", `pg_latency_seconds`, "summary"),
	qDB("pg_top_query_calls_per_second", `pg_top_query_calls_per_second`, "db", "user", "query"),
	qDB("pg_top_query_time_per_second", `pg_top_query_time_per_second`, "db", "user", "query"),
	qDB("pg_top_query_io_time_per_second", `pg_top_query_io_time_per_second`, "db", "user", "query"),
	qDB("pg_db_queries_per_second", `pg_db_queries_per_second`, "db"),
	qDB("pg_wal_current_lsn", `pg_wal_current_lsn`),
	qDB("pg_wal_throughput", `rate(pg_wal_current_lsn[$RANGE]) or rate(pg_wal_receive_lsn[$RANGE])`),
	qDB("pg_wal_receive_lsn", `pg_wal_receive_lsn`),
	qDB("pg_wal_reply_lsn", `pg_wal_reply_lsn`),
	qDB("pg_wal_replay_paused", `pg_wal_replay_paused`),
	qDB("pg_wal_receiver_status", `pg_wal_receiver_status`),
	qDB("pg_wal_size_bytes", `pg_wal_size_bytes`),
	qDB("pg_replication_slot_retained_wal_bytes", `pg_replication_slot_retained_wal_bytes`, "slot", "active", "wal_status"),
	qDB("pg_wal_archived_segments_total", `pg_wal_archived_segments_total`, "job"),
	qDB("pg_wal_archive_failures_total", `pg_wal_archive_failures_total`, "job"),
	qDB("pg_wal_archiving_status", `pg_wal_archiving_status`),
	qDB("pg_xid_age", `pg_xid_age`, "db"),
	qDB("pg_multixact_age", `pg_multixact_age`, "db"),
	qDB("pg_oldest_xmin_age", `pg_oldest_xmin_age`, "holder"),
	qDB("pg_transaction_seconds", `pg_transaction_seconds`, "db", "user", "query"),
	qDB("pg_database_size_bytes", `pg_database_size_bytes`, "db"),
	qDB("pg_table_size_bytes", `pg_table_size_bytes`, "db", "table"),
	qDB("pg_table_size_growth_bytes_per_second", `pg_table_size_growth_bytes_per_second`, "db", "table"),
	qDB("pg_db_table_bloat_bytes", `pg_db_table_bloat_bytes`, "db"),
	qDB("pg_db_index_bloat_bytes", `pg_db_index_bloat_bytes`, "db"),
	qDB("pg_table_bloat_bytes", `pg_table_bloat_bytes`, "db", "table"),
	qDB("pg_index_bloat_bytes", `pg_index_bloat_bytes`, "db", "table", "index"),
	qDB("pg_table_dead_tuple_bytes", `pg_table_dead_tuple_bytes`, "db", "table"),
	qDB("pg_table_dead_tuples", `pg_table_dead_tuples`, "db", "table"),
	qDB("pg_table_live_tuples", `pg_table_live_tuples`, "db", "table"),
	qDB("pg_table_seconds_since_last_autovacuum", `pg_table_seconds_since_last_autovacuum`, "db", "table"),
	qDB("pg_table_setting", `pg_table_setting`, "db", "table", "autovacuum_disabled", "autovacuum_vacuum_scale_factor", "autovacuum_vacuum_threshold", "autovacuum_vacuum_cost_delay", "autovacuum_vacuum_cost_limit", "autovacuum_analyze_scale_factor", "autovacuum_analyze_threshold"),
	qDB("pg_autovacuum_workers", `pg_autovacuum_workers`),
	qDB("pg_table_vacuum_in_progress", `pg_table_vacuum_in_progress`, "db", "table"),
	qDB("pg_table_vacuum_throttled", `pg_table_vacuum_throttled`, "db", "table"),
	qDB("pg_table_mods_since_analyze", `pg_table_mods_since_analyze`, "db", "table"),
	qDB("pg_table_reltuples", `pg_table_reltuples`, "db", "table"),
	qDB("pg_table_seconds_since_last_analyze", `pg_table_seconds_since_last_analyze`, "db", "table"),
	qDB("pg_checkpoints_scheduled_total", `pg_checkpoints_scheduled_total`, "type", "job"),
	qDB("pg_checkpoints_total", `pg_checkpoints_total`, "job"),
	qDB("pg_restartpoints_total", `pg_restartpoints_total`, "job"),
	qDB("pg_buffers_written_total", `rate(pg_buffers_written_total[$RANGE])`, "source"),
	qDB("pg_time_since_last_checkpoint_seconds", `pg_time_since_last_checkpoint_seconds`),
	qDB("pg_wal_since_last_checkpoint_bytes", `pg_wal_since_last_checkpoint_bytes`),

	qDB("mysql_up", `mysql_up`),
	qDB("mysql_scrape_error", `mysql_scrape_error`, "error", "warning"),
	qDB("mysql_info", `mysql_info`, "server_uuid", "server_version"),
	qDB("mysql_performance_schema_enabled", `mysql_performance_schema_enabled`),
	qDB("mysql_top_query_calls_per_second", `mysql_top_query_calls_per_second`, "schema", "query"),
	qDB("mysql_top_query_time_per_second", `mysql_top_query_time_per_second`, "schema", "query"),
	qDB("mysql_top_query_lock_time_per_second", `mysql_top_query_lock_time_per_second`, "schema", "query"),
	qDB("mysql_locked_queries", `mysql_locked_queries`, "schema", "query"),
	qDB("mysql_lock_awaiting_queries", `mysql_lock_awaiting_queries`, "schema", "blocking_query"),
	qDB("mysql_replication_io_status", `mysql_replication_io_status`, "source_server_uuid", "last_error", "state"),
	qDB("mysql_replication_sql_status", `mysql_replication_sql_status`, "source_server_uuid", "last_error", "state"),
	qDB("mysql_replication_lag_seconds", `mysql_replication_lag_seconds`, "source_server_uuid"),
	qDB("mysql_connections_max", `mysql_connections_max`),
	qDB("mysql_connections_current", `mysql_connections_current`),
	qDB("mysql_connections_total", `rate(mysql_connections_total[$RANGE])`),
	qDB("mysql_connections_aborted_total", `rate(mysql_connections_aborted_total[$RANGE])`),
	qDB("mysql_traffic_received_bytes_total", `rate(mysql_traffic_received_bytes_total[$RANGE])`),
	qDB("mysql_traffic_sent_bytes_total", `rate(mysql_traffic_sent_bytes_total[$RANGE])`),
	qDB("mysql_queries_total", `rate(mysql_queries_total[$RANGE])`),
	qDB("mysql_slow_queries_total", `rate(mysql_slow_queries_total[$RANGE])`),
	qDB("mysql_top_table_io_wait_time_per_second", `mysql_top_table_io_wait_time_per_second`, "schema", "table", "operation"),
	qDB("mysql_database_size_bytes", `mysql_database_size_bytes`, "db"),
	qDB("mysql_table_size_bytes", `mysql_table_size_bytes`, "db", "table"),
	qDB("mysql_table_size_growth_bytes_per_second", `mysql_table_size_growth_bytes_per_second`, "db", "table"),
	qDB("mysql_connection_errors_max_connections_total", `rate(mysql_connection_errors_max_connections_total[$RANGE])`),
	qDB("mysql_threads_running", `mysql_threads_running`),
	qDB("mysql_innodb_transaction_seconds", `mysql_innodb_transaction_seconds`, "query"),
	qDB("mysql_top_query_rows_examined_per_second", `mysql_top_query_rows_examined_per_second`, "schema", "query"),
	qDB("mysql_top_query_rows_sent_per_second", `mysql_top_query_rows_sent_per_second`, "schema", "query"),
	qDB("mysql_created_tmp_disk_tables_total", `rate(mysql_created_tmp_disk_tables_total[$RANGE])`),
	qDB("mysql_wsrep_cluster_size", `mysql_wsrep_cluster_size`),
	qDB("mysql_wsrep_cluster_status", `mysql_wsrep_cluster_status`, "status"),
	qDB("mysql_wsrep_local_state", `mysql_wsrep_local_state`, "state"),
	qDB("mysql_wsrep_ready", `mysql_wsrep_ready`),
	qDB("mysql_wsrep_connected", `mysql_wsrep_connected`),
	qDB("mysql_wsrep_flow_control_paused_seconds_total", `rate(mysql_wsrep_flow_control_paused_seconds_total[$RANGE])`),
	qDB("mysql_wsrep_local_recv_queue", `mysql_wsrep_local_recv_queue`),
	qDB("mysql_wsrep_local_send_queue", `mysql_wsrep_local_send_queue`),
	qDB("mysql_wsrep_local_cert_failures_total", `rate(mysql_wsrep_local_cert_failures_total[$RANGE])`),
	qDB("mysql_wsrep_local_bf_aborts_total", `rate(mysql_wsrep_local_bf_aborts_total[$RANGE])`),
	qDB("mysql_group_replication_cluster_size", `mysql_group_replication_cluster_size`),
	qDB("mysql_group_replication_members_online", `mysql_group_replication_members_online`),
	qDB("mysql_group_replication_member_state", `mysql_group_replication_member_state`, "state"),
	qDB("mysql_group_replication_transactions_in_queue", `mysql_group_replication_transactions_in_queue`),
	qDB("mysql_group_replication_transactions_remote_in_applier_queue", `mysql_group_replication_transactions_remote_in_applier_queue`),
	qDB("mysql_group_replication_conflicts_detected_total", `rate(mysql_group_replication_conflicts_detected_total[$RANGE])`),
	qDB("mysql_innodb_buffer_pool_read_requests_total", `rate(mysql_innodb_buffer_pool_read_requests_total[$RANGE])`),
	qDB("mysql_innodb_buffer_pool_reads_total", `rate(mysql_innodb_buffer_pool_reads_total[$RANGE])`),
	qDB("mysql_innodb_buffer_pool_write_requests_total", `rate(mysql_innodb_buffer_pool_write_requests_total[$RANGE])`),
	qDB("mysql_innodb_buffer_pool_pages_total", `mysql_innodb_buffer_pool_pages_total`),
	qDB("mysql_innodb_buffer_pool_pages_free", `mysql_innodb_buffer_pool_pages_free`),
	qDB("mysql_innodb_buffer_pool_pages_dirty", `mysql_innodb_buffer_pool_pages_dirty`),
	qDB("mysql_innodb_buffer_pool_pages_data", `mysql_innodb_buffer_pool_pages_data`),
	qDB("mysql_innodb_buffer_pool_wait_free_total", `rate(mysql_innodb_buffer_pool_wait_free_total[$RANGE])`),
	qDB("mysql_innodb_buffer_pool_pages_flushed_total", `rate(mysql_innodb_buffer_pool_pages_flushed_total[$RANGE])`),
	qDB("mysql_innodb_page_size_bytes", `mysql_innodb_page_size_bytes`),
	qDB("mysql_innodb_rows_read_total", `rate(mysql_innodb_rows_read_total[$RANGE])`),
	qDB("mysql_innodb_rows_inserted_total", `rate(mysql_innodb_rows_inserted_total[$RANGE])`),
	qDB("mysql_innodb_rows_updated_total", `rate(mysql_innodb_rows_updated_total[$RANGE])`),
	qDB("mysql_innodb_rows_deleted_total", `rate(mysql_innodb_rows_deleted_total[$RANGE])`),
	qDB("mysql_innodb_row_lock_waits_total", `rate(mysql_innodb_row_lock_waits_total[$RANGE])`),
	qDB("mysql_innodb_row_lock_time_seconds_total", `rate(mysql_innodb_row_lock_time_seconds_total[$RANGE])`),
	qDB("mysql_innodb_row_lock_current_waits", `mysql_innodb_row_lock_current_waits`),
	qDB("mysql_innodb_data_reads_total", `rate(mysql_innodb_data_reads_total[$RANGE])`),
	qDB("mysql_innodb_data_writes_total", `rate(mysql_innodb_data_writes_total[$RANGE])`),
	qDB("mysql_innodb_data_read_bytes_total", `rate(mysql_innodb_data_read_bytes_total[$RANGE])`),
	qDB("mysql_innodb_data_written_bytes_total", `rate(mysql_innodb_data_written_bytes_total[$RANGE])`),
	qDB("mysql_innodb_data_fsyncs_total", `rate(mysql_innodb_data_fsyncs_total[$RANGE])`),
	qDB("mysql_innodb_log_waits_total", `rate(mysql_innodb_log_waits_total[$RANGE])`),
	qDB("mysql_innodb_os_log_written_bytes_total", `rate(mysql_innodb_os_log_written_bytes_total[$RANGE])`),
	qDB("mysql_sort_merge_passes_total", `rate(mysql_sort_merge_passes_total[$RANGE])`),
	qDB("mysql_commit_total", `rate(mysql_commit_total[$RANGE])`),
	qDB("mysql_rollback_total", `rate(mysql_rollback_total[$RANGE])`),
	qDB("mysql_innodb_deadlocks_total", `rate(mysql_innodb_deadlocks_total[$RANGE])`),
	qDB("mysql_innodb_lock_wait_timeouts_total", `rate(mysql_innodb_lock_wait_timeouts_total[$RANGE])`),
	qDB("mysql_innodb_history_list_length", `mysql_innodb_history_list_length`),
	qDB("mysql_binlog_size_bytes", `mysql_binlog_size_bytes`),
	qDB("mysql_binlog_files", `mysql_binlog_files`),
	qDB("mysql_binlog_expire_seconds", `mysql_binlog_expire_seconds`),
	qDB("mysql_undo_size_bytes", `mysql_undo_size_bytes`),
	qDB("mysql_table_locks_waited_total", `rate(mysql_table_locks_waited_total[$RANGE])`),
	qDB("mysql_table_locks_immediate_total", `rate(mysql_table_locks_immediate_total[$RANGE])`),

	qDB("redis_up", `redis_up`),
	qDB("redis_scrape_error", `redis_exporter_last_scrape_error`, "err"),
	qDB("redis_instance_info", `redis_instance_info`, "redis_version", "role"),
	qDB("redis_commands_duration_seconds_total", `rate(redis_commands_duration_seconds_total[$RANGE])`, "cmd"),
	qDB("redis_commands_total", `rate(redis_commands_total[$RANGE])`, "cmd"),
	qDB("redis_db_keys", `redis_db_keys`, "db"),
	qDB("redis_db_keys_expiring", `redis_db_keys_expiring`, "db"),

	qDB("mongo_up", `mongo_up`),
	qDB("mongo_scrape_error", `mongo_scrape_error`, "error", "warning"),
	qDB("mongo_info", `mongo_info`, "server_version", "flavor"),
	qDB("mongo_rs_status", `mongo_rs_status`, "rs", "role"),
	qDB("mongo_rs_last_applied_timestamp_ms", `timestamp(mongo_rs_last_applied_timestamp_ms) - mongo_rs_last_applied_timestamp_ms/1000`),
	qDB("mongo_rs_member_status", `mongo_rs_member_status`, "rs", "member", "state"),
	qDB("mongo_rs_member_replication_lag_seconds", `mongo_rs_member_replication_lag_seconds`, "rs", "member"),
	qDB("mongo_rs_member_config_info", `mongo_rs_member_config_info`, "rs", "member", "arbiter", "votes"),
	qDB("mongo_rs_config_info", `mongo_rs_config_info`, "rs", "write_concern_majority_journal_default"),
	qDB("mongo_profiling_level", `mongo_profiling_level`, "db"),
	qDB("mongo_oplog_window_seconds", `mongo_oplog_window_seconds`),
	qDB("mongo_oplog_size_bytes", `mongo_oplog_size_bytes`),
	qDB("mongo_oplog_max_size_bytes", `mongo_oplog_max_size_bytes`),
	qDB("mongo_connections_current", `mongo_connections_current`),
	qDB("mongo_connections_active", `mongo_connections_active`),
	qDB("mongo_connections_max", `mongo_connections_max`),
	qDB("mongo_connections_created_total", `rate(mongo_connections_created_total[$RANGE])`),
	qDB("mongo_connections_rejected_total", `rate(mongo_connections_rejected_total[$RANGE])`),
	qDB("mongo_connections_by_app", `mongo_connections_by_app`, "app"),
	qDB("mongo_open_transactions", `mongo_open_transactions`, "app"),
	qDB("mongo_opcounters_total", `rate(mongo_opcounters_total[$RANGE])`, "op"),
	qDB("mongo_documents_returned_total", `rate(mongo_documents_returned_total[$RANGE])`),
	qDB("mongo_op_latency_seconds_total", `rate(mongo_op_latency_seconds_total[$RANGE])`, "type"),
	qDB("mongo_op_latency_ops_total", `rate(mongo_op_latency_ops_total[$RANGE])`, "type"),
	qDB("mongo_queued_operations", `mongo_queued_operations`, "type"),
	qDB("mongo_wt_tickets_available", `mongo_wt_tickets_available`, "type"),
	qDB("mongo_wt_cache_used_bytes", `mongo_wt_cache_used_bytes`),
	qDB("mongo_wt_cache_dirty_bytes", `mongo_wt_cache_dirty_bytes`),
	qDB("mongo_wt_cache_max_bytes", `mongo_wt_cache_max_bytes`),
	qDB("mongo_wt_pages_evicted_by_app_threads_total", `rate(mongo_wt_pages_evicted_by_app_threads_total[$RANGE])`),
	qDB("mongo_wt_app_threads_evicting_seconds_total", `rate(mongo_wt_app_threads_evicting_seconds_total[$RANGE])`),
	qDB("mongo_wt_checkpoints_total", `rate(mongo_wt_checkpoints_total[$RANGE])`),
	qDB("mongo_wt_checkpoint_seconds_total", `rate(mongo_wt_checkpoint_seconds_total[$RANGE])`),
	qDB("mongo_wt_journal_bytes_written_total", `rate(mongo_wt_journal_bytes_written_total[$RANGE])`),
	qDB("mongo_wt_journal_bytes_since_checkpoint", `mongo_wt_journal_bytes_since_checkpoint`),
	qDB("mongo_time_since_last_checkpoint_seconds", `mongo_time_since_last_checkpoint_seconds`),
	qDB("mongo_scanned_keys_total", `rate(mongo_scanned_keys_total[$RANGE])`),
	qDB("mongo_scanned_documents_total", `rate(mongo_scanned_documents_total[$RANGE])`),
	qDB("mongo_collection_scans_total", `rate(mongo_collection_scans_total[$RANGE])`),
	qDB("mongo_scan_and_order_total", `rate(mongo_scan_and_order_total[$RANGE])`),
	qDB("mongo_ttl_deleted_documents_total", `rate(mongo_ttl_deleted_documents_total[$RANGE])`),
	qDB("mongo_flow_control_time_acquiring_seconds_total", `rate(mongo_flow_control_time_acquiring_seconds_total[$RANGE])`),
	qDB("mongo_write_conflicts_total", `rate(mongo_write_conflicts_total[$RANGE])`),
	qDB("mongo_wt_cache_bytes_read_into_total", `rate(mongo_wt_cache_bytes_read_into_total[$RANGE])`),
	qDB("mongo_repl_apply_ops_total", `rate(mongo_repl_apply_ops_total[$RANGE])`),
	qDB("mongo_repl_buffer_operations", `mongo_repl_buffer_operations`),
	qDB("mongo_repl_buffer_bytes", `mongo_repl_buffer_bytes`),
	qDB("mongo_cursors_open", `mongo_cursors_open`),
	qDB("mongo_cursors_open_no_timeout", `mongo_cursors_open_no_timeout`),
	qDB("mongo_cursors_timed_out_total", `rate(mongo_cursors_timed_out_total[$RANGE])`),
	qDB("mongo_fsync_locked", `mongo_fsync_locked`),
	qDB("mongo_prepared_transactions", `mongo_prepared_transactions`),
	qDB("mongo_top_query_calls_per_second", `mongo_top_query_calls_per_second`, "db", "collection", "query"),
	qDB("mongo_top_query_time_per_second", `mongo_top_query_time_per_second`, "db", "collection", "query"),
	qDB("mongo_top_query_docs_returned_per_second", `mongo_top_query_docs_returned_per_second`, "db", "collection", "query"),
	qDB("mongo_top_query_docs_examined_per_second", `mongo_top_query_docs_examined_per_second`, "db", "collection", "query"),
	qDB("mongo_top_query_keys_examined_per_second", `mongo_top_query_keys_examined_per_second`, "db", "collection", "query"),
	qDB("mongo_operations_waiting_for_lock", `mongo_operations_waiting_for_lock`, "db"),
	qDB("mongo_long_running_operations", `mongo_long_running_operations`, "db", "collection", "query", "plan"),
	qDB("mongo_database_size_bytes", `mongo_database_size_bytes`, "db"),
	qDB("mongo_collection_size_bytes", `mongo_collection_size_bytes`, "db", "collection"),
	qDB("mongo_collection_storage_size_bytes", `mongo_collection_storage_size_bytes`, "db", "collection"),
	qDB("mongo_collection_size_growth_bytes_per_second", `mongo_collection_size_growth_bytes_per_second`, "db", "collection"),
	qDB("mongo_collection_free_storage_bytes", `mongo_collection_free_storage_bytes`, "db", "collection"),
	qDB("mongo_collection_documents", `mongo_collection_documents`, "db", "collection"),

	qDB("memcached_up", `memcached_up`),
	qDB("memcached_version", `memcached_version`, "version"),
	qDB("memcached_limit_bytes", `memcached_limit_bytes`),
	qDB("memcached_items_evicted_total", `rate(memcached_items_evicted_total[$RANGE])`),
	qDB("memcached_commands_total", `rate(memcached_commands_total[$RANGE])`, "command", "status"),

	qJVM("container_jvm_info", `container_jvm_info`, "java_version"),
	qJVM("container_jvm_heap_used_bytes", `container_jvm_heap_used_bytes`),
	qJVM("container_jvm_gc_time_seconds", `rate(container_jvm_gc_time_seconds[$RANGE])`, "gc"),
	qJVM("container_jvm_safepoint_time_seconds", `rate(container_jvm_safepoint_time_seconds[$RANGE])`),
	qJVM("container_jvm_safepoint_sync_time_seconds", `rate(container_jvm_safepoint_sync_time_seconds[$RANGE])`),
	qJVM("container_jvm_heap_max_size_bytes", `container_jvm_heap_max_size_bytes`),
	qJVM("container_jvm_alloc_bytes_total", `rate(container_jvm_alloc_bytes_total[$RANGE])`).WithFillFunc(timeseries.FillAvg),
	qJVM("container_jvm_alloc_objects_total", `rate(container_jvm_alloc_objects_total[$RANGE])`).WithFillFunc(timeseries.FillAvg),
	qJVM("container_jvm_lock_contentions_total", `rate(container_jvm_lock_contentions_total[$RANGE])`).WithFillFunc(timeseries.FillAvg),
	qJVM("container_jvm_lock_time_seconds_total", `rate(container_jvm_lock_time_seconds_total[$RANGE])`).WithFillFunc(timeseries.FillAvg),
	qJVM("container_jvm_profiling_status", `container_jvm_profiling_status`),

	Q("container_go_alloc_bytes_total", `rate(container_go_alloc_bytes_total[$RANGE])`).WithFillFunc(timeseries.FillAvg),
	Q("container_go_alloc_objects_total", `rate(container_go_alloc_objects_total[$RANGE])`).WithFillFunc(timeseries.FillAvg),

	qDotNet("container_dotnet_info", `container_dotnet_info`, "runtime_version"),
	qDotNet("container_dotnet_memory_allocated_bytes_total", `rate(container_dotnet_memory_allocated_bytes_total[$RANGE])`),
	qDotNet("container_dotnet_exceptions_total", `rate(container_dotnet_exceptions_total[$RANGE])`),
	qDotNet("container_dotnet_memory_heap_size_bytes", `container_dotnet_memory_heap_size_bytes`, "generation"),
	qDotNet("container_dotnet_gc_count_total", `rate(container_dotnet_gc_count_total[$RANGE])`, "generation"),
	qDotNet("container_dotnet_heap_fragmentation_percent", `container_dotnet_heap_fragmentation_percent`),
	qDotNet("container_dotnet_monitor_lock_contentions_total", `rate(container_dotnet_monitor_lock_contentions_total[$RANGE])`),
	qDotNet("container_dotnet_thread_pool_completed_items_total", `rate(container_dotnet_thread_pool_completed_items_total[$RANGE])`),
	qDotNet("container_dotnet_thread_pool_queue_length", `container_dotnet_thread_pool_queue_length`),
	qDotNet("container_dotnet_thread_pool_size", `container_dotnet_thread_pool_size`),

	Q("container_python_thread_lock_wait_time_seconds", `rate(container_python_thread_lock_wait_time_seconds[$RANGE])`),
	Q("container_nodejs_event_loop_blocked_time_seconds", `rate(container_nodejs_event_loop_blocked_time_seconds_total[$RANGE])`),

	qPod("fluxcd_git_repository_info", `fluxcd_git_repository_info`, "name", "namespace", "suspended", "url", "interval"),
	qPod("fluxcd_git_repository_status", `fluxcd_git_repository_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_oci_repository_info", `fluxcd_oci_repository_info`, "name", "namespace", "suspended", "url", "interval"),
	qPod("fluxcd_oci_repository_status", `fluxcd_oci_repository_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_helm_repository_info", `fluxcd_helm_repository_info`, "name", "namespace", "suspended", "url", "interval"),
	qPod("fluxcd_helm_repository_status", `fluxcd_helm_repository_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_helm_chart_info", `fluxcd_helm_chart_info`, "name", "namespace", "suspended", "chart", "interval", "version", "source_kind", "source_name", "source_namespace"),
	qPod("fluxcd_helm_chart_status", `fluxcd_helm_chart_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_helm_release_info", `fluxcd_helm_release_info`, "name", "namespace", "suspended", "chart", "interval", "version", "source_kind", "source_name", "source_namespace", "chart_ref_kind", "chart_ref_name", "chart_ref_namespace", "target_namespace"),
	qPod("fluxcd_helm_release_status", `fluxcd_helm_release_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_kustomization_info", `fluxcd_kustomization_info`, "name", "namespace", "suspended", "interval", "path", "source_kind", "source_name", "source_namespace", "target_namespace", "last_applied_revision", "last_attempted_revision"),
	qPod("fluxcd_kustomization_status", `fluxcd_kustomization_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_kustomization_dependency_info", `fluxcd_kustomization_dependency_info`, "name", "namespace", "depends_on_name", "depends_on_namespace"),
	qPod("fluxcd_kustomization_inventory_entry_info", `fluxcd_kustomization_inventory_entry_info`, "name", "namespace", "entry_id"),
	qPod("fluxcd_resourceset_info", `fluxcd_resourceset_info`, "name", "namespace", "last_applied_revision"),
	qPod("fluxcd_resourceset_status", `fluxcd_resourceset_status`, "name", "namespace", "type", "reason"),
	qPod("fluxcd_resourceset_dependency_info", `fluxcd_resourceset_dependency_info`, "name", "namespace", "depends_on_name", "depends_on_namespace", "depends_on_kind"),
	qPod("fluxcd_resourceset_inventory_entry_info", `fluxcd_resourceset_inventory_entry_info`, "name", "namespace", "entry_id"),

	qPod("argocd_application_info", `argocd_application_info`, "name", "namespace", "project", "source_type", "repo", "path", "chart", "dest_server", "dest_name", "dest_namespace"),
	qPod("argocd_application_sync_status", `argocd_application_sync_status`, "name", "namespace", "sync_status"),
	qPod("argocd_application_health_status", `argocd_application_health_status`, "name", "namespace", "health_status"),
	qPod("argocd_application_operation_status", `argocd_application_operation_status`, "name", "namespace", "operation_phase"),
	qPod("argocd_application_operation_finished_timestamp_seconds", `argocd_application_operation_finished_timestamp_seconds`, "name", "namespace"),
	qPod("argocd_application_resource_info", `argocd_application_resource_info`, "name", "namespace", "resource_group", "resource_kind", "resource_namespace", "resource_name"),
	qPod("argocd_application_resource_sync_status", `argocd_application_resource_sync_status`, "name", "namespace", "resource_group", "resource_kind", "resource_namespace", "resource_name", "sync_status"),
	qPod("argocd_application_resource_status", `argocd_application_resource_status`, "name", "namespace", "resource_group", "resource_kind", "resource_namespace", "resource_name", "status"),
	qPod("argocd_application_resource_health_status", `argocd_application_resource_health_status`, "name", "namespace", "resource_group", "resource_kind", "resource_namespace", "resource_name", "health_status"),

	qPod("pg_backup_target_info", `pg_backup_target_info`, "name", "namespace", "operator", "method", "path", "endpoint", "s3_bucket", "s3_endpoint", "gcs_bucket", "azure_container", "schedule", "retention_policy"),
	qPod("pg_cluster_status", `pg_cluster_status`, "name", "namespace", "operator", "type", "status", "reason"),
	qPod("pg_backup_last_successful_timestamp_seconds", `pg_backup_last_successful_timestamp_seconds`, "name", "namespace", "operator", "method"),
	qPod("pg_backup_first_recoverability_point_timestamp_seconds", `pg_backup_first_recoverability_point_timestamp_seconds`, "name", "namespace", "operator", "method"),
	qPod("pg_backup_last_failed_timestamp_seconds", `pg_backup_last_failed_timestamp_seconds`, "name", "namespace", "operator"),
	qPod("pg_backup_schedule_info", `pg_backup_schedule_info`, "name", "namespace", "operator", "cluster", "schedule"),
	qPod("pg_backup_next_scheduled_timestamp_seconds", `pg_backup_next_scheduled_timestamp_seconds`, "name", "namespace", "operator", "cluster"),
	qPod("pg_backup_info", `pg_backup_info`, "name", "namespace", "operator", "cluster", "method", "kind", "path"),
	qPod("pg_backup_status", `pg_backup_status`, "name", "namespace", "operator", "status"),
	qPod("pg_backup_completed_timestamp_seconds", `pg_backup_completed_timestamp_seconds`, "name", "namespace", "operator", "cluster"),

	qPod("mongo_backup_target_info", `mongo_backup_target_info`, "name", "namespace", "operator", "method", "s3_bucket", "s3_endpoint", "s3_prefix", "azure_container"),
	qPod("mongo_backup_schedule_info", `mongo_backup_schedule_info`, "name", "namespace", "operator", "task", "schedule", "method", "kind", "enabled"),
	qPod("mongo_backup_pitr_info", `mongo_backup_pitr_info`, "name", "namespace", "operator", "enabled"),
	qPod("mongo_cluster_status", `mongo_cluster_status`, "name", "namespace", "operator", "status"),
	qPod("mongo_backup_info", `mongo_backup_info`, "name", "namespace", "operator", "cluster", "method", "kind", "path"),
	qPod("mongo_backup_status", `mongo_backup_status`, "name", "namespace", "operator", "status"),
	qPod("mongo_backup_completed_timestamp_seconds", `mongo_backup_completed_timestamp_seconds`, "name", "namespace", "operator"),

	qPod("mysql_backup_target_info", `mysql_backup_target_info`, "name", "namespace", "operator", "method", "s3_bucket", "s3_endpoint", "s3_prefix", "azure_container"),
	qPod("mysql_backup_schedule_info", `mysql_backup_schedule_info`, "name", "namespace", "operator", "task", "schedule", "method"),
	qPod("mysql_backup_pitr_info", `mysql_backup_pitr_info`, "name", "namespace", "operator", "enabled"),
	qPod("mysql_cluster_status", `mysql_cluster_status`, "name", "namespace", "operator", "status"),
	qPod("mysql_backup_info", `mysql_backup_info`, "name", "namespace", "operator", "cluster", "method", "kind", "path"),
	qPod("mysql_backup_status", `mysql_backup_status`, "name", "namespace", "operator", "status"),
	qPod("mysql_backup_completed_timestamp_seconds", `mysql_backup_completed_timestamp_seconds`, "name", "namespace", "operator"),
}

var RecordingRules = map[string]func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues{
	qRecordingRuleApplicationLogMessages: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		for _, app := range w.Applications {
			appId := app.Id.String()
			for severity, msgs := range app.LogMessages {
				if len(msgs.Patterns) == 0 {
					if msgs.Messages.Reduce(timeseries.NanSum) > 0 {
						ls := model.Labels{"application": appId, "level": severity.String()}
						res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: msgs.Messages})
					}
				} else {
					for _, pattern := range msgs.Patterns {
						if pattern.Messages.Reduce(timeseries.NanSum) > 0 {
							ls := model.Labels{"application": appId, "level": severity.String()}
							ls["multiline"] = strconv.FormatBool(pattern.Multiline)
							ls["similar"] = strings.Join(pattern.SimilarPatternHashes.Items(), " ")
							ls["sample"] = pattern.Sample
							ls["words"] = pattern.Pattern.String()
							res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: pattern.Messages})
						}
					}
				}
			}
		}
		return res
	},
	qRecordingRuleApplicationTCPSuccessful: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.SuccessfulConnections })
	},
	qRecordingRuleApplicationTCPActive: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.Active })
	},
	qRecordingRuleApplicationTCPFailed: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.FailedConnections })
	},
	qRecordingRuleApplicationTCPConnectionTime: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.ConnectionTime })
	},
	qRecordingRuleApplicationTCPBytesSent: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.BytesSent })
	},
	qRecordingRuleApplicationTCPBytesReceived: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.BytesReceived })
	},
	qRecordingRuleApplicationTCPRetransmissions: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		return aggConnections(w, func(c *model.Connection) *timeseries.TimeSeries { return c.Retransmissions })
	},

	qRecordingRuleApplicationNetLatency: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues

		for _, app := range w.Applications {
			byDest := map[*model.Application][]*timeseries.TimeSeries{}
			for _, instance := range app.Instances {
				for _, u := range instance.Upstreams {
					dest := u.RemoteApplication()
					if dest == nil {
						continue
					}
					if !u.Rtt.IsEmpty() {
						byDest[dest] = append(byDest[dest], u.Rtt)
					}
				}
			}
			appId := app.Id.String()
			for dest, rtts := range byDest {
				sum := timeseries.NewAggregate(timeseries.NanSum).Add(rtts...).Get()
				count := timeseries.NewAggregate(timeseries.NanSum)
				for _, rtt := range rtts {
					count.Add(rtt.Map(timeseries.Defined))
				}
				ls := model.Labels{"app": appId, "dest": dest.Id.String(), "agg": "avg"}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: timeseries.Div(sum, count.Get())})
			}
		}
		return res
	},

	qRecordingRuleApplicationL7Requests: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		type key struct {
			status   string
			protocol model.Protocol
			dest     *model.Application
		}

		for _, app := range w.Applications {
			byProtoStatus := map[key]*timeseries.Aggregate{}
			for _, instance := range app.Instances {
				for _, u := range instance.Upstreams {
					dest := u.RemoteApplication()
					if dest == nil {
						continue
					}
					for proto, byStatus := range u.RequestsCount {
						for status, ts := range byStatus {
							bucket := "ok"
							if model.IsRequestStatusFailed(status) {
								bucket = "failed"
							}
							k := key{dest: dest, status: bucket, protocol: proto}
							agg := byProtoStatus[k]
							if agg == nil {
								agg = timeseries.NewAggregate(timeseries.NanSum)
								byProtoStatus[k] = agg
							}
							agg.Add(ts)

						}
					}
				}
			}
			appId := app.Id.String()
			for k, agg := range byProtoStatus {
				ts := agg.Get()
				if !ts.IsEmpty() {
					ls := model.Labels{"app": appId, "dest": k.dest.Id.String(), "proto": string(k.protocol), "status": k.status}
					res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
				}
			}
		}
		return res
	},

	qRecordingRuleApplicationL7Latency: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		type key struct {
			protocol model.Protocol
			dest     *model.Application
		}

		for _, app := range w.Applications {
			byProto := map[key]*timeseries.Aggregate{}
			for _, instance := range app.Instances {
				for _, u := range instance.Upstreams {
					dest := u.RemoteApplication()
					if dest == nil {
						continue
					}
					for proto, ts := range u.RequestsLatency {
						k := key{dest: dest, protocol: proto}
						agg := byProto[k]
						if agg == nil {
							agg = timeseries.NewAggregate(timeseries.NanSum)
							byProto[k] = agg
						}
						agg.Add(ts)

					}
				}
			}
			appId := app.Id.String()
			for k, agg := range byProto {
				ts := agg.Get()
				if !ts.IsEmpty() {
					ls := model.Labels{"app": appId, "dest": k.dest.Id.String(), "proto": string(k.protocol)}
					res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
				}
			}
		}
		return res
	},

	qRecordingRuleApplicationL7Histogram: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		type key struct {
			le   float32
			dest *model.Application
		}
		sum := map[key]*timeseries.Aggregate{}
		for _, app := range w.Applications {
			for _, instance := range app.Instances {
				for _, u := range instance.Upstreams {
					dest := u.RemoteApplication()
					if dest == nil {
						continue
					}
					if !dest.Category.Auxiliary() && app.Category.Auxiliary() {
						continue
					}
					for _, byLe := range u.RequestsHistogram {
						for le, ts := range byLe {
							k := key{dest: dest, le: le}
							agg := sum[k]
							if agg == nil {
								agg = timeseries.NewAggregate(timeseries.NanSum)
								sum[k] = agg
							}
							agg.Add(ts)
						}
					}
				}
			}
		}
		for k, agg := range sum {
			ts := agg.Get()
			if !ts.IsEmpty() {
				ls := model.Labels{"app": k.dest.Id.String(), "le": fmt.Sprintf("%f", k.le)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
		return res
	},

	qRecordingRuleApplicationL7InboundRequests: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		for _, app := range w.Applications {
			appId := app.Id.String()
			byBucket := map[string]*timeseries.Aggregate{}
			for status, ts := range app.InboundRequestsCount {
				if ts.IsEmpty() {
					continue
				}
				bucket := "ok"
				if model.IsRequestStatusFailed(status) {
					bucket = "failed"
				}
				agg := byBucket[bucket]
				if agg == nil {
					agg = timeseries.NewAggregate(timeseries.NanSum)
					byBucket[bucket] = agg
				}
				agg.Add(ts)
			}
			for bucket, agg := range byBucket {
				ts := agg.Get()
				if ts.IsEmpty() {
					continue
				}
				ls := model.Labels{"app": appId, "status": bucket}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
		return res
	},

	qRecordingRuleApplicationL7InboundHistogram: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		for _, app := range w.Applications {
			appId := app.Id.String()
			for le, ts := range app.InboundRequestsHistogram {
				if ts.IsEmpty() {
					continue
				}
				ls := model.Labels{"app": appId, "le": fmt.Sprintf("%f", le)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
		return res
	},

	qRecordingRuleApplicationLLMInboundHistogram: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		for _, app := range w.Applications {
			appId := app.Id.String()
			for le, ts := range app.InboundLLMRequestsHistogram {
				if ts.IsEmpty() {
					continue
				}
				ls := model.Labels{"app": appId, "le": fmt.Sprintf("%f", le)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
		return res
	},

	qRecordingRuleApplicationDNSRequests: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		for _, app := range w.Applications {
			appId := app.Id.String()
			for r, byStatus := range app.DNSRequests {
				for status, ts := range byStatus {
					if ts.IsEmpty() {
						continue
					}
					ls := model.Labels{"app": appId, "request_type": r.Type, "domain": r.Domain, "status": status}
					res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
				}
			}
		}
		return res
	},

	qRecordingRuleApplicationDNSLatency: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		for _, app := range w.Applications {
			appId := app.Id.String()
			for le, ts := range app.DNSRequestsHistogram {
				if ts.IsEmpty() {
					continue
				}
				ls := model.Labels{"app": appId, "le": fmt.Sprintf("%f", le)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
		return res
	},

	qRecordingRuleApplicationTraffic: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues

		for _, app := range w.Applications {
			appId := app.Id.String()
			if ts := app.TrafficStats.InternetEgress.Get(); !ts.IsEmpty() {
				ls := model.Labels{"app": appId, "kind": string(model.TrafficKindInternetEgress)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
			if ts := app.TrafficStats.CrossAZEgress.Get(); !ts.IsEmpty() {
				ls := model.Labels{"app": appId, "kind": string(model.TrafficKindCrossAZEgress)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
			if ts := app.TrafficStats.CrossAZIngress.Get(); !ts.IsEmpty() {
				ls := model.Labels{"app": appId, "kind": string(model.TrafficKindCrossAZIngress)}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
		return res
	},

	qRecordingRuleApplicationCategories: func(database *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var needSave bool
		for _, app := range w.Applications {
			if _, ok := p.Settings.ApplicationCategorySettings[app.Category]; !ok {
				if p.Settings.ApplicationCategorySettings == nil {
					p.Settings.ApplicationCategorySettings = map[model.ApplicationCategory]*db.ApplicationCategorySettings{}
				}
				p.Settings.ApplicationCategorySettings[app.Category] = nil
				needSave = true
			}
		}
		if needSave {
			if err := database.SaveProjectSettings(p); err != nil {
				klog.Errorln("failed to save project settings:", err)
			}
		}
		return nil
	},

	qRecordingRuleApplicationSLO: func(database *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		for _, app := range w.Applications {
			updateAvailabilitySLOFromAnnotations(database, p, w, app)
			latencyCfg, _ := w.CheckConfigs.GetLatency(app.Id, app.Category)
			updateLatencySLOFromAnnotations(database, p, app, model.Checks.SLOLatency, latencyCfg, model.ApplicationAnnotationSLOLatencyObjective, model.ApplicationAnnotationSLOLatencyThreshold, model.DefaultHistogramBuckets)
			llmLatencyCfg, _ := w.CheckConfigs.GetLLMLatency(app.Id)
			updateLatencySLOFromAnnotations(database, p, app, model.Checks.SLOLLMLatency, llmLatencyCfg, model.ApplicationAnnotationSLOLLMLatencyObjective, model.ApplicationAnnotationSLOLLMLatencyThreshold, model.LLMHistogramBuckets)
		}
		return nil
	},
	qRecordingRuleApplicationExternalEndpoint: func(db *db.DB, p *db.Project, w *model.World) []*model.MetricValues {
		var res []*model.MetricValues
		if w.Ctx.PointsCount() < 1 {
			return res
		}
		values := make([]float32, w.Ctx.PointsCount())
		for i := range values {
			values[i] = 1
		}
		ts := timeseries.NewWithData(w.Ctx.From, w.Ctx.Step, values)
		for _, app := range w.Applications {
			byDest := map[model.ApplicationId]*utils.StringSet{}
			for _, instance := range app.Instances {
				for _, u := range instance.Upstreams {
					dest := u.RemoteApplication()
					if dest == nil || dest.Id.Kind != model.ApplicationKindExternalService {
						continue
					}

					endpoints := byDest[dest.Id]
					if endpoints == nil {
						endpoints = utils.NewStringSet()
						byDest[dest.Id] = endpoints
					}
					if u.ActualRemoteIP != "" {
						endpoints.Add(net.JoinHostPort(u.ActualRemoteIP, u.ActualRemotePort))
					} else if u.ServiceRemoteIP != "" { // failed connections have no actual destination
						endpoints.Add(net.JoinHostPort(u.ServiceRemoteIP, u.ServiceRemotePort))
					}
				}
			}
			appId := app.Id.String()
			for destId, endpoints := range byDest {
				for _, ep := range endpoints.Items() {
					ls := model.Labels{"app": appId, "dest": destId.String(), model.LabelActualDestination: ep}
					res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
				}
			}
		}
		return res
	},
}

func aggConnections(w *model.World, tsF func(c *model.Connection) *timeseries.TimeSeries) []*model.MetricValues {
	var res []*model.MetricValues
	for _, app := range w.Applications {
		byDest := map[*model.Application]*timeseries.Aggregate{}
		for _, instance := range app.Instances {
			for _, u := range instance.Upstreams {
				dest := u.RemoteApplication()
				if dest == nil {
					continue
				}
				agg := byDest[dest]
				if agg == nil {
					agg = timeseries.NewAggregate(timeseries.NanSum)
					byDest[dest] = agg
				}
				agg.Add(tsF(u))
			}
		}
		appId := app.Id.String()
		for dest, agg := range byDest {
			ts := agg.Get()
			if !ts.IsEmpty() {
				ls := model.Labels{"app": appId, "dest": dest.Id.String()}
				res = append(res, &model.MetricValues{Labels: ls, LabelsHash: promModel.LabelsToSignature(ls), Values: ts})
			}
		}
	}
	return res
}

func updateAvailabilitySLOFromAnnotations(database *db.DB, p *db.Project, w *model.World, app *model.Application) {
	objectiveStr := app.GetAnnotation(model.ApplicationAnnotationSLOAvailabilityObjective)
	cfg, _ := w.CheckConfigs.GetAvailability(app.Id)
	if objectiveStr == "" {
		return
	}
	cfgSaved := cfg
	cfg.Source = model.CheckConfigSourceKubernetesAnnotations
	cfg.Custom = false
	cfg.Error = ""
	objective, err := parseObjective(objectiveStr)
	if err != nil {
		cfg.Error = fmt.Sprintf("Invalid annotation 'coroot.com/slo-availability-objective': %s", err)
	}
	if cfg.Error != "" {
		cfg.ObjectivePercentage = 0 // disable
	} else {
		cfg.ObjectivePercentage = objective
	}
	if cfg == cfgSaved {
		return
	}
	if err = database.SaveCheckConfig(p.Id, app.Id, model.Checks.SLOAvailability.Id, []model.CheckConfigSLOAvailability{cfg}); err != nil {
		klog.Errorln(err)
	}
}

func updateLatencySLOFromAnnotations(database *db.DB, p *db.Project, app *model.Application, check model.CheckConfig, cfg model.CheckConfigSLOLatency,
	objectiveAnnotation, thresholdAnnotation model.ApplicationAnnotation, buckets []float32) {
	objectiveStr := app.GetAnnotation(objectiveAnnotation)
	thresholdStr := app.GetAnnotation(thresholdAnnotation)
	if objectiveStr == "" && thresholdStr == "" {
		return
	}
	cfgSaved := cfg
	cfg.Source = model.CheckConfigSourceKubernetesAnnotations
	cfg.Custom = false
	cfg.Error = ""
	var err error
	objective := cfg.ObjectivePercentage
	threshold := cfg.ObjectiveBucket
	if objectiveStr != "" {
		objective, err = parseObjective(objectiveStr)
		if err != nil {
			cfg.Error = fmt.Sprintf("Invalid annotation 'coroot.com/%s': %s", strings.ReplaceAll(string(objectiveAnnotation), "_", "-"), err)
		}
	}
	if objective > 0 && thresholdStr != "" {
		threshold, err = parseThreshold(thresholdStr, buckets)
		if err != nil && cfg.Error == "" {
			cfg.Error = fmt.Sprintf("Invalid annotation 'coroot.com/%s': %s", strings.ReplaceAll(string(thresholdAnnotation), "_", "-"), err)
		}
	}
	if cfg.Error != "" {
		cfg.ObjectivePercentage = 0 // disable
	} else {
		cfg.ObjectivePercentage = objective
		cfg.ObjectiveBucket = threshold
	}
	if cfg == cfgSaved {
		return
	}
	if err = database.SaveCheckConfig(p.Id, app.Id, check.Id, []model.CheckConfigSLOLatency{cfg}); err != nil {
		klog.Errorln(err)
	}
}

func parseObjective(s string) (float32, error) {
	s = strings.TrimSpace(strings.TrimRight(strings.TrimSpace(s), "%"))
	v, err := strconv.ParseFloat(s, 32)
	return float32(v), err
}

func parseThreshold(s string, buckets []float32) (float32, error) {
	d, err := time.ParseDuration(strings.TrimSpace(s))
	if err != nil {
		return 0, err
	}
	v := model.RoundUpToBucket(float32(d.Seconds()), buckets)
	return v, err
}