feat: aggregate and regex-match series in check_http_metrics

pkg/tasks/check_http_metrics/README.md

@@ -48,7 +48,9 @@ The `check_http_metrics` task fetches metrics from an HTTP Prometheus endpoint a
 
 - **`name`**: Unique assertion name. Required.
 - **`metric`**: Prometheus metric name. Required.
-- **`labels`**: Label selector (subset matching). Must match exactly one series.
+- **`labels`**: Label selector (subset matching, exact values).
+- **`labelPatterns`**: Label selector with regex values, each matching the whole label value (e.g. `status: "[45].."`). Combined with `labels`.
+- **`aggregate`**: How matching series are combined. `single` (default): the selector must match exactly one series. `sum`: the sum of all matching series is compared. `each`: every matching series must pass on its own, failures name the offending series.
 - **`mode`**: `value` (current value) or `delta` (change since baseline). Default: `value`.
 - **`operator`**: Comparison operator: `eq`, `neq`, `gt`, `gte`, `lt`, `lte`. Required.
 - **`value`**: Expected numeric value. Required.
@@ -63,10 +65,32 @@ In `delta` mode, the task tracks changes over time:
 
 Negative deltas are valid for GAUGE and UNTYPED metrics. For COUNTER metrics, a decrease triggers `resetBehavior`.
 
+With `aggregate: sum` or `each`, baselines are kept per series. The first scrape records every matching series (even if there are none yet), and a series that first appears later counts from zero, so an error counter created during the run is still caught.
+
 #### Examples
 
 ```yaml
 assertions:
+  # Fail on any new 4xx/5xx REST response, naming each route and status that occurred
+  - name: no_rest_errors
+    metric: lodestar_api_rest_errors_total
+    aggregate: each
+    mode: delta
+    operator: eq
+    value: 0
+    missingMetric: pass
+    missingSeries: pass
+
+  # Sum all error responses across routes
+  - name: few_errors
+    metric: ngx_request_count
+    labelPatterns:
+      status: "[45].."
+    aggregate: sum
+    mode: delta
+    operator: lte
+    value: 5
+
   # Check counter increased by at least 1
   - name: counter_increased
     metric: my_counter

pkg/tasks/check_http_metrics/config.go

@@ -3,6 +3,7 @@ package checkhttpmetrics
 import (
 	"fmt"
 	"math"
+	"regexp"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -41,6 +42,15 @@ const (
 	AssertionModeDelta AssertionMode = "delta"
 )
 
+// AggregateMode determines how multiple matching series are combined.
+type AggregateMode string
+
+const (
+	AggregateModeSingle AggregateMode = "single"
+	AggregateModeSum    AggregateMode = "sum"
+	AggregateModeEach   AggregateMode = "each"
+)
+
 // Operator specifies the comparison operation between actual and expected values.
 type Operator string
 
@@ -54,17 +64,22 @@ const (
 )
 
 // AssertionConfig defines a single metric assertion to evaluate.
-// Labels are a subset selector that must match exactly one time series;
-// matching zero or multiple series is an error.
+// Labels and labelPatterns are a subset selector. With the default aggregate mode it must
+// match exactly one time series; "sum" adds up all matching series and "each" requires
+// every matching series to pass on its own.
 type AssertionConfig struct {
 	Name          string            `yaml:"name" json:"name" desc:"Unique human-readable assertion name."`
 	Metric        string            `yaml:"metric" json:"metric" desc:"Prometheus metric name."`
-	Labels        map[string]string `yaml:"labels" json:"labels" desc:"Label selector; all specified labels must match and select exactly one time series."`
+	Labels        map[string]string `yaml:"labels" json:"labels" desc:"Label selector; all specified labels must match exactly."`
+	LabelPatterns map[string]string `yaml:"labelPatterns" json:"labelPatterns" desc:"Label selector with regex values; each pattern must match the whole label value."`
+	Aggregate     AggregateMode     `yaml:"aggregate" json:"aggregate" desc:"How matching series are combined: 'single' (exactly one series), 'sum' (sum of all series) or 'each' (every series must pass)."`
 	Mode          AssertionMode     `yaml:"mode" json:"mode" desc:"Evaluation mode: 'value' (current value) or 'delta' (change since baseline)."`
 	Operator      Operator          `yaml:"operator" json:"operator" desc:"Comparison operator: eq, neq, gt, gte, lt, lte."`
 	Value         float64           `yaml:"value" json:"value" desc:"Expected value for comparison."`
 	MissingMetric *MissingBehavior  `yaml:"missingMetric" json:"missingMetric,omitempty" desc:"Override global missingMetric behavior for this assertion."`
 	MissingSeries *MissingBehavior  `yaml:"missingSeries" json:"missingSeries,omitempty" desc:"Override global missingSeries behavior for this assertion."`
+
+	labelRegexps map[string]*regexp.Regexp
 }
 
 // Config holds the task configuration for scraping a Prometheus metrics endpoint
@@ -138,7 +153,9 @@ func (c *Config) Validate() error {
 	// Validate assertions and check for duplicate names
 	seenNames := make(map[string]bool, len(c.Assertions))
 
-	for i, a := range c.Assertions {
+	for i := range c.Assertions {
+		a := &c.Assertions[i]
+
 		if a.Name == "" {
 			return fmt.Errorf("assertion[%d]: name is required", i)
 		}
@@ -161,6 +178,23 @@ func (c *Config) Validate() error {
 			return fmt.Errorf("assertion[%d] %q: %w", i, a.Name, err)
 		}
 
+		if err := validateAggregate(a.Aggregate); err != nil {
+			return fmt.Errorf("assertion[%d] %q: %w", i, a.Name, err)
+		}
+
+		labelRegexps := make(map[string]*regexp.Regexp, len(a.LabelPatterns))
+
+		for label, pattern := range a.LabelPatterns {
+			re, err := regexp.Compile("^(?:" + pattern + ")$")
+			if err != nil {
+				return fmt.Errorf("assertion[%d] %q: invalid labelPatterns[%s]: %w", i, a.Name, label, err)
+			}
+
+			labelRegexps[label] = re
+		}
+
+		a.labelRegexps = labelRegexps
+
 		if a.MissingMetric != nil {
 			if err := validateMissingBehavior(*a.MissingMetric); err != nil {
 				return fmt.Errorf("assertion[%d] %q: missingMetric: %w", i, a.Name, err)
@@ -221,6 +255,15 @@ func validateMode(m AssertionMode) error {
 	}
 }
 
+func validateAggregate(m AggregateMode) error {
+	switch m {
+	case AggregateModeSingle, AggregateModeSum, AggregateModeEach, "":
+		return nil
+	default:
+		return fmt.Errorf("invalid aggregate %q, must be one of: single, sum, each", m)
+	}
+}
+
 func validateOperator(o Operator) error {
 	switch o {
 	case OperatorEq, OperatorNeq, OperatorGt, OperatorGte, OperatorLt, OperatorLte: