From 37abb783e17238c792fb390bf370d4fa70f5dc51 Mon Sep 17 00:00:00 2001 From: Barnabas Busa Date: Tue, 13 Oct 2026 12:36:35 +0200 Subject: [PATCH] feat: aggregate and regex-match series in check_http_metrics --- pkg/tasks/check_http_metrics/README.md | 26 ++- pkg/tasks/check_http_metrics/config.go | 51 ++++- pkg/tasks/check_http_metrics/task.go | 219 ++++++++++++++++++++-- pkg/tasks/check_http_metrics/task_test.go | 200 +++++++++++++++++--- 4 files changed, 454 insertions(+), 42 deletions(-) diff --git a/pkg/tasks/check_http_metrics/README.md b/pkg/tasks/check_http_metrics/README.md index 54526471..5e18e80c 100644 --- a/pkg/tasks/check_http_metrics/README.md +++ b/pkg/tasks/check_http_metrics/README.md @@ -48,7 +48,9 @@ The `check_http_metrics` task fetches metrics from an HTTP Prometheus endpoint a - **`name`**: Unique assertion name. Required. - **`metric`**: Prometheus metric name. Required. -- **`labels`**: Label selector (subset matching). Must match exactly one series. +- **`labels`**: Label selector (subset matching, exact values). +- **`labelPatterns`**: Label selector with regex values, each matching the whole label value (e.g. `status: "[45].."`). Combined with `labels`. +- **`aggregate`**: How matching series are combined. `single` (default): the selector must match exactly one series. `sum`: the sum of all matching series is compared. `each`: every matching series must pass on its own, failures name the offending series. - **`mode`**: `value` (current value) or `delta` (change since baseline). Default: `value`. - **`operator`**: Comparison operator: `eq`, `neq`, `gt`, `gte`, `lt`, `lte`. Required. - **`value`**: Expected numeric value. Required. @@ -63,10 +65,32 @@ In `delta` mode, the task tracks changes over time: Negative deltas are valid for GAUGE and UNTYPED metrics. For COUNTER metrics, a decrease triggers `resetBehavior`. +With `aggregate: sum` or `each`, baselines are kept per series. The first scrape records every matching series (even if there are none yet), and a series that first appears later counts from zero, so an error counter created during the run is still caught. + #### Examples ```yaml assertions: + # Fail on any new 4xx/5xx REST response, naming each route and status that occurred + - name: no_rest_errors + metric: lodestar_api_rest_errors_total + aggregate: each + mode: delta + operator: eq + value: 0 + missingMetric: pass + missingSeries: pass + + # Sum all error responses across routes + - name: few_errors + metric: ngx_request_count + labelPatterns: + status: "[45].." + aggregate: sum + mode: delta + operator: lte + value: 5 + # Check counter increased by at least 1 - name: counter_increased metric: my_counter diff --git a/pkg/tasks/check_http_metrics/config.go b/pkg/tasks/check_http_metrics/config.go index af67d4e8..5493252e 100644 --- a/pkg/tasks/check_http_metrics/config.go +++ b/pkg/tasks/check_http_metrics/config.go @@ -3,6 +3,7 @@ package checkhttpmetrics import ( "fmt" "math" + "regexp" "time" "github.com/dustin/go-humanize" @@ -41,6 +42,15 @@ const ( AssertionModeDelta AssertionMode = "delta" ) +// AggregateMode determines how multiple matching series are combined. +type AggregateMode string + +const ( + AggregateModeSingle AggregateMode = "single" + AggregateModeSum AggregateMode = "sum" + AggregateModeEach AggregateMode = "each" +) + // Operator specifies the comparison operation between actual and expected values. type Operator string @@ -54,17 +64,22 @@ const ( ) // AssertionConfig defines a single metric assertion to evaluate. -// Labels are a subset selector that must match exactly one time series; -// matching zero or multiple series is an error. +// Labels and labelPatterns are a subset selector. With the default aggregate mode it must +// match exactly one time series; "sum" adds up all matching series and "each" requires +// every matching series to pass on its own. type AssertionConfig struct { Name string `yaml:"name" json:"name" desc:"Unique human-readable assertion name."` Metric string `yaml:"metric" json:"metric" desc:"Prometheus metric name."` - Labels map[string]string `yaml:"labels" json:"labels" desc:"Label selector; all specified labels must match and select exactly one time series."` + Labels map[string]string `yaml:"labels" json:"labels" desc:"Label selector; all specified labels must match exactly."` + LabelPatterns map[string]string `yaml:"labelPatterns" json:"labelPatterns" desc:"Label selector with regex values; each pattern must match the whole label value."` + Aggregate AggregateMode `yaml:"aggregate" json:"aggregate" desc:"How matching series are combined: 'single' (exactly one series), 'sum' (sum of all series) or 'each' (every series must pass)."` Mode AssertionMode `yaml:"mode" json:"mode" desc:"Evaluation mode: 'value' (current value) or 'delta' (change since baseline)."` Operator Operator `yaml:"operator" json:"operator" desc:"Comparison operator: eq, neq, gt, gte, lt, lte."` Value float64 `yaml:"value" json:"value" desc:"Expected value for comparison."` MissingMetric *MissingBehavior `yaml:"missingMetric" json:"missingMetric,omitempty" desc:"Override global missingMetric behavior for this assertion."` MissingSeries *MissingBehavior `yaml:"missingSeries" json:"missingSeries,omitempty" desc:"Override global missingSeries behavior for this assertion."` + + labelRegexps map[string]*regexp.Regexp } // Config holds the task configuration for scraping a Prometheus metrics endpoint @@ -138,7 +153,9 @@ func (c *Config) Validate() error { // Validate assertions and check for duplicate names seenNames := make(map[string]bool, len(c.Assertions)) - for i, a := range c.Assertions { + for i := range c.Assertions { + a := &c.Assertions[i] + if a.Name == "" { return fmt.Errorf("assertion[%d]: name is required", i) } @@ -161,6 +178,23 @@ func (c *Config) Validate() error { return fmt.Errorf("assertion[%d] %q: %w", i, a.Name, err) } + if err := validateAggregate(a.Aggregate); err != nil { + return fmt.Errorf("assertion[%d] %q: %w", i, a.Name, err) + } + + labelRegexps := make(map[string]*regexp.Regexp, len(a.LabelPatterns)) + + for label, pattern := range a.LabelPatterns { + re, err := regexp.Compile("^(?:" + pattern + ")$") + if err != nil { + return fmt.Errorf("assertion[%d] %q: invalid labelPatterns[%s]: %w", i, a.Name, label, err) + } + + labelRegexps[label] = re + } + + a.labelRegexps = labelRegexps + if a.MissingMetric != nil { if err := validateMissingBehavior(*a.MissingMetric); err != nil { return fmt.Errorf("assertion[%d] %q: missingMetric: %w", i, a.Name, err) @@ -221,6 +255,15 @@ func validateMode(m AssertionMode) error { } } +func validateAggregate(m AggregateMode) error { + switch m { + case AggregateModeSingle, AggregateModeSum, AggregateModeEach, "": + return nil + default: + return fmt.Errorf("invalid aggregate %q, must be one of: single, sum, each", m) + } +} + func validateOperator(o Operator) error { switch o { case OperatorEq, OperatorNeq, OperatorGt, OperatorGte, OperatorLt, OperatorLte: diff --git a/pkg/tasks/check_http_metrics/task.go b/pkg/tasks/check_http_metrics/task.go index 2ddedbc8..c60db02d 100644 --- a/pkg/tasks/check_http_metrics/task.go +++ b/pkg/tasks/check_http_metrics/task.go @@ -7,6 +7,8 @@ import ( "io" "math" "net/http" + "regexp" + "sort" "strings" "time" @@ -80,6 +82,7 @@ type Task struct { // State baselines map[string]float64 + seriesBaselines map[string]map[string]float64 scrapeCount int scrapeErrors int assertionErrors int @@ -87,10 +90,11 @@ type Task struct { func NewTask(ctx *types.TaskContext, options *types.TaskOptions) (types.Task, error) { return &Task{ - ctx: ctx, - options: options, - logger: ctx.Logger.GetLogger(), - baselines: make(map[string]float64, 8), + ctx: ctx, + options: options, + logger: ctx.Logger.GetLogger(), + baselines: make(map[string]float64, 8), + seriesBaselines: make(map[string]map[string]float64, 8), }, nil } @@ -154,6 +158,7 @@ type assertionResult struct { value float64 delta float64 err error + detail string waiting bool // true if we should keep waiting (missing metric/series with wait behavior) } @@ -182,7 +187,7 @@ func (t *Task) runCheck(ctx context.Context) (bool, error) { results := t.evaluateAssertions(metricFamilies) // Collect results - var passedAssertions, failedAssertions []string + var passedAssertions, failedAssertions, failedDetails []string var anyWaiting, anyFailed bool @@ -208,6 +213,7 @@ func (t *Task) runCheck(ctx context.Context) (bool, error) { t.assertionErrors++ failedAssertions = append(failedAssertions, r.name) + failedDetails = append(failedDetails, fmt.Sprintf("%s: %v", r.name, r.err)) anyFailed = true t.logger.Warnf("Assertion %q error: %v", r.name, r.err) @@ -219,6 +225,7 @@ func (t *Task) runCheck(ctx context.Context) (bool, error) { passedAssertions = append(passedAssertions, r.name) } else { failedAssertions = append(failedAssertions, r.name) + failedDetails = append(failedDetails, fmt.Sprintf("%s: %s", r.name, r.detail)) anyFailed = true } } @@ -245,7 +252,14 @@ func (t *Task) runCheck(ctx context.Context) (bool, error) { case anyFailed && t.config.FailOnCheckMiss: t.ctx.SetResult(types.TaskResultFailure) - return true, fmt.Errorf("assertions failed: %v", failedAssertions) + t.ctx.ReportProgress(0, fmt.Sprintf("Assertions failed: %s", strings.Join(failedDetails, "; "))) + + return true, fmt.Errorf("assertions failed: %s", strings.Join(failedDetails, "; ")) + + case anyFailed: + t.ctx.SetResult(types.TaskResultNone) + t.ctx.ReportProgress(0, fmt.Sprintf("Waiting... passed=%d failed=%d waiting=%d, failed: %s (attempt %d)", + len(passedAssertions), len(failedAssertions), countWaiting(results), strings.Join(failedDetails, "; "), t.scrapeCount)) default: t.ctx.SetResult(types.TaskResultNone) @@ -391,17 +405,21 @@ func (t *Task) evaluateAssertion(a *AssertionConfig, metricFamilies map[string]* missingSeriesBehavior = *a.MissingSeries } + if a.Aggregate == AggregateModeSum || a.Aggregate == AggregateModeEach { + return t.evaluateSeries(a, metricFamilies[a.Metric], missingMetricBehavior, missingSeriesBehavior) + } + // Find the metric family mf, ok := metricFamilies[a.Metric] if !ok { - return t.handleMissing(result, missingMetricBehavior, fmt.Sprintf("metric %q not found", a.Metric)) + return t.handleMissing(&result, missingMetricBehavior, fmt.Sprintf("metric %q not found", a.Metric)) } // Find matching series - value, err := t.findMetricValue(mf, a.Labels) + value, err := t.findMetricValue(mf, a.Labels, a.labelRegexps) if err != nil { if strings.Contains(err.Error(), "no matching series") { - return t.handleMissing(result, missingSeriesBehavior, err.Error()) + return t.handleMissing(&result, missingSeriesBehavior, err.Error()) } result.err = err @@ -471,13 +489,168 @@ func (t *Task) evaluateAssertion(a *AssertionConfig, metricFamilies map[string]* result.passed = evaluateOperator(a.Operator, compareValue, a.Value) if !result.passed { + result.detail = fmt.Sprintf("%g (want %s %g)", compareValue, a.Operator, a.Value) t.logger.Debugf("Assertion %q failed: %.4f %s %.4f = false", a.Name, compareValue, a.Operator, a.Value) } return result } -func (t *Task) handleMissing(result assertionResult, behavior MissingBehavior, message string) assertionResult { +func (t *Task) evaluateSeries(a *AssertionConfig, mf *dto.MetricFamily, missingMetric, missingSeries MissingBehavior) assertionResult { + result := assertionResult{ + name: a.Name, + value: math.NaN(), + delta: math.NaN(), + } + + series := make(map[string]float64) + + if mf != nil { + for _, m := range mf.GetMetric() { + if !matchLabels(m.GetLabel(), a.Labels) || !matchLabelPatterns(m.GetLabel(), a.labelRegexps) { + continue + } + + key := seriesKey(m.GetLabel()) + + value, err := getMetricValue(m, mf.GetType()) + if err != nil { + result.err = fmt.Errorf("failed to extract metric value for {%s}: %w", key, err) + + return result + } + + if math.IsNaN(value) || math.IsInf(value, 0) { + result.err = fmt.Errorf("metric value is non-finite for {%s}: %v", key, value) + + return result + } + + series[key] = value + } + } + + isDelta := a.Mode == AssertionModeDelta + + baselines, hasBaseline := t.seriesBaselines[a.Name] + if isDelta && !hasBaseline { + t.seriesBaselines[a.Name] = series + t.baselines[a.Name] = sumValues(series) + result.waiting = true + + t.logger.Debugf("Assertion %q: recorded baseline for %d series", a.Name, len(series)) + + return result + } + + if mf == nil { + return t.handleMissing(&result, missingMetric, fmt.Sprintf("metric %q not found", a.Metric)) + } + + if len(series) == 0 { + return t.handleMissing(&result, missingSeries, fmt.Sprintf("no matching series for labels %v / patterns %v", a.Labels, a.LabelPatterns)) + } + + isCounter := mf.GetType() == dto.MetricType_COUNTER + compare := make(map[string]float64, len(series)) + + for key, value := range series { + if !isDelta { + compare[key] = value + + continue + } + + baseline := baselines[key] + + if isCounter && value < baseline { + switch t.config.ResetBehavior { + case ResetBehaviorFail: + result.err = fmt.Errorf("counter reset detected for {%s}: current %.4f < baseline %.4f", key, value, baseline) + + return result + case ResetBehaviorRebaseline: + baselines[key] = value + baseline = value + + t.logger.Warnf("Assertion %q: counter reset detected for {%s}, rebaselining to %.4f", a.Name, key, value) + case ResetBehaviorIgnore: + t.logger.Warnf("Assertion %q: counter reset detected for {%s}, ignoring", a.Name, key) + } + } + + compare[key] = value - baseline + } + + total := sumValues(compare) + + if isDelta { + result.value = sumValues(series) + result.delta = total + } else { + result.value = total + } + + keys := make([]string, 0, len(compare)) + for key := range compare { + keys = append(keys, key) + } + + sort.Strings(keys) + + if a.Aggregate == AggregateModeSum { + result.passed = evaluateOperator(a.Operator, total, a.Value) + + if !result.passed { + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, fmt.Sprintf("{%s} %g", key, compare[key])) + } + + result.detail = fmt.Sprintf("sum %g (want %s %g): %s", total, a.Operator, a.Value, strings.Join(parts, ", ")) + } + + return result + } + + failing := make([]string, 0) + + for _, key := range keys { + if !evaluateOperator(a.Operator, compare[key], a.Value) { + failing = append(failing, fmt.Sprintf("{%s} %g", key, compare[key])) + } + } + + result.passed = len(failing) == 0 + + if !result.passed { + result.detail = fmt.Sprintf("%d of %d series (want %s %g): %s", len(failing), len(keys), a.Operator, a.Value, strings.Join(failing, ", ")) + } + + return result +} + +func seriesKey(labels []*dto.LabelPair) string { + parts := make([]string, 0, len(labels)) + for _, lp := range labels { + parts = append(parts, fmt.Sprintf("%s=%q", lp.GetName(), lp.GetValue())) + } + + sort.Strings(parts) + + return strings.Join(parts, ",") +} + +func sumValues(values map[string]float64) float64 { + total := 0.0 + for _, v := range values { + total += v + } + + return total +} + +func (t *Task) handleMissing(result *assertionResult, behavior MissingBehavior, message string) assertionResult { switch behavior { case MissingBehaviorFail: result.err = errors.New(message) @@ -489,14 +662,14 @@ func (t *Task) handleMissing(result assertionResult, behavior MissingBehavior, m result.waiting = true } - return result + return *result } -func (t *Task) findMetricValue(mf *dto.MetricFamily, labels map[string]string) (float64, error) { +func (t *Task) findMetricValue(mf *dto.MetricFamily, labels map[string]string, patterns map[string]*regexp.Regexp) (float64, error) { var matchingMetrics []*dto.Metric for _, m := range mf.GetMetric() { - if matchLabels(m.GetLabel(), labels) { + if matchLabels(m.GetLabel(), labels) && matchLabelPatterns(m.GetLabel(), patterns) { matchingMetrics = append(matchingMetrics, m) } } @@ -537,6 +710,26 @@ func matchLabels(metricLabels []*dto.LabelPair, wantLabels map[string]string) bo return true } +func matchLabelPatterns(metricLabels []*dto.LabelPair, patterns map[string]*regexp.Regexp) bool { + if len(patterns) == 0 { + return true + } + + labelMap := make(map[string]string, len(metricLabels)) + + for _, lp := range metricLabels { + labelMap[lp.GetName()] = lp.GetValue() + } + + for k, re := range patterns { + if !re.MatchString(labelMap[k]) { + return false + } + } + + return true +} + var errNoMetricValue = errors.New("metric has no extractable value") func getMetricValue(m *dto.Metric, mType dto.MetricType) (float64, error) { diff --git a/pkg/tasks/check_http_metrics/task_test.go b/pkg/tasks/check_http_metrics/task_test.go index d52e122f..8c1e0301 100644 --- a/pkg/tasks/check_http_metrics/task_test.go +++ b/pkg/tasks/check_http_metrics/task_test.go @@ -491,9 +491,10 @@ func newTestTask(cfg *Config) *Task { } return &Task{ - config: c, - logger: logger, - baselines: make(map[string]float64), + config: c, + logger: logger, + baselines: make(map[string]float64), + seriesBaselines: make(map[string]map[string]float64), } } @@ -629,7 +630,7 @@ func TestFindMetricValue(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { task := newTestTask(&Config{}) - val, err := task.findMetricValue(tt.mf, tt.labels) + val, err := task.findMetricValue(tt.mf, tt.labels, nil) switch { case tt.wantErr != "" && err == nil: @@ -663,7 +664,7 @@ func TestHandleMissing(t *testing.T) { t.Run(tt.name, func(t *testing.T) { task := newTestTask(&Config{}) result := assertionResult{name: "test"} - result = task.handleMissing(result, tt.behavior, "metric not found") + result = task.handleMissing(&result, tt.behavior, "metric not found") if tt.wantErr && result.err == nil { t.Error("expected error, got nil") @@ -1336,10 +1337,11 @@ func newTestTaskWithContext(cfg *Config) (*Task, types.Variables, *types.TaskRes } task := &Task{ - ctx: ctx, - config: config, - logger: logger, - baselines: make(map[string]float64), + ctx: ctx, + config: config, + logger: logger, + baselines: make(map[string]float64), + seriesBaselines: make(map[string]map[string]float64), } return task, outputs, &lastResult @@ -1689,11 +1691,12 @@ http_requests_total{method="POST",status="200"} 50 } task := &Task{ - ctx: ctx, - config: cfg, - logger: logger, - baselines: make(map[string]float64), - httpClient: &http.Client{Timeout: cfg.RequestTimeout.Duration}, + ctx: ctx, + config: cfg, + logger: logger, + baselines: make(map[string]float64), + seriesBaselines: make(map[string]map[string]float64), + httpClient: &http.Client{Timeout: cfg.RequestTimeout.Duration}, } // Run check @@ -1777,11 +1780,12 @@ requests_total %d } task := &Task{ - ctx: ctx, - config: cfg, - logger: logger, - baselines: make(map[string]float64), - httpClient: &http.Client{Timeout: cfg.RequestTimeout.Duration}, + ctx: ctx, + config: cfg, + logger: logger, + baselines: make(map[string]float64), + seriesBaselines: make(map[string]map[string]float64), + httpClient: &http.Client{Timeout: cfg.RequestTimeout.Duration}, } // First scrape - records baseline, doesn't complete @@ -1885,11 +1889,12 @@ memory_usage{host="server1"} 60.0 } task := &Task{ - ctx: ctx, - config: cfg, - logger: logger, - baselines: make(map[string]float64), - httpClient: &http.Client{Timeout: cfg.RequestTimeout.Duration}, + ctx: ctx, + config: cfg, + logger: logger, + baselines: make(map[string]float64), + seriesBaselines: make(map[string]map[string]float64), + httpClient: &http.Client{Timeout: cfg.RequestTimeout.Duration}, } done, err := task.runCheck(context.Background()) @@ -1929,3 +1934,150 @@ memory_usage{host="server1"} 60.0 t.Errorf("values[server1_cpu_ok] = %v, want 45.5", v) } } + +func restErrorFamily(values map[string]float64) *dto.MetricFamily { + metrics := make([]*dto.Metric, 0, len(values)) + for route, value := range values { + metrics = append(metrics, makeCounter(value, map[string]string{"route": route, "status": "404"})) + } + + metrics = append(metrics, makeCounter(1000, map[string]string{"route": "getBlockV2", "status": "200"})) + + return makeCounterFamily(testMetric1, metrics...) +} + +func mustValidAssertion(t *testing.T, a *AssertionConfig) *AssertionConfig { + t.Helper() + + cfg := validBaseConfig() + cfg.URL = testMetricsURL + cfg.Assertions = []AssertionConfig{*a} + + if err := cfg.Validate(); err != nil { + t.Fatalf("unexpected validation error: %v", err) + } + + return &cfg.Assertions[0] +} + +func TestEvaluateAssertion_EachDelta_NamesFailingSeries(t *testing.T) { + task := newTestTask(&Config{ResetBehavior: ResetBehaviorFail}) + assertion := mustValidAssertion(t, &AssertionConfig{ + Name: "rest_errors", + Metric: testMetric1, + LabelPatterns: map[string]string{"status": "[45].."}, + Aggregate: AggregateModeEach, + Mode: AssertionModeDelta, + Operator: OperatorEq, + Value: 0, + }) + + first := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{ + testMetric1: restErrorFamily(map[string]float64{"getStateV2": 2}), + }) + if !first.waiting { + t.Fatal("first scrape should record baselines and wait") + } + + second := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{ + testMetric1: restErrorFamily(map[string]float64{"getStateV2": 2, "getAggregatedAttestationV2": 7}), + }) + if second.passed || second.err != nil { + t.Fatalf("expected failure, got passed=%v err=%v", second.passed, second.err) + } + + want := `1 of 2 series (want eq 0): {route="getAggregatedAttestationV2",status="404"} 7` + if second.detail != want { + t.Fatalf("detail = %q, want %q", second.detail, want) + } + + if second.delta != 7 { + t.Fatalf("delta = %v, want 7", second.delta) + } +} + +func TestEvaluateAssertion_EachDelta_SeriesMissingAtStart(t *testing.T) { + task := newTestTask(&Config{MissingMetric: MissingBehaviorPass, MissingSeries: MissingBehaviorPass}) + assertion := mustValidAssertion(t, &AssertionConfig{ + Name: "rest_errors", + Metric: testMetric1, + LabelPatterns: map[string]string{"status": "[45].."}, + Aggregate: AggregateModeEach, + Mode: AssertionModeDelta, + Operator: OperatorEq, + Value: 0, + }) + + if r := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{}); !r.waiting { + t.Fatal("first scrape without the metric should still record an empty baseline") + } + + if r := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{}); !r.passed { + t.Fatal("missing metric with pass behavior should pass") + } + + r := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{ + testMetric1: restErrorFamily(map[string]float64{"getAggregatedAttestationV2": 3}), + }) + if r.passed { + t.Fatal("series appearing after the baseline should count from zero") + } +} + +func TestEvaluateAssertion_SumValue(t *testing.T) { + task := newTestTask(&Config{}) + assertion := mustValidAssertion(t, &AssertionConfig{ + Name: "rest_errors", + Metric: testMetric1, + LabelPatterns: map[string]string{"status": "[45].."}, + Aggregate: AggregateModeSum, + Operator: OperatorLte, + Value: 5, + }) + + r := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{ + testMetric1: restErrorFamily(map[string]float64{"a": 2, "b": 4}), + }) + if r.passed || r.value != 6 { + t.Fatalf("got passed=%v value=%v, want failure with value 6", r.passed, r.value) + } + + want := `sum 6 (want lte 5): {route="a",status="404"} 2, {route="b",status="404"} 4` + if r.detail != want { + t.Fatalf("detail = %q, want %q", r.detail, want) + } +} + +func TestEvaluateAssertion_SingleWithLabelPatterns(t *testing.T) { + task := newTestTask(&Config{}) + assertion := mustValidAssertion(t, &AssertionConfig{ + Name: "ok_requests", + Metric: testMetric1, + LabelPatterns: map[string]string{"status": "2.."}, + Operator: OperatorGt, + Value: 0, + }) + + r := task.evaluateAssertion(assertion, map[string]*dto.MetricFamily{ + testMetric1: restErrorFamily(map[string]float64{"a": 2}), + }) + if !r.passed || r.value != 1000 { + t.Fatalf("got passed=%v value=%v, want pass with value 1000", r.passed, r.value) + } +} + +func TestConfig_Validate_AggregateAndPatterns(t *testing.T) { + cfg := validBaseConfig() + cfg.URL = testMetricsURL + cfg.Assertions = []AssertionConfig{{Name: "a", Metric: testMetric1, Operator: OperatorEq, Aggregate: "max"}} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "invalid aggregate") { + t.Fatalf("expected invalid aggregate error, got %v", err) + } + + cfg.Assertions = []AssertionConfig{{Name: "a", Metric: testMetric1, Operator: OperatorEq, LabelPatterns: map[string]string{"status": "("}}} + + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "invalid labelPatterns") { + t.Fatalf("expected invalid labelPatterns error, got %v", err) + } +}