From 6134f657774941a0434d3ec45af9f796ba6a16c3 Mon Sep 17 00:00:00 2001 From: Buildkite Date: Tue, 29 Sep 2026 23:36:30 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20sync'ing=20generated=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/admin/code-hosts/aws-codecommit.mdx | 2 +- docs/admin/code-hosts/azuredevops.mdx | 2 +- docs/admin/code-hosts/bitbucket-cloud.mdx | 2 +- docs/admin/code-hosts/bitbucket-server.mdx | 2 +- docs/admin/code-hosts/gerrit.mdx | 2 +- docs/admin/code-hosts/github.mdx | 2 +- docs/admin/code-hosts/gitlab.mdx | 2 +- docs/admin/code-hosts/gitolite.mdx | 2 +- docs/admin/code-hosts/other.mdx | 2 +- docs/admin/code-hosts/phabricator.mdx | 2 +- docs/admin/config/settings.mdx | 2 +- docs/admin/config/site-config.mdx | 7 +- docs/admin/repo/perforce.mdx | 2 +- docs/ai/index.mdx | 5 +- docs/ai/models.mdx | 5 +- docs/self-hosted/observability/alerts.mdx | 186 +++++++++++++++++- docs/self-hosted/observability/dashboards.mdx | 94 ++++++--- 17 files changed, 276 insertions(+), 45 deletions(-) diff --git a/docs/admin/code-hosts/aws-codecommit.mdx b/docs/admin/code-hosts/aws-codecommit.mdx index 03e037fcc..51432989a 100644 --- a/docs/admin/code-hosts/aws-codecommit.mdx +++ b/docs/admin/code-hosts/aws-codecommit.mdx @@ -37,7 +37,7 @@ AWS CodeCommit connections support the following configuration options, which ar {/* SCHEMA_SYNC_START: admin/code_hosts/aws_codecommit.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // REQUIRED: diff --git a/docs/admin/code-hosts/azuredevops.mdx b/docs/admin/code-hosts/azuredevops.mdx index f89960094..299933204 100644 --- a/docs/admin/code-hosts/azuredevops.mdx +++ b/docs/admin/code-hosts/azuredevops.mdx @@ -69,7 +69,7 @@ Azure DevOps connections support the following configuration options, which are {/* SCHEMA_SYNC_START: admin/code_hosts/azuredevops.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json // Authentication alternatives: token OR windowsPassword diff --git a/docs/admin/code-hosts/bitbucket-cloud.mdx b/docs/admin/code-hosts/bitbucket-cloud.mdx index 5962dd708..63403c52a 100644 --- a/docs/admin/code-hosts/bitbucket-cloud.mdx +++ b/docs/admin/code-hosts/bitbucket-cloud.mdx @@ -116,7 +116,7 @@ Bitbucket Cloud connections support the following configuration options, which a {/* SCHEMA_SYNC_START: admin/code_hosts/bitbucket_cloud.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // The workspace access token to use when authenticating with Bitbucket Cloud. diff --git a/docs/admin/code-hosts/bitbucket-server.mdx b/docs/admin/code-hosts/bitbucket-server.mdx index 1b652d780..3cc3a2af4 100644 --- a/docs/admin/code-hosts/bitbucket-server.mdx +++ b/docs/admin/code-hosts/bitbucket-server.mdx @@ -202,7 +202,7 @@ Bitbucket Server / Bitbucket Data Center connections support the following confi {/* SCHEMA_SYNC_START: admin/code_hosts/bitbucket_server.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json // Authentication alternatives: token OR password diff --git a/docs/admin/code-hosts/gerrit.mdx b/docs/admin/code-hosts/gerrit.mdx index 67e9f819c..4d1dca75f 100644 --- a/docs/admin/code-hosts/gerrit.mdx +++ b/docs/admin/code-hosts/gerrit.mdx @@ -113,7 +113,7 @@ Gerrit connections support the following configuration options, which are specif {/* SCHEMA_SYNC_START: admin/code_hosts/gerrit.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // If non-null, enforces Gerrit repository permissions. This requires that there is an item in the [site configuration json](https://sourcegraph.com/docs/admin/config/site_config#auth-providers) `auth.providers` field, of type "gerrit" with the same `url` field as specified in this `GerritConnection`. diff --git a/docs/admin/code-hosts/github.mdx b/docs/admin/code-hosts/github.mdx index 47253ac70..a0081bbc0 100644 --- a/docs/admin/code-hosts/github.mdx +++ b/docs/admin/code-hosts/github.mdx @@ -458,7 +458,7 @@ GitHub connections support the following configuration options, which are specif {/* SCHEMA_SYNC_START: admin/code_hosts/github.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json // Authentication alternatives: token OR gitHubAppDetails OR externalAccount OR useRandomExternalAccount diff --git a/docs/admin/code-hosts/gitlab.mdx b/docs/admin/code-hosts/gitlab.mdx index 4d30afbfc..ed2a395f5 100644 --- a/docs/admin/code-hosts/gitlab.mdx +++ b/docs/admin/code-hosts/gitlab.mdx @@ -189,7 +189,7 @@ See [Internal rate limits](/admin/code-hosts/rate-limits#internal-rate-limits). {/* SCHEMA_SYNC_START: admin/code_hosts/gitlab.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // If non-null, enforces GitLab repository permissions. This requires that there be an item in the `auth.providers` field of type "gitlab" with the same `url` field as specified in this `GitLabConnection`. diff --git a/docs/admin/code-hosts/gitolite.mdx b/docs/admin/code-hosts/gitolite.mdx index 3c943d668..4b99d1861 100644 --- a/docs/admin/code-hosts/gitolite.mdx +++ b/docs/admin/code-hosts/gitolite.mdx @@ -25,7 +25,7 @@ To connect Gitolite to Sourcegraph: {/* SCHEMA_SYNC_START: admin/code_hosts/gitolite.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // A list of repositories to never mirror from this Gitolite instance. Supports excluding by exact name ({"name": "foo"}). diff --git a/docs/admin/code-hosts/other.mdx b/docs/admin/code-hosts/other.mdx index 296d5269d..1b7d5c0c2 100644 --- a/docs/admin/code-hosts/other.mdx +++ b/docs/admin/code-hosts/other.mdx @@ -68,7 +68,7 @@ Repositories must be listed individually: {/* SCHEMA_SYNC_START: admin/code_hosts/other_external_service.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // A list of repositories to never mirror by name after applying repositoryPathPattern. Supports excluding by exact name ({"name": "myrepo"}) or regular expression ({"pattern": ".*secret.*"}). diff --git a/docs/admin/code-hosts/phabricator.mdx b/docs/admin/code-hosts/phabricator.mdx index f2b879273..8fae1be74 100644 --- a/docs/admin/code-hosts/phabricator.mdx +++ b/docs/admin/code-hosts/phabricator.mdx @@ -76,7 +76,7 @@ The Sourcegraph instance's site admin must [update the `corsOrigin` site config {/* SCHEMA_SYNC_START: admin/code_hosts/phabricator.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // SSH cipher to use when cloning via SSH. Must be a valid choice from `ssh -Q cipher`. diff --git a/docs/admin/config/settings.mdx b/docs/admin/config/settings.mdx index 4dcfa913b..05ee7c8fc 100644 --- a/docs/admin/config/settings.mdx +++ b/docs/admin/config/settings.mdx @@ -27,7 +27,7 @@ Settings options and their default values are shown below. {/* SCHEMA_SYNC_START: admin/config/settings.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { diff --git a/docs/admin/config/site-config.mdx b/docs/admin/config/site-config.mdx index f4f453a0d..b3b1c767f 100644 --- a/docs/admin/config/site-config.mdx +++ b/docs/admin/config/site-config.mdx @@ -21,7 +21,7 @@ All site configuration options and their default values are shown below. {/* SCHEMA_SYNC_START: admin/config/site.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { @@ -292,6 +292,7 @@ All site configuration options and their default values are shown below. // Experimental features and settings. "experimentalFeatures": { + "agents.contextFilters": null, "batchChanges.enableForkNameSuffix": false, "batchChanges.enablePerforce": false, "codeintelSyntacticIndexing.enabled": false, @@ -307,9 +308,7 @@ All site configuration options and their default values are shown below. "debug.log": { "extsvc.gitlab": false }, - "deepSearch.contextFilters": { - "exclude": null - }, + "deepSearch.contextFilters": null, "deepSearch.enabled": false, "deepSearch.model": null, "deepSearch.sharing.enabled": false, diff --git a/docs/admin/repo/perforce.mdx b/docs/admin/repo/perforce.mdx index eddc923d0..861dba94e 100644 --- a/docs/admin/repo/perforce.mdx +++ b/docs/admin/repo/perforce.mdx @@ -228,7 +228,7 @@ With this setting, Sourcegraph will ignore any rules with a host other than `*`, {/* SCHEMA_SYNC_START: admin/code_hosts/perforce.schema.json */} {/* WARNING: This section is auto-generated during releases. Do not edit manually. */} -{/* Last updated: 2026-09-27T11:21:12Z */} +{/* Last updated: 2026-09-29T23:36:29Z */} ```json { // If non-null, enforces Perforce depot permissions. diff --git a/docs/ai/index.mdx b/docs/ai/index.mdx index 02726b986..f7129b281 100644 --- a/docs/ai/index.mdx +++ b/docs/ai/index.mdx @@ -1,6 +1,6 @@ # Sourcegraph AI features -This reference lists the AI-powered features in Sourcegraph. +This reference lists the main AI-powered features in Sourcegraph. {/* BEGIN GENERATED: ai-features-reference */} {/* DO NOT EDIT: generated via: sg generate ai-docs */} @@ -11,7 +11,8 @@ This reference lists the AI-powered features in Sourcegraph. - [Smart Hover](/code-navigation/smart-hover) - [Query Assist](/code-search/query-assist) - [Agentic Batch Changes](/agentic-batch-changes) +- [Dictation](https://sourcegraph.com/changelog/agent-dictation) -See the [model usage breakdown](/ai/models) for details. +See the [model usage breakdown](/ai/models) for a comprehensive list. {/* END GENERATED: ai-features-reference */} \ No newline at end of file diff --git a/docs/ai/models.mdx b/docs/ai/models.mdx index 7918dc4d1..4bbec5b93 100644 --- a/docs/ai/models.mdx +++ b/docs/ai/models.mdx @@ -9,13 +9,12 @@ This reference lists the models currently used by [Sourcegraph AI features](/ai) | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Claude Sonnet 5 | [Deep Search](/deep-search) | | GPT-5.4 Nano | Deep Search categorization, [Smart Hover](/code-navigation/smart-hover), Agentic Batch Changes categorization | -| Claude Haiku 4.5 | Deep Search answer summaries | +| GPT-6 Luna | Deep Search answer summaries, Coding agent low Finder, Coding agent medium Finder, Coding agent high Finder | | GPT-5.6 Luna | Deep Search titles, Deep Search compaction, [Code Finder](/api/mcp#code-finder), [Diff Tour](/diff-tour), Agentic Batch Changes titles, Agentic Batch Changes workspace analysis, Agentic Batch Changes schema help, Agentic Batch Changes repository search | | Gemini 3.5 Flash-Lite | [Query Assist](/code-search/query-assist) | | Claude Opus 5.5 | [Agentic Batch Changes](/agentic-batch-changes), Agentic Batch Changes compaction | | GPT-6 Sol | Coding agent low, Coding agent medium | | GPT-6 Astra | Coding agent high, Coding agent low Oracle, Coding agent medium Oracle, Coding agent high Oracle | -| GPT-6 Luna | Coding agent low Finder, Coding agent medium Finder, Coding agent high Finder | -| GPT Transcribe | Dictation | +| GPT Transcribe | [Dictation](https://sourcegraph.com/changelog/agent-dictation) | {/* END GENERATED: ai-models-reference */} \ No newline at end of file diff --git a/docs/self-hosted/observability/alerts.mdx b/docs/self-hosted/observability/alerts.mdx index ef1ac5139..0c560a99f 100644 --- a/docs/self-hosted/observability/alerts.mdx +++ b/docs/self-hosted/observability/alerts.mdx @@ -7586,9 +7586,9 @@ Generated query for critical alert: `max((sum(increase(src_deepsearch_question_o
Technical details -Custom query for warning alert: `max(((sum(increase(src_batch_change_agent_message_errors_total{op="Handle"\}[5m])) / (sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[5m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"\}[5m]))) * 100) and ((sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[5m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"}[5m]))) >= 5)) > 20)` +Custom query for warning alert: `max(((sum(increase(src_batch_change_agent_message_errors_total{op="Handle"\}[30m])) / (sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"\}[30m]))) * 100) and ((sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"}[30m]))) >= 5)) > 20)` -Custom query for critical alert: `max(((sum(increase(src_batch_change_agent_message_errors_total{op="Handle"\}[5m])) / (sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[5m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"\}[5m]))) * 100) and ((sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[5m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"}[5m]))) >= 5)) > 50)` +Custom query for critical alert: `max(((sum(increase(src_batch_change_agent_message_errors_total{op="Handle"\}[30m])) / (sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"\}[30m]))) * 100) and ((sum(increase(src_batch_change_agent_message_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_message_errors_total\{op="Handle"}[30m]))) >= 5)) > 50)`
@@ -7632,6 +7632,118 @@ Generated query for critical alert: `max((max(src_workerutil_queue_duration_seco
+## batch-change-agents: batch_change_agent_runs_over_context_budget + +

runs stopped because the conversation no longer fits the context budget over 1h

+ +**Descriptions** + +- warning batch-change-agents: 0+ runs stopped because the conversation no longer fits the context budget over 1h +- critical batch-change-agents: 3+ runs stopped because the conversation no longer fits the context budget over 1h + +**Next steps** + +- Each run here failed the user`s message with "the conversation no longer fits the agent`s context budget, even after compaction". The user cannot recover on their own, since an agent has a single thread. +- The critical threshold means either several agents are affected or one user has retried repeatedly without success; both need someone to look at the affected threads. +- Search worker logs for that message to find the affected threadIDs, then look at the preceding "compacted thread" entries for the same thread to see what the compacter kept and why the result was still too large. +- If the thread has a batch spec or a single tool result that alone approaches the budget, compaction cannot help; otherwise treat this as a compaction strategy or estimator bug. +- More help interpreting this metric is available in the [dashboards reference](dashboards#batch-change-agents-batch_change_agent_runs_over_context_budget). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_batch-change-agents_batch_change_agent_runs_over_context_budget", + "critical_batch-change-agents_batch_change_agent_runs_over_context_budget" +] +``` + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Generated query for warning alert: `max((sum(increase(src_agent_runs_total{stop_reason="max_input_tokens",workload="batch_change_agent"}[1h]))) > 0)` + +Generated query for critical alert: `max((sum(increase(src_agent_runs_total{stop_reason="max_input_tokens",workload="batch_change_agent"}[1h]))) > 3)` + +
+ +
+ +## batch-change-agents: batch_change_agent_compaction_failures + +

failed context compactions over 30m

+ +**Descriptions** + +- warning batch-change-agents: 2+ failed context compactions over 30m +- critical batch-change-agents: more than 50% of context compactions failed over 1h + +**Next steps** + +- A failed compaction fails the message it ran in, so this shows up as user-visible errors on long threads. +- Search worker logs for "agent: compaction failed" to find the underlying error: summarizer model call failures point at the LLM provider, and SaveThreadCompaction errors point at the database or a concurrent run on the same thread. +- Check the "Compaction summarizer" group: summarizer input sizes near the model`s context window mean the summarizer call itself is being rejected. +- More help interpreting this metric is available in the [dashboards reference](dashboards#batch-change-agents-batch_change_agent_compaction_failures). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_batch-change-agents_batch_change_agent_compaction_failures", + "critical_batch-change-agents_batch_change_agent_compaction_failures" +] +``` + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Generated query for warning alert: `max((sum(increase(src_agent_compactions_total{status!="success",workload="batch_change_agent"}[30m]))) > 2)` + +Custom query for critical alert: `max(((sum(increase(src_agent_compactions_total{status!="success",workload="batch_change_agent"\}[1h])) / sum(increase(src_agent_compactions_total\{workload="batch_change_agent"\}[1h])) * 100) and (sum(increase(src_agent_compactions_total\{workload="batch_change_agent"}[1h])) >= 4)) > 50)` + +
+ +
+ +## batch-change-agents: batch_change_agent_wake_error_rate + +

wake processing error rate over 5m

+ +**Descriptions** + +- warning batch-change-agents: 20%+ wake processing error rate over 5m for 15m0s +- critical batch-change-agents: 50%+ wake processing error rate over 5m for 15m0s + +**Next steps** + +- Check worker logs for errors from the batch_change_agent_wake worker`s Handle operation; they carry the agentID and threadID and the underlying error. +- Wake jobs only read inbox state and enqueue a message, so failures usually point at the database or at the message queue store rather than at the LLM provider. +- Compare with the "Job queue stores" group: a matching error rate on the wake worker store means the store, not the handler, is failing. +- More help interpreting this metric is available in the [dashboards reference](dashboards#batch-change-agents-batch_change_agent_wake_error_rate). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_batch-change-agents_batch_change_agent_wake_error_rate", + "critical_batch-change-agents_batch_change_agent_wake_error_rate" +] +``` + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Custom query for warning alert: `max(((sum(increase(src_batch_change_agent_wake_errors_total{op="Handle"\}[30m])) / (sum(increase(src_batch_change_agent_wake_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_wake_errors_total\{op="Handle"\}[30m]))) * 100) and ((sum(increase(src_batch_change_agent_wake_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_wake_errors_total\{op="Handle"}[30m]))) >= 5)) > 20)` + +Custom query for critical alert: `max(((sum(increase(src_batch_change_agent_wake_errors_total{op="Handle"\}[30m])) / (sum(increase(src_batch_change_agent_wake_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_wake_errors_total\{op="Handle"\}[30m]))) * 100) and ((sum(increase(src_batch_change_agent_wake_total\{op="Handle"\}[30m])) + sum(increase(src_batch_change_agent_wake_errors_total\{op="Handle"}[30m]))) >= 5)) > 50)` + +
+ +
+ ## batch-change-agents: batch_change_agent_records_reset

stalled jobs reset to queued over 1h by queue

@@ -7665,6 +7777,43 @@ Generated query for warning alert: `max((sum by (domain) (increase(src_dbworker_
+## batch-change-agents: batch_change_agent_record_reset_failures + +

stalled jobs marked failed over 1h by queue

+ +**Descriptions** + +- warning batch-change-agents: 0+ stalled jobs marked failed over 1h by queue +- critical batch-change-agents: 5+ stalled jobs marked failed over 1h by queue + +**Next steps** + +- The job stalled repeatedly and ran out of retries, so the message or wake it carried was dropped; the user`s message is left in a failed state without an answer. +- Find the affected records via the failure_message column on batch_change_agent_jobs or batch_change_agent_wake_jobs, and check worker logs for panics or OOM-kills around the stall times. +- Sustained resets in `batch_change_agent_records_reset` for the same queue point at a crash loop rather than an isolated bad job. +- More help interpreting this metric is available in the [dashboards reference](dashboards#batch-change-agents-batch_change_agent_record_reset_failures). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_batch-change-agents_batch_change_agent_record_reset_failures", + "critical_batch-change-agents_batch_change_agent_record_reset_failures" +] +``` + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Generated query for warning alert: `max((sum by (domain) (increase(src_dbworker_resetter_record_reset_failures_total{domain=~"^batch_change_agent_.*_resetter$"}[1h]))) > 0)` + +Generated query for critical alert: `max((sum by (domain) (increase(src_dbworker_resetter_record_reset_failures_total{domain=~"^batch_change_agent_.*_resetter$"}[1h]))) > 5)` + +
+ +
+ ## batch-change-agents: batch_change_agent_message_stream_publish_errors

message stream publish errors per second

@@ -7698,6 +7847,39 @@ Generated query for warning alert: `max((sum(rate(src_batch_change_agents_stream
+## batch-change-agents: batch_change_agent_thread_stream_publish_errors + +

thread stream publish errors per second

+ +**Descriptions** + +- warning batch-change-agents: 0+ thread stream publish errors per second for 5m0s + +**Next steps** + +- These are publish attempts that failed after retries were exhausted, so clients miss the prompt to refetch the thread until their next reconnect or navigation. +- Check Redis health and connectivity from the worker; `batch_change_agent_message_stream_publish_errors` failing at the same time confirms a shared Redis problem. +- Thread state itself is unaffected: the entries are persisted before publishing. +- More help interpreting this metric is available in the [dashboards reference](dashboards#batch-change-agents-batch_change_agent_thread_stream_publish_errors). +- **Silence this alert:** If you are aware of this alert and want to silence notifications for it, add the following to your site configuration and set a reminder to re-evaluate the alert: + +```json +"observability.silenceAlerts": [ + "warning_batch-change-agents_batch_change_agent_thread_stream_publish_errors" +] +``` + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Generated query for warning alert: `max((sum(rate(src_batch_change_agents_thread_stream_publish_errors_total[5m]))) > 0)` + +
+ +
+ ## batch-changes: batch_changes_records_reset

stalled jobs reset to queued over 1h by queue

diff --git a/docs/self-hosted/observability/dashboards.mdx b/docs/self-hosted/observability/dashboards.mdx index 7f1d9f514..04a168e87 100644 --- a/docs/self-hosted/observability/dashboards.mdx +++ b/docs/self-hosted/observability/dashboards.mdx @@ -37540,7 +37540,7 @@ histogram_quantile(0.50, sum(rate(src_batch_change_agent_message_duration_second

Message processing error rate over 5m

-Percentage of message handler invocations that ended in an error. The alert is gated on at least 5 attempts in the window so it stays quiet during idle periods. +Percentage of message handler invocations that ended in an error. The alerts evaluate the rate over 30m and are gated on at least 5 attempts in that window, so they stay quiet during idle periods. Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_message_error_rate) for 2 alerts related to this panel. @@ -37928,6 +37928,56 @@ histogram_quantile(0.95, sum by (le, workload) (rate(src_agent_compaction_durati
+### Batch Change Agents: Agent loop health + +#### batch-change-agents: batch_change_agent_runs_over_context_budget + +

Runs stopped because the conversation no longer fits the context budget over 1h

+ +Agent runs that ended with stop reason max_input_tokens: the prompt exceeded the budget even after compaction, or the provider rejected an already compacted prompt. Expected to be zero. + +Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_runs_over_context_budget) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100500` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Query: + +``` +sum(increase(src_agent_runs_total{workload="batch_change_agent",stop_reason="max_input_tokens"}[1h])) +``` +
+ +
+ +#### batch-change-agents: batch_change_agent_compaction_failures + +

Failed context compactions over 30m

+ +Compactions that did not complete successfully. The warning tolerates an isolated provider error. The critical alert fires when more than half of the compactions over 1h failed (gated on at least 4 compactions), which means long threads cannot make progress regardless of traffic volume. + +Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_compaction_failures) for 2 alerts related to this panel. + +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100501` on your Sourcegraph instance. + +*Managed by the Sourcegraph Code Plane team.* + +
+Technical details + +Query: + +``` +sum(increase(src_agent_compactions_total{workload="batch_change_agent",status!="success"}[30m])) +``` +
+ +
+ ### Batch Change Agents: Compaction summarizer #### batch-change-agents: batch_change_agent_compaction_p95_summary_input_tokens @@ -37938,7 +37988,7 @@ Provider-reported input size of the summarizer call. The transcript is capped by This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100500` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100600` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -37964,7 +38014,7 @@ Number of wake jobs being processed concurrently. The wake worker only resolves This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100600` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100700` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -37988,7 +38038,7 @@ Rate of wake jobs by result: woken created an automated turn, empty found no unr This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100601` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100701` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38012,7 +38062,7 @@ sum by (outcome) (rate(src_batch_change_agent_wake_outcomes_total[5m])) * 60 This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100602` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100702` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38032,11 +38082,11 @@ histogram_quantile(0.95, sum(rate(src_batch_change_agent_wake_duration_seconds_b

Wake processing error rate over 5m

-Percentage of wake handler invocations that ended in an error. Wake failures are retried and rarely user-visible, but a sustained nonzero rate means threads are not being re-activated when new inbox items arrive. +Percentage of wake handler invocations that ended in an error. Wake failures are retried and rarely user-visible, but a sustained rate means threads are not being re-activated when new inbox items arrive. The alerts evaluate the rate over 30m and are gated on at least 5 attempts in that window, so they stay quiet during idle periods. -This panel has no related alerts. +Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_wake_error_rate) for 2 alerts related to this panel. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100610` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100710` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38062,7 +38112,7 @@ Rate of successful worker store operations (dequeue, mark-complete, etc.) for th This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100700` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100800` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38086,7 +38136,7 @@ Percentage of worker store operations that errored, per queue. A sustained nonze This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100701` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100801` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38110,7 +38160,7 @@ sum by (domain) (increase(src_workerutil_dbworker_store_errors_total{domain=~"^b This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100702` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100802` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38136,7 +38186,7 @@ Number of stalled batch change agent jobs that the resetter moved back to the qu Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_records_reset) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100800` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100900` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38158,9 +38208,9 @@ sum by (domain) (increase(src_dbworker_resetter_record_resets_total{domain=~"^ba Number of stalled jobs that exhausted their retries and were marked as failed, per queue. A nonzero value means some agent messages or wakes were permanently dropped after repeated crashes. -This panel has no related alerts. +Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_record_reset_failures) for 2 alerts related to this panel. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100801` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100901` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38184,7 +38234,7 @@ sum by (domain) (increase(src_dbworker_resetter_record_reset_failures_total{doma This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100802` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100902` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38210,7 +38260,7 @@ Rate of per-message stream events (turn deltas, tool calls, completion, etc.) su This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100900` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=101000` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38234,7 +38284,7 @@ Rate of per-message stream events that could not be published to Redis after ret Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_message_stream_publish_errors) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100901` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=101001` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38258,7 +38308,7 @@ Rate of Redis subscription transport errors (a subscription died mid-stream). Su This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100902` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=101002` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38282,7 +38332,7 @@ Rate of thread-level stream events (new message created, title updated) successf This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100910` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=101010` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38304,9 +38354,9 @@ sum by (event_type) (rate(src_batch_change_agents_thread_stream_publish_success_ Rate of thread-level stream events that could not be published to Redis after retries. Clients recover thread state on their next refetch. -This panel has no related alerts. +Refer to the [alerts reference](alerts#batch-change-agents-batch_change_agent_thread_stream_publish_errors) for 1 alert related to this panel. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100911` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=101011` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.* @@ -38330,7 +38380,7 @@ Rate of Redis subscription transport errors on the thread-level stream. This panel has no related alerts. -To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=100912` on your Sourcegraph instance. +To see this panel, visit `/-/debug/grafana/d/batch-change-agents/batch-change-agents?viewPanel=101012` on your Sourcegraph instance. *Managed by the Sourcegraph Code Plane team.*