fix(supervisor): count pods from a limit=1 list instead of an aggregate metric (#4442)

The pod-count backpressure source read
`apiserver_storage_objects{resource="pods"}` from an apiserver
`/metrics` scrape. That gauge is a periodically-refreshed cached count,
and it is served by whichever apiserver replica the scrape lands on —
replicas disagree with each other at the same instant, by enough to
swamp the engage/release hysteresis band. Engage and release timing was
therefore partly a function of scrape routing.

This replaces it with a single `limit=1` list of the workload namespace
and computes `remainingItemCount + items.length`. One pod object
transferred, no informer, no watch cache.

Two request-shape constraints are load-bearing and called out in the
code: passing a label or field selector makes the apiserver omit
`remainingItemCount` entirely, and setting `resourceVersion` serves a
cached count rather than a quorum read. Neither is passed.

`remainingItemCount` is only set when the list is truncated, so
`_continue` is the truncation signal — if it is absent the returned page
is the whole collection and `items.length` is already exact. If the list
*is* truncated and the count is missing or implausible, the fetcher
throws rather than guessing.

Failure semantics are unchanged: a throw lands in the monitor's existing
catch, exactly as the previous parse did. The hysteresis, verdict shape,
and gauge are untouched. RBAC is unchanged — the existing role already
grants `pods: list`.

The `/metrics` non-resource grant in the deployment role becomes unused,
and the scrape-timeout env var is now a slight misnomer. Both left alone
deliberately: the grant may be wanted again for other apiserver signals,
and renaming the var would need a coordinated config change for no
behavioural gain.

Tests cover the not-truncated, truncated, missing-count, negative-count
and timeout paths.

Author: nicktrn

.server-changes/pod-count-exact-list.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: improvement
+---
+
+Self-hosted Kubernetes deployments now measure running-task count more accurately when deciding whether to pause pulling new work, so the safeguard engages closer to its configured thresholds.

apps/supervisor/src/backpressure/k8sPodCountSignalSource.test.ts

@@ -1,52 +1,49 @@
 import { describe, it, expect } from "vitest";
-import { parsePodCount, K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
+import { K8sPodCountSignalSource } from "./k8sPodCountSignalSource.js";
+import { podCountFromList, withTimeout } from "../clients/kubernetes.js";
 
-describe("parsePodCount", () => {
-  it("reads the pods object count", () => {
-    const text = [
-      "# HELP apiserver_storage_objects Number of stored objects",
-      "# TYPE apiserver_storage_objects gauge",
-      'apiserver_storage_objects{resource="pods"} 8421',
-      'apiserver_storage_objects{resource="configmaps"} 17',
-    ].join("\n");
-    expect(parsePodCount(text)).toBe(8421);
+describe("podCountFromList", () => {
+  it("returns items.length when the list is not truncated", () => {
+    expect(podCountFromList({ items: [{}], metadata: {} })).toBe(1);
   });
 
-  it("is tolerant of extra labels in any order", () => {
-    const text = 'apiserver_storage_objects{group="",resource="pods",extra="x"} 12';
-    expect(parsePodCount(text)).toBe(12);
+  it("returns zero for an empty namespace", () => {
+    expect(podCountFromList({ items: [], metadata: {} })).toBe(0);
   });
 
-  it("parses scientific notation", () => {
-    const text = 'apiserver_storage_objects{resource="pods"} 1.2e+04';
-    expect(parsePodCount(text)).toBe(12000);
+  it("adds remainingItemCount when the list is truncated", () => {
+    const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: 24492 } };
+    expect(podCountFromList(list)).toBe(24493);
   });
 
-  it("throws when the pods metric is absent", () => {
-    const text = 'apiserver_storage_objects{resource="configmaps"} 17';
-    expect(() => parsePodCount(text)).toThrow(/not found/);
+  it("throws when truncated but remainingItemCount is absent", () => {
+    const list = { items: [{}], metadata: { _continue: "tok" } };
+    expect(() => podCountFromList(list)).toThrow(/remainingItemCount/);
   });
 
-  it("throws on a non-finite value (e.g. 1e999)", () => {
-    const text = 'apiserver_storage_objects{resource="pods"} 1e999';
-    expect(() => parsePodCount(text)).toThrow();
+  it("throws when truncated but remainingItemCount is negative", () => {
+    const list = { items: [{}], metadata: { _continue: "tok", remainingItemCount: -1 } };
+    expect(() => podCountFromList(list)).toThrow(/remainingItemCount/);
   });
+});
 
-  it("throws on a negative value", () => {
-    const text = 'apiserver_storage_objects{resource="pods"} -5';
-    expect(() => parsePodCount(text)).toThrow();
+describe("withTimeout", () => {
+  it("rejects once the deadline passes", async () => {
+    await expect(withTimeout(new Promise(() => {}), 10, "pod count list")).rejects.toThrow(
+      /timed out/
+    );
   });
-});
 
-function metrics(count: number): string {
-  return `apiserver_storage_objects{resource="pods"} ${count}`;
-}
+  it("passes a value through when it settles first", async () => {
+    await expect(withTimeout(Promise.resolve(7), 1000, "pod count list")).resolves.toBe(7);
+  });
+});
 
 describe("K8sPodCountSignalSource", () => {
   it("engages at the engage threshold and reports the count", async () => {
     const counts: number[] = [];
     const source = new K8sPodCountSignalSource({
-      fetchMetrics: async () => metrics(10000),
+      fetchPodCount: async () => 10000,
       engageThreshold: 10000,
       releaseThreshold: 5000,
       reportPodCount: (c) => counts.push(c),
@@ -59,7 +56,7 @@ describe("K8sPodCountSignalSource", () => {
 
   it("does not engage below the engage threshold", async () => {
     const source = new K8sPodCountSignalSource({
-      fetchMetrics: async () => metrics(9999),
+      fetchPodCount: async () => 9999,
       engageThreshold: 10000,
       releaseThreshold: 5000,
     });
@@ -69,7 +66,7 @@ describe("K8sPodCountSignalSource", () => {
   it("stays engaged in the hysteresis band, releases only below release threshold", async () => {
     let count = 10000;
     const source = new K8sPodCountSignalSource({
-      fetchMetrics: async () => metrics(count),
+      fetchPodCount: async () => count,
       engageThreshold: 10000,
       releaseThreshold: 5000,
     });
@@ -82,9 +79,9 @@ describe("K8sPodCountSignalSource", () => {
     expect((await source.read()).engaged).toBe(false); // band again -> stays off
   });
 
-  it("propagates scrape failures (monitor fails open on throw)", async () => {
+  it("propagates fetch failures (monitor fails open on throw)", async () => {
     const source = new K8sPodCountSignalSource({
-      fetchMetrics: async () => {
+      fetchPodCount: async () => {
         throw new Error("connection refused");
       },
       engageThreshold: 10000,

apps/supervisor/src/backpressure/k8sPodCountSignalSource.ts

@@ -1,22 +1,7 @@
 import type { BackpressureSignalSource, BackpressureVerdict } from "./backpressureMonitor.js";
 
-// Reads the apiserver's stored-pod-object count from a Prometheus /metrics scrape.
-const POD_COUNT_RE = /^apiserver_storage_objects\{[^}]*resource="pods"[^}]*\}\s+([0-9.eE+]+)/m;
-
-export function parsePodCount(metricsText: string): number {
-  const match = metricsText.match(POD_COUNT_RE);
-  if (!match) {
-    throw new Error('apiserver_storage_objects{resource="pods"} not found in metrics');
-  }
-  const value = Number(match[1]);
-  if (!Number.isFinite(value)) {
-    throw new Error(`unparseable pod count: ${match[1]}`);
-  }
-  return value;
-}
-
 export type K8sPodCountSignalSourceOptions = {
-  fetchMetrics: () => Promise<string>;
+  fetchPodCount: () => Promise<number>;
   engageThreshold: number;
   releaseThreshold: number;
   reportPodCount?: (count: number) => void;
@@ -29,8 +14,7 @@ export class K8sPodCountSignalSource implements BackpressureSignalSource {
   constructor(private readonly opts: K8sPodCountSignalSourceOptions) {}
 
   async read(): Promise<BackpressureVerdict> {
-    const text = await this.opts.fetchMetrics();
-    const count = parsePodCount(text);
+    const count = await this.opts.fetchPodCount();
     this.opts.reportPodCount?.(count);
 
     if (this.engaged) {

apps/supervisor/src/clients/kubernetes.ts

@@ -2,7 +2,6 @@ import * as k8s from "@kubernetes/client-node";
 import type { Informer, KubernetesObject, ListPromise } from "@kubernetes/client-node";
 import { assertExhaustive } from "@trigger.dev/core/utils";
 import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
-import * as https from "node:https";
 
 export const RUNTIME_ENV = process.env.KUBERNETES_PORT ? "kubernetes" : "local";
 
@@ -54,55 +53,90 @@ function getKubeConfig() {
 export { k8s };
 
 /**
- * Builds a function that scrapes the apiserver's Prometheus /metrics endpoint.
- * One lightweight aggregate read - not a pod listing. Requires the service
- * account to be granted GET on the /metrics non-resource URL.
+ * createPodCountFetcher sizes a namespace's pod collection with a single `limit=1`
+ * list: one pod transferred, no informer, no watch cache.
+ *
+ * This is an ESTIMATE, not an exact count. Kubernetes documents `remainingItemCount`
+ * as intended for estimating collection size and reserves the right not to set it or
+ * make it exact. Counting exactly would mean paginating the whole collection, which is
+ * what this deliberately avoids. Treat the value as a tight estimate from a quorum read
+ * at request time, and set thresholds with that in mind.
+ *
+ * Two request-shape constraints, both load-bearing. A label or field selector makes
+ * the apiserver omit `remainingItemCount` entirely, and setting `resourceVersion`
+ * serves a cached count instead of a quorum read - so neither is passed.
  */
-export function createApiserverMetricsFetcher(timeoutMs: number): () => Promise<string> {
-  const kubeConfig = getKubeConfig();
+export function createPodCountFetcher(
+  api: K8sApi,
+  namespace: string,
+  timeoutMs: number
+): () => Promise<number> {
+  const serverTimeoutSeconds = Math.max(1, Math.floor(timeoutMs / 1000));
+  let pending: Promise<unknown> | undefined;
 
   return async () => {
-    const cluster = kubeConfig.getCurrentCluster();
-    if (!cluster) {
-      throw new Error("no current cluster in kubeconfig");
+    if (pending) {
+      throw new Error("pod count list still in flight from a previous tick");
     }
-    const url = new URL(`${cluster.server}/metrics`);
-    const opts: https.RequestOptions = {
-      method: "GET",
-      protocol: url.protocol,
-      hostname: url.hostname,
-      port: url.port,
-      path: url.pathname,
-    };
-    // applyToHTTPSOptions sets the cluster CA, client cert/key, and auth headers
-    // (incl. exec plugins) on the request - so TLS verifies against the cluster
-    // CA, not the system store. The fetch-options path attaches the CA as an
-    // https.Agent, which global fetch (undici) ignores.
-    await kubeConfig.applyToHTTPSOptions(opts);
-
-    return new Promise<string>((resolve, reject) => {
-      const req = https.request(opts, (res) => {
-        const status = res.statusCode ?? 0;
-        let body = "";
-        res.setEncoding("utf8");
-        res.on("data", (chunk) => {
-          body += chunk;
-        });
-        res.on("end", () => {
-          if (status >= 200 && status < 300) {
-            resolve(body);
-          } else {
-            reject(new Error(`apiserver /metrics scrape failed: ${status}`));
-          }
-        });
-      });
-      // Without this a hung connect/TLS/read never settles, and the monitor's
-      // refreshInFlight guard would freeze the source (silent fail-open).
-      req.setTimeout(timeoutMs, () => {
-        req.destroy(new Error(`apiserver /metrics scrape timed out after ${timeoutMs}ms`));
-      });
-      req.on("error", reject);
-      req.end();
+
+    const request = api.core.listNamespacedPod({
+      namespace,
+      limit: 1,
+      timeoutSeconds: serverTimeoutSeconds,
     });
+
+    pending = request
+      .catch(() => {})
+      .finally(() => {
+        pending = undefined;
+      });
+
+    return podCountFromList(await withTimeout(request, timeoutMs, "pod count list"));
   };
 }
+
+/**
+ * podCountFromList turns a `limit=1` pod list into a population estimate.
+ *
+ * `remainingItemCount` is only set when the list is truncated, so `_continue` is the
+ * truncation signal: absent means the returned page is the whole collection and its
+ * length is exact. When truncated the total leans on `remainingItemCount`, which is
+ * documented as an estimate - so the result is an estimate too. Truncated without a
+ * usable count is unknowable, so it throws rather than returning a low number the
+ * caller would act on.
+ */
+export function podCountFromList(list: {
+  items: unknown[];
+  metadata?: { _continue?: string; remainingItemCount?: number };
+}): number {
+  if (!list.metadata?._continue) {
+    return list.items.length;
+  }
+
+  const remaining = list.metadata.remainingItemCount;
+  if (typeof remaining !== "number" || !Number.isFinite(remaining) || remaining < 0) {
+    throw new Error("pod list truncated but remainingItemCount absent or invalid");
+  }
+
+  return list.items.length + remaining;
+}
+
+/**
+ * withTimeout rejects if `promise` outlives `timeoutMs`, so a hung request cannot
+ * freeze the caller. It cannot cancel: the k8s client threads no AbortSignal through to
+ * fetch, so an abandoned request keeps running. Callers must therefore also bound the
+ * request server-side (`timeoutSeconds`) and refuse to start a second one while the
+ * first is pending, or a blackholed connection accumulates one socket per attempt.
+ */
+export function withTimeout<T>(promise: Promise<T>, timeoutMs: number, what: string): Promise<T> {
+  let timer: NodeJS.Timeout;
+  const deadline = new Promise<never>((_resolve, reject) => {
+    timer = setTimeout(
+      () => reject(new Error(`${what} timed out after ${timeoutMs}ms`)),
+      timeoutMs
+    );
+    timer.unref();
+  });
+
+  return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
+}

apps/supervisor/src/index.ts

@@ -21,7 +21,7 @@ import {
   CheckpointClient,
   isKubernetesEnvironment,
 } from "@trigger.dev/core/v3/serverOnly";
-import { createK8sApi, createApiserverMetricsFetcher } from "./clients/kubernetes.js";
+import { createK8sApi, createPodCountFetcher } from "./clients/kubernetes.js";
 import { collectDefaultMetrics, Counter, Gauge, Histogram } from "prom-client";
 import { register } from "./metrics.js";
 import { PodCleaner } from "./services/podCleaner.js";
@@ -276,14 +276,16 @@ class ManagedSupervisor {
       // RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here.
       const podCountGauge = new Gauge({
         name: "supervisor_cluster_pod_count",
-        help: "Total pod objects stored in the cluster, scraped for backpressure",
+        help: "Pod objects in the workload namespace, counted for backpressure",
         registers: [register],
       });
       this.backpressureMonitors.push(
         new BackpressureMonitor({
           enabled: true,
           source: new K8sPodCountSignalSource({
-            fetchMetrics: createApiserverMetricsFetcher(
+            fetchPodCount: createPodCountFetcher(
+              createK8sApi(),
+              env.KUBERNETES_NAMESPACE,
               env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS
             ),
             engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,