langgraph-api 0.16.0.dev7__tar.gz → 0.16.0.dev8__tar.gz

{langgraph_api-0.16.0.dev7 → langgraph_api-0.16.0.dev8}/PKG-INFO

@@ -1,10 +1,11 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.16.0.dev7
+Version: 0.16.0.dev8
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
 Requires-Python: >=3.11
+Requires-Dist: anyio>=4.14.2
 Requires-Dist: cloudpickle>=3.0.0
 Requires-Dist: cryptography>=50.0.0
 Requires-Dist: grpcio-health-checking<1.82.0,>=1.81.0

{langgraph_api-0.16.0.dev7 → langgraph_api-0.16.0.dev8}/benchmark/README.md

@@ -192,9 +192,18 @@ be compared on the bytes they occupy, which is what the feature exists to reduce
 2. Fill in `deploymentId` and `baseUrl` in `CAPACITY_DEPLOYMENTS` at the top of [`capacity-ci.js`](capacity-ci.js).
 3. Scheduled run: two parallel CI jobs (one per deployment), aggregate job posts Slack summary.
 
-Each cell also records the deployment's own view of its window (queue and API pod counts, CPU peaks) from the control plane's deployment metrics route, and the posted table lists it under each deployment. Every tier starts at one queue pod and autoscales during a cell, so a capacity number is partly a pod-count number; pinning pods at the tier maximum would need the benchmark deployments recreated as `external_docker` sources, the only kind whose `resource_spec` the API accepts.
+Each cell also records the deployment's own view of its window (queue and API pod counts, CPU peaks) from the control plane's deployment metrics route. The posted message names any component at or over 95% of its CPU request as that tier's bottleneck, and the full per-cell table goes to the message's thread. Every tier starts at one queue pod and autoscales during a cell, so a capacity number is partly a pod-count number; pinning pods at the tier maximum would need the benchmark deployments recreated as `external_docker` sources, the only kind whose `resource_spec` the API accepts.
 
-Both the daily and the capacity reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`), flagging a figure only when it leaves that runner's own observed range.
+### Slack reports
+
+Every report follows one shape so the channel reads the same way each morning, on a phone as well as a desktop:
+
+- The first line is the verdict: a dot (green quiet, yellow something regressed or a bottleneck or errors, red something failed), the report name and date, `manual run` when dispatched by hand, the API version, and a clause saying what needs attention. The second line says what the benchmark is, for a reader outside the team.
+- Then `Failed`, `Regressed`, `Improved` and `Bottlenecks` or `Errors` lines, worst first. `Regressed` prints `none` rather than going quiet.
+- One table per deployment, about 40 characters wide so it fits a phone, with `▲ better` / `▼ worse` markers on cells that left their usual range.
+- One legend line (pass rule, arrows, dot colors), then a link to the thread, which holds the per-cell or per-run numbers and the longer *How to read*, and to the run's artifacts.
+
+Both reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`). A figure is flagged only when it leaves that cell's own observed range, and only once five same-kind nights exist; before that the table says how many nights it has. The daily benchmark posts one digest per workflow once its last runner finishes (`daily-report.js`, fed by each runner's artifact), not one message per runner.
 
 Local commands:
 

{langgraph_api-0.16.0.dev7 → langgraph_api-0.16.0.dev8}/benchmark/capacity-ci.js

@@ -18,23 +18,33 @@ import {
   requireApiKey,
   setResourceTiers,
 } from "./host-v2.js";
-import { cellKey, compare, formatDelta, loadCapacityHistory } from "./history.js";
-import { describeRunner, runnerShortName } from "./runner-descriptions.js";
+import { RUNNERS, readApiVersion } from "./daily-report.js";
 import {
-  cellWindow,
-  formatServerMetrics,
-  summarizeServerMetrics,
-} from "./server-metrics.js";
+  MIN_HISTORY_NIGHTS,
+  cellKey,
+  flaggedChange,
+  formatMove,
+  loadCapacityHistory,
+} from "./history.js";
+import {
+  describeRunnerBriefly,
+  runnerColumnName,
+  runnerGloss,
+  runnerShortName,
+} from "./runner-descriptions.js";
+import { cellWindow, peakCpu, podRange, summarizeServerMetrics } from "./server-metrics.js";
+import { dotFor, postReport } from "./slack.js";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 /**
  * @typedef {{startLoad?: string, stepSize?: string, numSteps?: string, warmupLoad?: string}} StaircaseGeometry
- * @type {Array<{slug: string, deploymentId: string, baseUrl: string, tiers: string[], staircase?: StaircaseGeometry}>}
+ * @type {Array<{slug: string, name: string, deploymentId: string, baseUrl: string, tiers: string[], staircase?: StaircaseGeometry}>}
  */
 const CAPACITY_DEPLOYMENTS = [
   {
     slug: "daily-cap-benchmark-dedicated",
+    name: "Dedicated",
     deploymentId: "22a22e9d-521e-4635-9bcb-a096a22820cc",
     baseUrl:
       "https://daily-cap-benchmark-dedicat-9669b63626045e0681f1f1fda08ef4ce.staging.langgraph.app",
@@ -45,6 +55,7 @@ const CAPACITY_DEPLOYMENTS = [
   },
   {
     slug: "daily-cap-benchmark-serverless",
+    name: "Serverless",
     deploymentId: "becfa8bb-e2e3-4d96-8d08-3191b17c5d4d",
     baseUrl:
       "https://daily-cap-benchmark-serverl-b5dc9d9f7fa05990b279939863ae08de.staging.langgraph.app",
@@ -56,8 +67,6 @@ const CAPACITY_DEPLOYMENTS = [
   },
 ];
 
-const RUNNERS = ["wait_write", "stream_write", "threads_search_metadata"];
-
 // Open model: a fixed arrival rate per step, so capacity reads directly as the
 // service rate. Closed-model users back off as the server slows and flatter it.
 const STAIRCASE_EXECUTOR = "constant-arrival-rate";
@@ -195,135 +204,354 @@ export function capacityVus(summary) {
   return perSec == null ? String(vus) : `${vus} (${perSec}/s)`;
 }
 
-export function formatAggregateTable(
-  deployments,
-  summaries,
-  warnings = [],
-  now = new Date(),
-  history = {},
-) {
+const BOTTLENECK_CPU_PCT = 95;
+const COMPONENTS = [
+  ["queue", "queue"],
+  ["api", "api"],
+  ["db", "database"],
+  ["redis", "redis"],
+];
+
+function deploymentName(deployment) {
+  return deployment.name ?? deployment.slug;
+}
+
+function tierLabel(deployment, tier) {
+  const prefix = `${deploymentName(deployment).toUpperCase()}_`;
+  return tier.startsWith(prefix) ? tier.slice(prefix.length) : tier;
+}
+
+function cellName(deployment, tier, runner) {
+  return `${deploymentName(deployment)} ${tierLabel(deployment, tier)} ${runnerColumnName(runner)}`;
+}
+
+function indexCells(deployments, summaries) {
   /** @type {Record<string, Record<string, Record<string, object>>>} */
   const byDeployment = Object.fromEntries(deployments.map((d) => [d.slug, {}]));
-
   for (const summary of summaries) {
-    const config = summary.config ?? {};
-    const { deploymentSlug, computeTier, benchmarkType } = config;
+    const { deploymentSlug, computeTier, benchmarkType } = summary.config ?? {};
     if (deploymentSlug && computeTier && benchmarkType) {
       byDeployment[deploymentSlug] ??= {};
       byDeployment[deploymentSlug][computeTier] ??= {};
       byDeployment[deploymentSlug][computeTier][benchmarkType] = summary;
     }
   }
+  return byDeployment;
+}
 
-  const expected = deployments.reduce(
-    (total, deployment) => total + deployment.tiers.length * RUNNERS.length,
-    0,
-  );
-  const failed = summaries.filter((summary) => summary.error).length;
-  const stepSeconds = (() => {
-    const seen = [...new Set(
-      summaries.map((s) => s.config?.plateauDurationSeconds).filter((v) => v != null).map(String),
-    )];
-    if (seen.length === 0) return STAIRCASE_PLATEAU_DURATION;
-    return seen.sort((a, b) => Number(a) - Number(b)).join("/");
-  })();
-  const missing = expected - summaries.length;
+function pastNights(deployment, tier, runner, summary, history) {
+  const executor = summary?.config?.k6Executor ?? "constant-vus";
+  // Nights measured in users and nights measured in req/s are different
+  // scales; only the same executor is a baseline.
+  return (history[cellKey({ deploymentSlug: deployment.slug, computeTier: tier, benchmarkType: runner })] ?? [])
+    .filter((night) => (night.executor ?? "constant-vus") === executor)
+    .map((night) => night.capacityVUs);
+}
+
+function analyseDeployment(deployment, cells, history) {
+  const rows = [];
+  let nights = 0;
+  for (const tier of deployment.tiers) {
+    for (const runner of RUNNERS) {
+      const summary = cells[tier]?.[runner];
+      const past = summary && !summary.error ? pastNights(deployment, tier, runner, summary, history) : [];
+      nights = Math.max(nights, past.length);
+      const change =
+        summary && !summary.error
+          ? flaggedChange(summary.capacity?.targetVUs ?? 0, past, { higherIsBetter: true })
+          : null;
+      rows.push({
+        deployment,
+        tier,
+        runner,
+        summary,
+        change,
+        hot: hotComponents(summary),
+      });
+    }
+  }
+  return { rows, nights };
+}
+
+function hotComponents(summary) {
+  const groups = summary?.serverMetricsAtCapacity?.groups ?? {};
+  return COMPONENTS.flatMap(([label, name]) => {
+    const cpu = peakCpu(groups[label]);
+    return cpu != null && cpu >= BOTTLENECK_CPU_PCT ? [{ name, cpu: Math.round(cpu) }] : [];
+  });
+}
+
+function bottleneckLines(rows) {
+  const groups = new Map();
+  for (const row of rows) {
+    for (const { name, cpu } of row.hot) {
+      const key = `${row.deployment.slug}|${name}`;
+      const group = groups.get(key) ?? { deployment: row.deployment, name, tiers: new Set(), cpus: [] };
+      group.tiers.add(tierLabel(row.deployment, row.tier));
+      group.cpus.push(cpu);
+      groups.set(key, group);
+    }
+  }
+  const order = (group) => COMPONENTS.findIndex(([, name]) => name === group.name);
+  return [...groups.values()].sort((a, b) => order(a) - order(b)).map(({ deployment, name, tiers, cpus }) => {
+    const low = Math.min(...cpus);
+    const high = Math.max(...cpus);
+    const range = low === high ? `${high}%` : `${low} to ${high}%`;
+    return `${deploymentName(deployment)} ${[...tiers].join(", ")}: ${name} at ${range}`;
+  });
+}
+
+function crashedTiers(rows) {
+  const byTier = new Map();
+  for (const row of rows) {
+    if (!row.summary?.error) continue;
+    const key = `${row.deployment.slug}|${row.tier}`;
+    (byTier.get(key) ?? byTier.set(key, { row, runners: [] }).get(key)).runners.push(row.runner);
+  }
+  return [...byTier.values()].map(({ row, runners }) => {
+    const where = `${deploymentName(row.deployment)} ${tierLabel(row.deployment, row.tier)}`;
+    const detail = runners.length === RUNNERS.length
+      ? `${where}, all ${RUNNERS.length} tests (ERR)`
+      : `${where} ${runners.map(runnerColumnName).join(", ")} (ERR)`;
+    return { where, detail };
+  });
+}
+
+function moveLine(row, unit) {
+  const now = row.summary.capacity?.targetVUs ?? 0;
+  return `${cellName(row.deployment, row.tier, row.runner)} ${formatMove(Math.round(row.change.median), now, row.change.deltaPct, unit)}`;
+}
+
+export function capacityCell(summary) {
+  if (!summary) return "-";
+  if (summary.error) return "ERR";
+  return String(summary.capacity?.targetVUs ?? 0);
+}
+
+function plural(n, word) {
+  return `${n} ${word}${n === 1 ? "" : "s"}`;
+}
+
+function stepSecondsOf(summaries) {
+  const seen = [...new Set(
+    summaries.map((s) => s.config?.plateauDurationSeconds).filter((v) => v != null).map(String),
+  )];
+  if (seen.length === 0) return STAIRCASE_PLATEAU_DURATION;
+  return seen.sort((a, b) => Number(a) - Number(b)).join("/");
+}
+
+/** A dispatch can override the configured geometry; the report describes the run. */
+function geometryOf(deployment, cells) {
+  const ran = ranGeometry(cells) ?? deployment.staircase ?? {};
+  return {
+    start: Number(ran.startLoad ?? STAIRCASE_START_LOAD),
+    step: Number(ran.stepSize ?? STAIRCASE_STEP_SIZE),
+  };
+}
+
+function capacityTable(deployment, rows, unit) {
+  const labels = deployment.tiers.map((tier) => tierLabel(deployment, tier));
+  const labelWidth = Math.max(2, ...labels.map((l) => l.length)) + 1;
+  const column = (text, arrow = " ") => `${text.padStart(6)}${arrow}`;
+  const lines = [
+    `*${deploymentName(deployment)}* · ${unit} at SLO`,
+    "```",
+    `${"".padEnd(labelWidth)}${RUNNERS.map((r) => column(runnerColumnName(r))).join("")}`,
+  ];
+  for (const tier of deployment.tiers) {
+    const cells = rows.filter((row) => row.tier === tier);
+    lines.push(
+      `${tierLabel(deployment, tier).padEnd(labelWidth)}${cells.map((row) => column(capacityCell(row.summary), row.change?.arrow)).join("")}`,
+    );
+  }
+  lines.push("```");
+  return lines;
+}
+
+export function formatAggregateTable(
+  deployments,
+  summaries,
+  warnings = [],
+  now = new Date(),
+  history = {},
+  apiVersion = null,
+  manual = false,
+) {
+  const byDeployment = indexCells(deployments, summaries);
+  const analysed = deployments.map((deployment) => ({
+    deployment,
+    ...analyseDeployment(deployment, byDeployment[deployment.slug] ?? {}, history),
+  }));
+  const rows = analysed.flatMap((a) => a.rows);
+
+  const expected = rows.length;
+  const failed = rows.filter((row) => row.summary?.error).length;
+  const missing = rows.filter((row) => !row.summary).length;
+  const jobFailed = process.env.CAPACITY_JOB_RESULT === "failure";
+  const regressed = rows.filter((row) => row.change && !row.change.better);
+  const improved = rows.filter((row) => row.change?.better);
+  const bottlenecks = bottleneckLines(rows);
+  const checked = analysed.some((a) => a.nights >= MIN_HISTORY_NIGHTS);
+  const allWarnings = [...warnings, ...toolOverheadWarnings(summaries)];
 
   const executor = executorOf(summaries);
-  const open = executor === OPEN_MODEL;
   const unit = unitFor(executor);
+  const date = now.toLocaleDateString("en-US", { month: "short", day: "numeric", timeZone: "UTC" });
+  const ran = failed + missing === 0 ? `all ${expected} cells ran` : `${expected - failed - missing} of ${expected} cells ran`;
+  const verdict = [];
+  const crashed = crashedTiers(rows);
+  if (crashed.length > 0) verdict.push(`${crashed.map((tier) => tier.where).join(", ")} crashed`);
+  if (missing > 0) verdict.push(`${missing} never ran`);
+  if (jobFailed) verdict.push("job failed");
+  if (regressed.length > 0) verdict.push(plural(regressed.length, "regression"));
+  if (improved.length > 0) verdict.push(plural(improved.length, "improvement"));
+  if (bottlenecks.length > 0) verdict.push(plural(bottlenecks.length, "bottleneck"));
+  if (verdict.length === 0) verdict.push("all quiet");
+
+  const dot = dotFor({
+    failed: failed + missing > 0 || jobFailed,
+    attention: regressed.length + bottlenecks.length + allWarnings.length > 0,
+  });
   const lines = [
-    `*Daily Capacity Benchmark* · ${now.toISOString().slice(0, 10)} · ${expected - missing - failed}/${expected} cells`,
-    "*Goal:* how much load each tier serves before it breaks the SLO, and the runs/sec it delivers there. Compares tiers; not a customer workload.",
-    `*How:* ${open ? "the offered request rate" : "the number of users"} climbs in steps of ${stepSeconds}s. A step passes at success >= ${SLO.minSuccessRate}%, p50 <= ${SLO.maxP50Ms / 1000}s, p95 <= ${SLO.maxP95Ms / 1000}s. Capacity = the top passing step, with the step below also passing.${open ? " Requests the tool had to drop count as failures." : ""}`,
-    "*Graph:* `default`, 1 step, no LLM, 1KB checkpoint. Server overhead, not a customer workload.",
-    `*Tests:* ${RUNNERS.map((runner) => `${runnerShortName(runner)} = ${describeRunner(runner)}`).join(" · ")}`,
-    `*Reading a cell:* \`400 (398/s)\` held 400 ${unit} at SLO, serving 398 runs/sec · \`0 (~46/s ceiling)\` no step met SLO, best rate any step served · \`ERR\` crashed, no data · \`-\` never ran`,
-    "*Pods:* every tier starts at one queue pod and grows during a cell, so a low number can mean few pods rather than a slow server. Pod counts and CPU peaks are listed under each table where the control plane returned them.",
+    `${dot} *Capacity · ${date}*${manual ? " · manual run" : ""}${apiVersion ? ` · API ${apiVersion}` : ""} · ${ran} · ${verdict.join(", ")}`,
+    describeCapacity(analysed, summaries, executor),
   ];
 
-  // Say out loud when the run was not clean. Without this a run where half the
-  // matrix never executed posts a tidy-looking table and reads as authoritative.
-  const problems = [];
-  if (failed > 0) problems.push(`${failed} failed`);
-  if (missing > 0) problems.push(`${missing} never ran`);
-  if (process.env.CAPACITY_JOB_RESULT === "failure") {
-    problems.push("the capacity job itself failed");
+  const failedParts = crashed.map((tier) => tier.detail);
+  for (const row of rows) {
+    if (!row.summary) failedParts.push(`${cellName(row.deployment, row.tier, row.runner)} never ran`);
   }
-  if (problems.length > 0) {
-    lines.push(
-      `:warning: ${problems.join(", ")} of ${expected} cells - treat this table as incomplete`,
-    );
+  if (jobFailed) failedParts.push("the capacity job exited with failure");
+  if (failedParts.length > 0) {
+    lines.push(`*Failed:* ${failedParts.join(" · ")} · treat the table as incomplete`);
+  }
+
+  if (!checked) {
+    const nights = Math.max(...analysed.map((a) => a.nights));
+    lines.push(`*Regressed:* not checked, ${nights} of ${MIN_HISTORY_NIGHTS} nights of history`);
+  } else {
+    lines.push(`*Regressed:* ${regressed.length > 0 ? regressed.map((row) => moveLine(row, unit)).join(" · ") : "none"}`);
+    if (improved.length > 0) lines.push(`*Improved:* ${improved.map((row) => moveLine(row, unit)).join(" · ")}`);
+  }
+  lines.push(`*Bottlenecks:* ${bottlenecks.length > 0 ? bottlenecks.join(" · ") : "none reported"}`);
+  for (const warning of allWarnings) lines.push(`:warning: ${warning}`);
+
+  for (const { deployment, rows: deploymentRows, nights } of analysed) {
+    lines.push("", ...capacityTable(deployment, deploymentRows, unitFor(executorOf(deploymentRows.map((r) => r.summary).filter(Boolean)))));
+    if (checked && nights < MIN_HISTORY_NIGHTS) {
+      lines.push(`_${nights} of ${MIN_HISTORY_NIGHTS} nights of history, no change flags yet_`);
+    }
+  }
+
+  lines.push(
+    "",
+    `▲ better ▼ worse than the last ${MIN_HISTORY_NIGHTS}+ nights · \`0\` no step passed · \`ERR\` crashed · \`-\` never ran · 🟢 quiet · 🟡 regression or bottleneck · 🔴 cells failed`,
+  );
+  return lines.join("\n");
+}
+
+function describeCapacity(analysed, summaries, executor) {
+  const open = executor === OPEN_MODEL;
+  const unit = unitFor(executor);
+  const climb = analysed
+    .map(({ deployment, rows }) => {
+      const cells = indexCells([deployment], rows.map((r) => r.summary).filter(Boolean))[deployment.slug] ?? {};
+      const { start, step } = geometryOf(deployment, cells);
+      return `+${step} from ${start} (${deploymentName(deployment).toLowerCase()})`;
+    })
+    .join(" or ");
+  const where = analysed.length === 1 ? "one staging deployment" : `${analysed.length} staging deployments`;
+  return (
+    `k6 ${open ? "open-model" : "closed-model"} staircase, ${where} resized per tier: ${open ? "request rate" : "virtual users"} ${climb} every ${stepSecondsOf(summaries)} s, per path: ${RUNNERS.map(runnerGloss).join(", ")}. ` +
+    `Cell = highest ${unit} where success ≥ ${SLO.minSuccessRate}%, p50 ≤ ${SLO.maxP50Ms / 1000} s, p95 ≤ ${SLO.maxP95Ms / 1000} s held for two steps. ` +
+    "One-step graph, no LLM: server ceiling, not customer traffic."
+  );
+}
+
+export function howToRead() {
+  return [
+    "*How to read*",
+    `• *Tests:* ${RUNNERS.map((r) => `${runnerColumnName(r)} = ${describeRunnerBriefly(r)}`).join(" · ")}.`,
+    `• *Regressed / Improved:* a cell outside its own usual range over the last ${MIN_HISTORY_NIGHTS}+ nights measured the same way.`,
+    `• *Bottleneck:* a component at or over ${BOTTLENECK_CPU_PCT}% of its CPU request while the tier held its top step. It is what stops the tier going higher.`,
+    "• Graph is one step, no LLM, so this is server overhead, not a customer workload.",
+  ].join("\n");
+}
+
+function servedRate(summary) {
+  const rate = summary?.capacity?.successfulRunsPerSec;
+  if (rate != null) return String(Math.round(rate));
+  const ceiling = summary?.throughputCeiling?.successfulRunsPerSec;
+  return ceiling != null ? `0 (ceiling ${Math.round(ceiling)})` : "-";
+}
+
+function serverColumns(summaries) {
+  const columns = [];
+  for (const [label, name] of COMPONENTS) {
+    const groups = summaries.map((s) => s?.serverMetrics?.groups?.[label]).filter(Boolean);
+    if (groups.length === 0) continue;
+    if (groups.some(podRange)) columns.push({ label, kind: "pods", width: 7 });
+    if (groups.some((g) => peakCpu(g) != null)) columns.push({ label, kind: "cpu", width: 6 });
   }
-  for (const warning of [...warnings, ...toolOverheadWarnings(summaries)]) {
-    lines.push(`:warning: ${warning}`);
+  return columns;
+}
+
+function serverCell(summary, column) {
+  const group = summary?.serverMetrics?.groups?.[column.label];
+  if (column.kind === "pods") {
+    const range = podRange(group);
+    return range ? `${range.min}→${range.max}` : "-";
   }
-  lines.push("");
+  const cpu = peakCpu(group);
+  return cpu != null ? String(Math.round(cpu)) : "-";
+}
 
+export function formatServerView(deployments, summaries) {
+  const byDeployment = indexCells(deployments, summaries);
+  const blocks = [];
   for (const deployment of deployments) {
     const cells = byDeployment[deployment.slug] ?? {};
-    const geometry = ranGeometry(cells) ?? deployment.staircase ?? {};
-    const start = Number(geometry.startLoad ?? STAIRCASE_START_LOAD);
-    const step = Number(geometry.stepSize ?? STAIRCASE_STEP_SIZE);
-    const steps = Number(geometry.numSteps ?? STAIRCASE_NUM_STEPS);
-    const dedicated = deployment.tiers.some((tier) => tier.startsWith("DEDICATED_"));
-    const cellExecutor = executorOf(Object.values(cells).flatMap((byRunner) => Object.values(byRunner)));
-    const mode = cellExecutor === OPEN_MODEL ? "open model, fixed request rate" : "closed model, fixed users";
-    lines.push(
-      `*${deployment.slug}* · ${dedicated ? "compute and database resized per tier" : "compute resized per tier, one shared database tier"} · ${start} to ${start + step * (steps - 1)} ${unitFor(cellExecutor)}, +${step} per step · ${mode}`,
-    );
-    lines.push("```");
-    lines.push(
-      `${"".padEnd(16)}${RUNNERS.map((runner) => runnerShortName(runner).padEnd(20)).join("")}`,
-    );
-    for (const tier of deployment.tiers) {
-      let row = tier.padEnd(16);
-      for (const runner of RUNNERS) {
-        const summary = cells[tier]?.[runner];
-        row += capacityVus(summary).padEnd(20);
-      }
-      lines.push(row);
+    const present = Object.values(cells).flatMap((byRunner) => Object.values(byRunner));
+    if (present.length === 0) continue;
+    const served = deployment.tiers
+      .map((tier) => `${tierLabel(deployment, tier)} ${RUNNERS.map((r) => servedRate(cells[tier]?.[r])).join(" / ")}`)
+      .join(" · ");
+    const servedLine = `Runs/sec actually served at the top step, as ${RUNNERS.map(runnerColumnName).join(" / ")}: ${served}`;
+    // The served rate is k6's; the pod and CPU columns are the dashboard's, which is best effort.
+    const columns = serverColumns(present);
+    if (columns.length === 0) {
+      blocks.push(`*${deploymentName(deployment)}*\n${servedLine}`);
+      continue;
+    }
+    const labelWidth = Math.max(...deployment.tiers.map((t) => tierLabel(deployment, t).length)) + 8;
+    const spans = [];
+    for (const column of columns) {
+      const last = spans[spans.length - 1];
+      if (last && last.label === column.label) last.width += column.width;
+      else spans.push({ label: column.label, width: column.width });
     }
-    lines.push("```");
+    const lines = [
+      `*${deploymentName(deployment)}*`,
+      "```",
+      `${"".padEnd(labelWidth)}${spans.map((s) => s.label.padStart(s.width)).join("")}`,
+      `${"".padEnd(labelWidth)}${columns.map((c) => c.kind.padStart(c.width)).join("")}`,
+    ];
     for (const tier of deployment.tiers) {
       for (const runner of RUNNERS) {
-        const served = formatServerMetrics(cells[tier]?.[runner]?.serverMetrics);
-        if (served) lines.push(`\`${tier} ${runnerShortName(runner)}\`: ${served}`);
-      }
-    }
-    const drift = historyLine(deployment, cells, history);
-    if (drift) lines.push(drift);
-    lines.push("");
-  }
-  return lines.join("\n").trimEnd();
-}
-
-function historyLine(deployment, cells, history) {
-  const moved = [];
-  let nights = 0;
-  for (const tier of deployment.tiers) {
-    for (const runner of RUNNERS) {
-      const summary = cells[tier]?.[runner];
-      if (!summary || summary.error) continue;
-      const executor = summary.config?.k6Executor ?? "constant-vus";
-      // Nights measured in users and nights measured in req/s are different
-      // scales; only the same executor is a baseline.
-      const past = (history[cellKey({ deploymentSlug: deployment.slug, computeTier: tier, benchmarkType: runner })] ?? [])
-        .filter((night) => (night.executor ?? "constant-vus") === executor);
-      const verdict = compare(summary.capacity?.targetVUs ?? 0, past.map((p) => p.capacityVUs));
-      if (!verdict) continue;
-      nights = Math.max(nights, verdict.nights);
-      if (verdict.outside) {
-        moved.push(
-          `${tier} ${runnerShortName(runner)} ${summary.capacity?.targetVUs ?? 0} vs median ${Math.round(verdict.median)} (${formatDelta(verdict.deltaPct)}, usual ±${verdict.spreadPct.toFixed(0)}%)`,
-        );
+        const summary = cells[tier]?.[runner];
+        if (!summary?.serverMetrics) continue;
+        const label = `${tierLabel(deployment, tier)} ${runnerColumnName(runner)}`;
+        lines.push(`${label.padEnd(labelWidth)}${columns.map((c) => serverCell(summary, c).padStart(c.width)).join("")}`);
       }
     }
+    lines.push("```", servedLine);
+    blocks.push(lines.join("\n"));
   }
-  if (nights === 0) return null;
-  return moved.length > 0
-    ? `*vs last ${nights} nights:* ${moved.join(" · ")}`
-    : `*vs last ${nights} nights:* every cell within its usual range`;
+  if (blocks.length === 0) return null;
+  return [
+    "*Pods and CPU per cell*, peak over the whole cell including the steps that failed. Pods are start → end. CPU is % of the pod's request, so over 100 means bursting above it.",
+    ...blocks,
+  ].join("\n\n");
 }
 
 function toolOverheadWarnings(summaries) {
@@ -651,12 +879,20 @@ async function runDeploymentMatrix(deployment, apiKey) {
   return { summaryPaths, failures, warnings };
 }
 
+// The steps past capacity saturate everything, so a bottleneck read over the
+// whole cell names a component the tier never hit while it was passing.
 async function attachServerMetrics(deployment, apiKey, summaryPath) {
   const summary = JSON.parse(readFileSync(summaryPath, "utf8"));
   const window = cellWindow(summary.steps);
   if (!window) throw new Error("steps carry no timestamps");
-  const section = await getDeploymentMetrics(deployment.deploymentId, apiKey, window);
-  summary.serverMetrics = summarizeServerMetrics(section);
+  summary.serverMetrics = summarizeServerMetrics(
+    await getDeploymentMetrics(deployment.deploymentId, apiKey, window),
+  );
+  const capacityStep = summary.capacity?.step;
+  const held = capacityStep ? cellWindow(summary.steps.filter((s) => s.step <= capacityStep)) : null;
+  summary.serverMetricsAtCapacity = held
+    ? summarizeServerMetrics(await getDeploymentMetrics(deployment.deploymentId, apiKey, held))
+    : null;
   writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
 }
 
@@ -675,49 +911,26 @@ function collectSummaries(directory) {
   );
 }
 
-async function postSlack(message) {
-  const token = process.env.SLACK_BOT_TOKEN;
-  const channel = process.env.SLACK_CHANNEL;
-  if (!token || !channel) {
-    console.log(
-      "SLACK_BOT_TOKEN or SLACK_CHANNEL not set; skipping Slack post",
-    );
-    console.log(message);
-    return;
-  }
-
-  const response = await fetch("https://slack.com/api/chat.postMessage", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json; charset=utf-8",
-      Authorization: `Bearer ${token}`,
-    },
-    body: JSON.stringify({ channel, text: message }),
-  });
-  const result = await response.json();
-  if (!result.ok) {
-    throw new Error(`Slack API error: ${result.error ?? "unknown"}`);
-  }
-  console.log("Posted aggregate summary to Slack");
-}
-
 async function runAggregate(summariesDir, { postToSlack, historyDir }) {
   const summaries = collectSummaries(summariesDir);
   const warnings = collectWarnings(summariesDir);
-  const message = formatAggregateTable(
+  const history = historyDir ? loadCapacityHistory(historyDir) : {};
+  let text = formatAggregateTable(
     CAPACITY_DEPLOYMENTS,
     summaries,
     warnings,
     new Date(),
-    historyDir ? loadCapacityHistory(historyDir) : {},
+    history,
+    process.env.API_VERSION || readApiVersion(),
+    process.env.MANUAL === "true",
   );
-  console.log(message);
+  const artifactUrl = process.env.GITHUB_RUN_URL;
+  text += `\n*Thread:* pods and CPU per cell, and how to read${artifactUrl ? ` · <${artifactUrl}|Artifacts:> raw k6 summaries` : ""}`;
+  const thread = [[formatServerView(CAPACITY_DEPLOYMENTS, summaries), howToRead()].filter(Boolean).join("\n\n")];
   if (postToSlack) {
-    const artifactUrl = process.env.GITHUB_RUN_URL;
-    const slackMessage = artifactUrl
-      ? `${message}\n\n<${artifactUrl}|Download artifacts>`
-      : message;
-    await postSlack(slackMessage);
+    await postReport({ text, thread });
+  } else {
+    for (const part of [text, ...thread]) console.log(`${part}\n`);
   }
 }