langgraph-api 0.18.0.dev10__tar.gz → 0.18.0.dev12__tar.gz

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.18.0.dev10
+Version: 0.18.0.dev12
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
@@ -24,7 +24,7 @@ Requires-Dist: langsmith[otel]>=0.6.3
 Requires-Dist: mcp<3,>=2.3.0
 Requires-Dist: opentelemetry-api>=0.0.1
 Requires-Dist: opentelemetry-exporter-otlp-proto-http>=0.0.1
-Requires-Dist: opentelemetry-exporter-prometheus<0.64,>=0.63b1
+Requires-Dist: opentelemetry-exporter-prometheus<0.67,>=0.63b1
 Requires-Dist: opentelemetry-sdk>=0.0.1
 Requires-Dist: orjson>=3.9.7
 Requires-Dist: packaging>=23.2

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/REPORTS.md

@@ -16,13 +16,24 @@ in the same channel is separate and not covered here.
 - **Failed / Errors / Regressed / Bottlenecks** lines only appear when there is
   something to say, worst first.
 - **Regressed / Improved.** A figure is flagged only when it leaves its own
-  usual range over the recent nights: the 9 before tonight for the daily and
-  delta reports, the last 7 for capacity, where ▲ better and ▼ worse also
-  mark the cell in the table. The usual range is half the spread
+  usual range over the recent nights: the 9 latest on tonight's CPUs for the
+  daily reports, the 9 before tonight for delta, the 7 latest on tonight's CPUs
+  for capacity, where ▲ better and ▼ worse also mark the cell in the table. The usual range is half the spread
   between the best and worst of those nights, and never less than 10%, so a
   series that swings 40% night to night is not flagged for moving 15%.
   Flags start once five nights of history exist; until then the post says how
   many it has.
+- **Machines.** Staging hands out machines with CPUs of different
+  generations, and the oldest (Intel Broadwell) runs the same code about 1.5x
+  slower than AMD Rome. After each run, 20 short runs ask the benchmark graph
+  which CPU its workers are on (capacity asks after every step and keeps every
+  worker it saw), so the check never warms a deployment before it is timed. The first line of the daily reports names it
+  (`cpus differ by runner` when the paths got different ones), a `New machine:`
+  line says when a path or tier moved since the night before, and flags only
+  compare nights on the same CPUs, so a night on a slower machine starts its
+  own baseline instead of reading as a regression. A deployment with several
+  workers is compared by the set of CPUs they ran on; the thread gives the mix.
+  Nights from before this check only compare with each other.
 - **The thread** holds the full numbers: per-run or per-cell detail and
   improvements with before and after values; the daily threads also give the
   load that ran.
@@ -84,7 +95,11 @@ it is shown but never flagged).
 
 The daily benchmark against a second deployment that scales to zero when
 idle. Scheduled runs redeploy it first, so the first minute includes the cold
-start; a manual run skips the redeploy unless it is asked for. 200 users per
+start; a manual run skips the redeploy unless it is asked for. Its pods must
+run on their cell's application machines (host-backend's
+`HOST_DEV_ZERO_REQUIRED_CELL_POOL_PROJECT_IDS`, set for this deployment and the
+serverless capacity one); other scale-to-zero deployments only prefer them and
+also land on Redis machines or in another cell. 200 users per
 path instead of the daily's 1,000 (300 for search), and the pass bar is p95
 under 25 s to allow for the cold start. Read it like the [daily benchmark](#daily-benchmark).
 
@@ -94,7 +109,8 @@ under 25 s to allow for the cold start. Read it like the [daily benchmark](#dail
 before it breaks its latency target, and what runs out first?
 
 **How it runs.** Two staging deployments, one dedicated and one serverless,
-are resized to each tier (L, M, S) in turn. For each tier and path, k6 sends a
+get a new revision on scheduled runs and are then resized to each tier
+(L, M, S) in turn. For each tier and path, k6 sends a
 fixed request rate regardless of how fast the server answers (an "open
 model", the way real traffic arrives), and raises it every 60 s: dedicated by
 50 req/s starting at 50, serverless by 5 starting at 10.
@@ -118,7 +134,8 @@ number.
 **Thread.** Improvements with before and after values, then per deployment
 two small tables grouped by tier: peak CPU as % of the pod's request, and pods
 at the start and end of the cell. Both cover the whole cell, including steps
-that failed. Last, the request rate actually served at the top step.
+that failed. Then the request rate actually served at the top step, and the
+CPU each tier ran on (`Machines:`, per path where a tier's paths differ).
 
 ## Delta channel
 
@@ -162,7 +179,8 @@ what the run returned.
 
 **Thread.** Turn, write, read and data at turns 350, 700, 1050 and 1400, with
 `gain`: how many times better delta is (`same` within 15%, below `1.0x` when
-delta is worse). Then the run's size, failures and correctness checks.
+delta is worse). Then the run's size, failures, correctness checks and the
+CPUs the workers ran on.
 
 ## Changing a report
 

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/capacity-ci.js

@@ -22,9 +22,11 @@ import { RUNNERS } from "./daily-report.js";
 import {
   MIN_HISTORY_NIGHTS,
   cellKey,
+  cpuSignature,
   flaggedChange,
   formatMove,
   loadCapacityHistory,
+  sameCpu,
 } from "./history.js";
 import { runnerColumnName, runnerShortName } from "./runner-descriptions.js";
 import { cellWindow, peakCpu, podRange, summarizeServerMetrics } from "./server-metrics.js";
@@ -229,15 +231,33 @@ function indexCells(deployments, summaries) {
   return byDeployment;
 }
 
-function pastNights(deployment, tier, runner, summary, history) {
+function sameExecutorNights(deployment, tier, runner, summary, history) {
   const executor = summary?.config?.k6Executor ?? "constant-vus";
   // Nights measured in users and nights measured in req/s are different
   // scales; only the same executor is a baseline.
   return (history[cellKey({ deploymentSlug: deployment.slug, computeTier: tier, benchmarkType: runner })] ?? [])
-    .filter((night) => (night.executor ?? "constant-vus") === executor)
+    .filter((night) => (night.executor ?? "constant-vus") === executor);
+}
+
+// The baseline stays a week of nights on tonight's cpu; the workflow fetches more so a week can be found.
+const PAST_NIGHTS = 7;
+
+function pastNights(deployment, tier, runner, summary, history) {
+  return sameCpu(sameExecutorNights(deployment, tier, runner, summary, history), summary?.serverCpus)
+    .slice(0, PAST_NIGHTS)
     .map((night) => night.capacityVUs);
 }
 
+/** Tonight's CPUs against the newest comparable night's, when both are known and differ. */
+function newMachine(row, history) {
+  if (!row.summary || row.summary.error) return null;
+  const tonight = cpuSignature(row.summary.serverCpus);
+  const before = cpuSignature(sameExecutorNights(row.deployment, row.tier, row.runner, row.summary, history)[0]?.serverCpus);
+  return tonight && before && tonight !== before
+    ? `${deploymentName(row.deployment)} ${tierLabel(row.deployment, row.tier)} on ${tonight} (was ${before})`
+    : null;
+}
+
 function analyseDeployment(deployment, cells, history) {
   const rows = [];
   let nights = 0;
@@ -404,12 +424,16 @@ export function formatAggregateTable(
   if (reasons.size > 0) lines.push(`*Why:* ${[...reasons].join(" · ")}`);
   if (regressed.length > 0) lines.push(`*Regressed:* ${regressed.map((row) => moveLine(row, unit)).join(" · ")}`);
   if (bottlenecks.length > 0) lines.push(`*Bottlenecks:* ${bottlenecks.join(" · ")}`);
+  // A tier's runners share its pods, so one line per tier.
+  const moved = [...new Set(rows.map((row) => newMachine(row, history)).filter(Boolean))];
+  if (moved.length > 0) lines.push(`*New machine:* ${moved.join(" · ")} · compared only with nights on the same cpu`);
   for (const warning of allWarnings) lines.push(`:warning: ${warning}`);
 
   for (const { deployment, rows: deploymentRows, nights } of analysed) {
     lines.push("", ...capacityTable(deployment, deploymentRows, unitFor(executorOf(deploymentRows.map((r) => r.summary).filter(Boolean)))));
     if (nights < MIN_HISTORY_NIGHTS) {
-      lines.push(`_${nights} of ${MIN_HISTORY_NIGHTS} nights of history, no change flags yet_`);
+      const cpuKnown = deploymentRows.some((row) => cpuSignature(row.summary?.serverCpus));
+      lines.push(`_${nights} of ${MIN_HISTORY_NIGHTS} nights of history${cpuKnown ? " on this cpu" : ""}, no change flags yet_`);
     }
   }
   return lines.join("\n");
@@ -473,6 +497,17 @@ function serverTable(deployment, cells, kind) {
   return lines.join("\n");
 }
 
+/** "Machines: L amd rome · M intel broadwell", per runner only where a tier's runners differ. */
+function machines(deployment, cells) {
+  const parts = [];
+  for (const tier of deployment.tiers) {
+    const cpus = RUNNERS.map((r) => cpuSignature(cells[tier]?.[r]?.serverCpus) ?? "-");
+    if (cpus.every((cpu) => cpu === "-")) continue;
+    parts.push(`${tierLabel(deployment, tier)} ${cpus.every((cpu) => cpu === cpus[0]) ? cpus[0] : cpus.join(" / ")}`);
+  }
+  return parts.length > 0 ? `Machines: ${parts.join(" · ")}` : null;
+}
+
 export function formatServerView(deployments, summaries) {
   const byDeployment = indexCells(deployments, summaries);
   const blocks = [];
@@ -484,6 +519,7 @@ export function formatServerView(deployments, summaries) {
       .map((tier) => `${tierLabel(deployment, tier)} ${RUNNERS.map((r) => servedRate(cells[tier]?.[r])).join(" / ")}`)
       .join(" · ");
     const servedLine = `Runs/sec actually served at the top step, as ${RUNNERS.map(runnerColumnName).join(" / ")}: ${served}`;
+    const machinesLine = machines(deployment, cells);
     // The served rate is k6's; the pod and CPU tables are the dashboard's, which is best effort.
     const cpu = serverTable(deployment, cells, "cpu");
     const pods = serverTable(deployment, cells, "pods");
@@ -493,6 +529,7 @@ export function formatServerView(deployments, summaries) {
         cpu ? `_CPU, % of request_\n${cpu}` : null,
         pods ? `_Pods, start → end_\n${pods}` : null,
         servedLine,
+        machinesLine,
       ]
         .filter(Boolean)
         .join("\n"),

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/capacity-ci.test.js

@@ -440,3 +440,43 @@ test("a crashed cell's multi-line error keeps its detail on the Why line", () =>
   const table = formatAggregateTable(deployments, [buildErrorSummary({ deploymentSlug: "dep", computeTier: "TIER_A", benchmarkType: "wait_write" }, error)]);
   assert.match(table, /^\*Why:\* PATCH \/resource-tiers failed: 409 \{ "detail": "Unable to resize the deployment database\." \}$/m);
 });
+
+const ROME = { "AuthenticAMD 143/49": 1 };
+const BROADWELL = { "GenuineIntel 6/79": 1 };
+const nightsOn = (serverCpus, ...values) => nights(...values).map((night) => ({ ...night, serverCpus }));
+
+test("a cell that moved to an older cpu says so and is not compared with the faster cpu's nights", () => {
+  const deployments = [{ slug: "serverless", name: "Serverless", tiers: ["SERVERLESS_M"] }];
+  const tonight = (targetVUs, serverCpus) => ({
+    capacity: { targetVUs },
+    serverCpus,
+    config: { deploymentSlug: "serverless", computeTier: "SERVERLESS_M", benchmarkType: "wait_write", k6Executor: "constant-arrival-rate" },
+  });
+  const history = { "serverless|SERVERLESS_M|wait_write": nightsOn(ROME, 40, 40, 40, 40, 40) };
+
+  const moved = formatAggregateTable(deployments, [tonight(25, BROADWELL)], [], NIGHT, history);
+  assert.doesNotMatch(moved, /Regressed:/, "against the rome nights this would be -37%");
+  assert.match(moved, /^\*New machine:\* Serverless M on intel broadwell \(was amd rome\) · compared only with nights on the same cpu$/m);
+  assert.match(moved, /^_0 of 5 nights of history on this cpu, no change flags yet_$/m);
+
+  const stayed = formatAggregateTable(deployments, [tonight(25, ROME)], [], NIGHT, history);
+  assert.match(stayed, /^\*Regressed:\* Serverless M wait 40 → 25 req\/s \(-37%\)$/m, "the same drop on the same cpu is still a regression");
+  assert.doesNotMatch(stayed, /New machine/);
+});
+
+test("the thread names each tier's cpu, per runner only where they differ", () => {
+  const deployments = [{ slug: "serverless", name: "Serverless", tiers: ["SERVERLESS_L", "SERVERLESS_S"] }];
+  const cell = (tier, benchmarkType, serverCpus) => ({
+    capacity: { targetVUs: 30, successfulRunsPerSec: 29.5 },
+    serverCpus,
+    config: { deploymentSlug: "serverless", computeTier: tier, benchmarkType, k6Executor: "constant-arrival-rate" },
+  });
+  const view = formatServerView(deployments, [
+    cell("SERVERLESS_L", "wait_write", ROME),
+    cell("SERVERLESS_L", "stream_write", ROME),
+    cell("SERVERLESS_L", "threads_search_metadata", ROME),
+    cell("SERVERLESS_S", "wait_write", BROADWELL),
+    cell("SERVERLESS_S", "stream_write", ROME),
+  ]);
+  assert.match(view, /^Machines: L amd rome · S intel broadwell \/ amd rome \/ -$/m);
+});

langgraph_api-0.18.0.dev12/benchmark/cpu-probe.js

@@ -0,0 +1,55 @@
+/*
+ * Which CPUs the deployment's workers run on, asked of the benchmark graph:
+ * every run returns the `host` and `host_cpu` of the worker that ran it.
+ * Staging hands out machines of several CPU generations and the oldest runs the
+ * same code about 1.5x slower, so the reports only compare nights that ran on
+ * the same kind. Callers ask from handleSummary, after the measured scenario,
+ * so the probe never warms a fresh scale-to-zero deployment before its cold
+ * start is timed, and it sees the workers a deployment added under load.
+ */
+import http from 'k6/http';
+
+// A dedicated deployment spreads runs over up to ten workers; twenty runs see most of them.
+const PROBE_RUNS = 20;
+// After an overloaded staircase step a run waits behind the backlog; the summary should not wait with it.
+const PROBE_BUDGET_MS = 30000;
+
+/** `{ "<host>": "<vendor family/model>" }` for each worker that answered. */
+export function probeServerCpus(baseUrl, apiKey) {
+  const headers = { 'Content-Type': 'application/json' };
+  if (apiKey) headers['x-api-key'] = apiKey;
+  const body = JSON.stringify({
+    assistant_id: 'benchmark',
+    input: {},
+    context: { steps: 1, expand: 1 },
+    config: { recursion_limit: 3 },
+  });
+  const deadline = Date.now() + PROBE_BUDGET_MS;
+  const cpuByHost = {};
+  for (let i = 0; i < PROBE_RUNS && Date.now() < deadline; i++) {
+    const response = http.post(`${baseUrl}/runs/wait`, body, {
+      headers,
+      timeout: Math.max(1000, deadline - Date.now()),
+      // Not part of the measurement, so a failed probe must not count against http_req_failed.
+      responseCallback: http.expectedStatuses({ min: 100, max: 599 }),
+    });
+    if (response.status !== 200) continue;
+    let output;
+    try {
+      output = response.json();
+    } catch (error) {
+      continue;
+    }
+    if (typeof output?.host === 'string' && typeof output?.host_cpu === 'string') {
+      cpuByHost[output.host] = output.host_cpu;
+    }
+  }
+  return cpuByHost;
+}
+
+/** `{ "<vendor family/model>": workers seen }`, or null when no worker answered. */
+export function countByCpu(cpuByHost) {
+  const counts = {};
+  for (const cpu of Object.values(cpuByHost ?? {})) counts[cpu] = (counts[cpu] ?? 0) + 1;
+  return Object.keys(counts).length > 0 ? counts : null;
+}

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/daily-report.js

@@ -8,7 +8,7 @@ import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
 import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
-import { MIN_HISTORY_NIGHTS, compare, flaggedChange, formatDelta, formatMove, loadDailyHistory, sparkline } from './history.js';
+import { MIN_HISTORY_NIGHTS, compare, cpuSignature, describeCpus, flaggedChange, formatDelta, formatMove, loadDailyHistory, sameCpu, sparkline } from './history.js';
 import { runnerColumnName, runnerShortName } from './runner-descriptions.js';
 import { dotFor, footer, postReport } from './slack.js';
 
@@ -241,6 +241,24 @@ export const NIGHTS = 10;
 
 const baseline = (run) => (run.history ?? []).slice(0, NIGHTS - 1);
 
+// The trend draws every night; the flags compare only nights on tonight's kinds of CPU.
+const comparable = (run) => sameCpu(run.history ?? [], run.summary?.metrics?.serverCpus).slice(0, NIGHTS - 1);
+
+const cpuOf = (run) => cpuSignature(run.summary?.metrics?.serverCpus);
+
+/** Tonight's CPUs against the newest past night's, when both are known and differ. */
+function newMachine(run) {
+  const tonight = cpuOf(run);
+  const before = cpuSignature(run.history?.[0]?.serverCpus);
+  return tonight && before && tonight !== before ? `${runName(run)} on ${tonight} (was ${before})` : null;
+}
+
+function cpuClause(runs) {
+  const known = runs.map(cpuOf).filter(Boolean);
+  if (known.length === 0) return '';
+  return known.every((cpu) => cpu === known[0]) ? ` · ${known[0]}` : ' · cpus differ by runner';
+}
+
 function p95Of(metrics) {
   return succeeded(metrics) && isNumber(metrics?.p95Duration) ? metrics.p95Duration : null;
 }
@@ -281,7 +299,7 @@ export function buildDigest(runs, context = {}) {
   } = context;
 
   const changesByRun = new Map(
-    runs.map((run) => [run, run.summary?.metrics ? figureChanges(run.summary.metrics, baseline(run)) : []])
+    runs.map((run) => [run, run.summary?.metrics ? figureChanges(run.summary.metrics, comparable(run)) : []])
   );
   const unhealthy = runs.filter((run) => !isHealthy({ stepOutcome: run.stepOutcome, metrics: run.summary?.metrics }));
   const withErrors = runs.filter((run) => hasErrors(run.summary?.metrics));
@@ -295,7 +313,7 @@ export function buildDigest(runs, context = {}) {
       if (moved.length > 0) list.push(`${runName(run)} ${moved.map((c, i) => (i === 0 ? `${c.label} ${c.move}` : `${c.label} ${roundedDelta(c.deltaPct)}`)).join(', ')}`);
     }
   }
-  const nights = Math.max(0, ...runs.map((run) => baseline(run).length));
+  const nights = Math.max(0, ...runs.map((run) => comparable(run).length));
   const checked = nights >= MIN_HISTORY_NIGHTS;
   const tools = runs.map((run) => [run, toolLine(run.summary?.metrics)]).filter(([, line]) => line);
 
@@ -312,7 +330,7 @@ export function buildDigest(runs, context = {}) {
   const date = now.toLocaleDateString('en-US', { month: 'short', day: 'numeric', timeZone: 'UTC' });
   const passed = runs.length - unhealthy.length;
   const lines = [
-    `${dot} *Daily Benchmark${titleSuffix} · ${date}*${manual ? ' · manual run' : ''}${apiVersion ? ` · API ${apiVersion}` : ''} · ${passed} of ${runs.length} passed · ${clauses.join(' · ')}`,
+    `${dot} *Daily Benchmark${titleSuffix} · ${date}*${manual ? ' · manual run' : ''}${apiVersion ? ` · API ${apiVersion}` : ''}${cpuClause(runs)} · ${passed} of ${runs.length} passed · ${clauses.join(' · ')}`,
   ];
   if (unhealthy.length > 0) {
     lines.push(`*Failed:* ${unhealthy.map((run) => `${runName(run)}: ${failureReason(run)}`).join(' · ')}`);
@@ -333,10 +351,12 @@ export function buildDigest(runs, context = {}) {
     );
   }
   if (regressed.length > 0) lines.push(`*Regressed:* ${regressed.join(' · ')}`);
+  const moved = runs.map(newMachine).filter(Boolean);
+  if (moved.length > 0) lines.push(`*New machine:* ${moved.join(' · ')} · compared only with nights on the same cpu`);
   for (const [run, line] of tools) lines.push(`:warning: ${runName(run)}: ${line}`);
 
   lines.push(...digestTable(runs));
-  if (!checked) lines.push(`_${nights} of ${MIN_HISTORY_NIGHTS} nights of history, no change flags yet_`);
+  if (!checked) lines.push(`_${nights} of ${MIN_HISTORY_NIGHTS} nights of history${runs.some(cpuOf) ? ' on this cpu' : ''}, no change flags yet_`);
   lines.push(footer(docsAnchor, artifactUrl));
 
   return { text: lines.join('\n'), thread: [perRunDetail(runs, changesByRun, improved, loadSentence(runs))] };
@@ -365,9 +385,10 @@ function perRunDetail(runs, changesByRun, improved, load) {
       lines.push(`${runName(run)}: ${noResults(run)}`);
       continue;
     }
-    const counts = `${count(metrics.totalRuns)} runs, ${count(metrics.failedRuns)} failed`;
+    const cpus = describeCpus(metrics.serverCpus);
+    const counts = `${count(metrics.totalRuns)} runs, ${count(metrics.failedRuns)} failed${cpus ? ` · ${cpus}` : ''}`;
     const changes = changesByRun.get(run);
-    const nights = baseline(run).length;
+    const nights = comparable(run).length;
     const drift =
       changes.length > 0
         ? `vs ${nights} nights: ${changes.map((c) => `${c.label} ${formatDelta(c.deltaPct)}${c.flagged ? ` ${c.arrow}` : ''}`).join(' · ')}`

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/daily-report.test.js

@@ -516,3 +516,53 @@ test('every table line in the channel message and the thread fits a phone', () =
   assert.ok(widestCodeLine(text) <= PHONE_WIDTH, `channel table is ${widestCodeLine(text)} wide`);
   assert.ok(widestCodeLine(thread[0]) <= PHONE_WIDTH, `thread table is ${widestCodeLine(thread[0])} wide`);
 });
+
+const ROME = { 'AuthenticAMD 143/49': 1 };
+const BROADWELL = { 'GenuineIntel 6/79': 1 };
+const on = (serverCpus) => () => ({ serverCpus });
+
+test('a slow night on an older cpu is compared only with nights on that cpu, so it is no regression', () => {
+  const past = [...nights([9.6, 9.9, 10.2], on(BROADWELL)), ...nights([4.5, 4.5, 4.5, 4.5, 4.5, 4.5], on(ROME))];
+  const tonight = { ...healthyMetrics, p95Duration: 10.1, serverCpus: BROADWELL };
+  const { text, thread } = buildDigest([run('wait_write', { summary: { metrics: tonight }, history: past })], base);
+  assert.match(text, /^🟢 \*Daily Benchmark · Aug 17\* · API 0\.14\.0 · intel broadwell · 1 of 1 passed · all quiet$/m);
+  assert.doesNotMatch(text, /Regressed:/, 'against the rome nights this would be +124%');
+  assert.match(text, /^_3 of 5 nights of history on this cpu, no change flags yet_$/m);
+  assert.match(thread[0], /^wait_write: 133,000 runs, 0 failed · intel broadwell · vs 3 nights: /m);
+});
+
+test('a regression on the same cpu is still flagged', () => {
+  const past = nights([4.5, 4.5, 4.5, 4.5, 4.5, 4.5], on(ROME));
+  const { text } = buildDigest([run('wait_write', { summary: { metrics: { ...healthyMetrics, p95Duration: 9.0, serverCpus: ROME } }, history: past })], base);
+  assert.match(text, /^\*Regressed:\* wait_write p95 4\.5 → 9\.0 s \(\+100%\)$/m);
+});
+
+test('a night on a different cpu than the night before says so instead of flagging', () => {
+  const past = nights([4.5, 4.5, 4.5, 4.5, 4.5, 4.5], on(ROME));
+  const { text } = buildDigest([run('stream_write', { summary: { metrics: { ...healthyMetrics, p95Duration: 9.0, serverCpus: BROADWELL } }, history: past })], base);
+  assert.match(text, /^\*New machine:\* stream_write on intel broadwell \(was amd rome\) · compared only with nights on the same cpu$/m);
+  assert.doesNotMatch(text, /Regressed:/);
+  assert.match(text, /^_0 of 5 nights of history on this cpu, no change flags yet_$/m);
+  assert.match(text, /^stream\s+9\.0 [▁-█]+$/m, 'the trend still draws every night');
+});
+
+test('nights before the probe keep comparing with each other', () => {
+  const past = nights([4.5, 4.5, 4.5, 4.5, 4.5, 4.5]);
+  const { text } = buildDigest([run('wait_write', { summary: { metrics: { ...healthyMetrics, p95Duration: 9.0 } }, history: past })], base);
+  assert.match(text, /^\*Regressed:\* wait_write p95 4\.5 → 9\.0 s/m);
+  assert.doesNotMatch(text, /New machine|intel|amd/);
+});
+
+test('runners on different cpus say so in the header and name their cpu in the thread', () => {
+  const mixed = { 'AuthenticAMD 143/49': 6, 'GenuineIntel 6/79': 4 };
+  const runs = [
+    run('wait_write', { summary: { metrics: { ...healthyMetrics, serverCpus: BROADWELL } } }),
+    run('stream_write', { summary: { metrics: { ...healthyMetrics, serverCpus: mixed } } }),
+  ];
+  const { text, thread } = buildDigest(runs, base);
+  assert.match(text, /· API 0\.14\.0 · cpus differ by runner · 2 of 2 passed/);
+  assert.match(thread[0], /^wait_write: 133,000 runs, 0 failed · intel broadwell · /m);
+  assert.match(thread[0], /^stream_write: 133,000 runs, 0 failed · 6 amd rome, 4 intel broadwell · /m);
+  const same = buildDigest([runs[1]], base).text;
+  assert.match(same, /· API 0\.14\.0 · amd rome \+ intel broadwell · 1 of 1 passed/);
+});

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/delta-report.js

@@ -10,7 +10,7 @@ import readline from 'node:readline';
 import { fileURLToPath } from 'node:url';
 
 import { ERROR_ATTENTION, MIN_SUCCESS_RATE, describeThreshold, errorParts, findLatestSummary, isHealthy, readApiVersion, readSummary } from './daily-report.js';
-import { MIN_HISTORY_NIGHTS, flaggedChange, formatMove, sparkline } from './history.js';
+import { MIN_HISTORY_NIGHTS, describeCpus, flaggedChange, formatMove, sparkline } from './history.js';
 import { dotFor, footer, postReport } from './slack.js';
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -345,8 +345,9 @@ function threadText(run, metrics, guard, improved) {
   const users = Number(run.load?.loadSize) || null;
   const turns = Number(run.load?.turns) || null;
   const shape = users && turns ? `${count(users)} users × ${count(turns)} turns × ${run.cells.length} setups · ` : '';
+  const cpus = describeCpus(metrics?.serverCpus);
   const health = metrics
-    ? `${count(metrics.failedRuns)} failed of ${count(metrics.totalRuns)} turns · ${guard > 0 ? `correctness check failed × ${count(guard)}` : 'no correctness failures'}`
+    ? `${count(metrics.failedRuns)} failed of ${count(metrics.totalRuns)} turns · ${guard > 0 ? `correctness check failed × ${count(guard)}` : 'no correctness failures'}${cpus ? ` · ${cpus}` : ''}`
     : 'no results';
   return [
     '*As the thread grows*',

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/delta-report.test.js

@@ -194,3 +194,8 @@ test('a single downloaded artifact is read from the results directory itself', a
   assert.equal(run.stepOutcome, 'success');
   assert.equal(run.cells.find((c) => c.cell === 'delta_50').buckets['1-50'].saveMeanMs, 30);
 });
+
+test('the run line names the cpus the workers ran on', () => {
+  const { thread } = buildReport(run({ summary: { serverCpus: { 'AuthenticAMD 143/49': 6, 'GenuineIntel 6/79': 4 } } }), base);
+  assert.match(thread[0], /^\*Run\* · .* · no correctness failures · 6 amd rome, 4 intel broadwell$/m);
+});

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/history.js

@@ -138,6 +138,7 @@ export function loadCapacityHistory(dir) {
       if (summary.error) continue;
       (history[cellKey(config)] ??= []).push({
         executor: config.k6Executor ?? "constant-vus",
+        serverCpus: summary.serverCpus ?? null,
         capacityVUs: summary.capacity?.targetVUs ?? 0,
         runsPerSec:
           summary.capacity?.successfulRunsPerSec ??
@@ -149,6 +150,59 @@ export function loadCapacityHistory(dir) {
   return history;
 }
 
+// The CPUs staging's E2 machines come with, keyed `vendor family/model` as cpu-probe.js records them.
+const CPU_NAMES = {
+  "GenuineIntel 6/63": "intel haswell",
+  "GenuineIntel 6/79": "intel broadwell",
+  "GenuineIntel 6/85": "intel skylake or cascade lake",
+  "AuthenticAMD 23/49": "amd rome",
+  "AuthenticAMD 25/1": "amd milan",
+};
+
+/** gVisor prints an AMD family as `(extended << 4) | 0xF`, 143 where Linux says 23. */
+function cpuName(key) {
+  const match = /^(\S+) (\d+)\/(\d+)$/.exec(key);
+  if (!match) return key;
+  const [, vendor, familyText, model] = match;
+  const raw = Number(familyText);
+  const family = raw > 0xf && (raw & 0xf) === 0xf ? 0xf + (raw >> 4) : raw;
+  const normalized = `${vendor} ${family}/${model}`;
+  return CPU_NAMES[normalized] ?? normalized;
+}
+
+function cpuCounts(serverCpus) {
+  const counts = {};
+  for (const [key, workers] of Object.entries(serverCpus ?? {})) {
+    const name = cpuName(key);
+    counts[name] = (counts[name] ?? 0) + (isNumber(workers) ? workers : 0);
+  }
+  return counts;
+}
+
+/** The kinds of CPU a night ran on, the key the reports compare nights by; null when unknown. */
+export function cpuSignature(serverCpus) {
+  const names = Object.keys(cpuCounts(serverCpus)).sort();
+  return names.length > 0 ? names.join(" + ") : null;
+}
+
+/** "amd rome" for one worker, "6 amd rome, 4 intel broadwell" for several. */
+export function describeCpus(serverCpus) {
+  const entries = Object.entries(cpuCounts(serverCpus)).sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+  if (entries.length === 0) return null;
+  const workers = entries.reduce((sum, [, n]) => sum + n, 0);
+  return workers === 1 ? entries[0][0] : entries.map(([name, n]) => `${n} ${name}`).join(", ");
+}
+
+/**
+ * The nights that ran on tonight's kinds of CPU. A night without the probe only
+ * matches a night without it, so reports keep their history until the probe
+ * starts answering.
+ */
+export function sameCpu(nights, serverCpus) {
+  const tonight = cpuSignature(serverCpus);
+  return nights.filter((night) => cpuSignature(night?.serverCpus) === tonight);
+}
+
 function median(values) {
   const sorted = [...values].sort((a, b) => a - b);
   const mid = Math.floor(sorted.length / 2);

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/history.test.js

@@ -8,6 +8,8 @@ import {
   MIN_HISTORY_NIGHTS,
   cellKey,
   compare,
+  cpuSignature,
+  describeCpus,
   flaggedChange,
   formatDelta,
   formatMove,
@@ -15,6 +17,7 @@ import {
   historyCandidates,
   loadCapacityHistory,
   loadDailyHistory,
+  sameCpu,
   sparkline,
 } from "./history.js";
 
@@ -112,7 +115,7 @@ test("capacity history groups nights by cell and skips crashed cells", () => {
   const key = cellKey({ deploymentSlug: "d", computeTier: "L", benchmarkType: "wait_write" });
   assert.deepEqual(history[key].map((n) => n.capacityVUs).sort((a, b) => a - b), [700, 1000]);
   const serverless = history[cellKey({ deploymentSlug: "s", computeTier: "L", benchmarkType: "wait_write" })];
-  assert.deepEqual(serverless, [{ executor: "constant-vus", capacityVUs: 0, runsPerSec: 46.3 }], "a cell with no passing step keeps its ceiling, and a night without an executor is closed-model");
+  assert.deepEqual(serverless, [{ executor: "constant-vus", serverCpus: null, capacityVUs: 0, runsPerSec: 46.3 }], "a cell with no passing step keeps its ceiling, and a night without an executor is closed-model");
 });
 
 test("capacity history remembers which executor measured each night", () => {
@@ -125,6 +128,45 @@ test("capacity history remembers which executor measured each night", () => {
   assert.equal(night.executor, "constant-arrival-rate");
 });
 
+test("capacity history remembers the CPUs each night ran on", () => {
+  const root = mkdtempSync(path.join(tmpdir(), "cap-cpu-"));
+  writeFileSync(path.join(runDir(root, 1), "staircase_summary_s.json"), JSON.stringify({
+    config: { deploymentSlug: "s", computeTier: "M", benchmarkType: "wait_write" },
+    capacity: { targetVUs: 40, successfulRunsPerSec: 39.6 },
+    serverCpus: { "GenuineIntel 6/79": 1 },
+  }));
+  const [night] = loadCapacityHistory(root)[cellKey({ deploymentSlug: "s", computeTier: "M", benchmarkType: "wait_write" })];
+  assert.equal(cpuSignature(night.serverCpus), "intel broadwell");
+});
+
+test("cpus are named the same inside and outside gVisor, and unknown ones keep their numbers", () => {
+  assert.equal(cpuSignature({ "AuthenticAMD 143/49": 1 }), "amd rome", "gVisor's AMD family 143 is Linux's 23");
+  assert.equal(cpuSignature({ "AuthenticAMD 23/49": 1 }), "amd rome");
+  assert.equal(cpuSignature({ "GenuineIntel 6/79": 1 }), "intel broadwell");
+  assert.equal(cpuSignature({ "GenuineIntel 6/207": 1 }), "GenuineIntel 6/207");
+  assert.equal(cpuSignature({ "AuthenticAMD 175/17": 1 }), "AuthenticAMD 25/17", "an unnamed AMD still gets the Linux family");
+  assert.equal(cpuSignature(null), null);
+  assert.equal(cpuSignature({}), null);
+});
+
+test("a mix of cpus is one signature, and the description counts the workers", () => {
+  const mixed = { "GenuineIntel 6/79": 4, "AuthenticAMD 143/49": 6 };
+  assert.equal(cpuSignature(mixed), "amd rome + intel broadwell");
+  assert.equal(describeCpus(mixed), "6 amd rome, 4 intel broadwell");
+  assert.equal(describeCpus({ "AuthenticAMD 143/49": 1 }), "amd rome", "one worker needs no count");
+  assert.equal(describeCpus(null), null);
+});
+
+test("only nights on the same cpus compare, and nights before the probe only compare with each other", () => {
+  const rome = { serverCpus: { "AuthenticAMD 143/49": 1 }, p95Duration: 4.5 };
+  const broadwell = { serverCpus: { "GenuineIntel 6/79": 1 }, p95Duration: 9.8 };
+  const before = { p95Duration: 5.1 };
+  const nights = [broadwell, rome, before, rome];
+  assert.deepEqual(sameCpu(nights, rome.serverCpus), [rome, rome]);
+  assert.deepEqual(sameCpu(nights, broadwell.serverCpus), [broadwell]);
+  assert.deepEqual(sameCpu(nights, null), [before]);
+});
+
 test("sparkline keeps noise mid-height, draws a real step full height, and leaves gaps for missing nights", () => {
   assert.equal(sparkline([5, 5.2, 4.9, 5.1, 5]), "▄▅▄▅▄");
   assert.equal(sparkline([12.8, 12.5, 4.6, 4.8]), "██▁▁");

{langgraph_api-0.18.0.dev10 → langgraph_api-0.18.0.dev12}/benchmark/ramp.js

@@ -2,6 +2,7 @@ import { sleep } from 'k6';
 import { Counter, Trend } from 'k6/metrics';
 import { Benchmarks } from './benchmark-runners/dist/benchmarks.js';
 import { get_profile } from './benchmark-runners/dist/benchmark_profiles.js';
+import { countByCpu, probeServerCpus } from './cpu-probe.js';
 
 // Custom metrics
 const runDuration = new Trend('run_duration');
@@ -199,6 +200,8 @@ export function handleSummary(data) {
   const successfulRuns = data.metrics.successful_runs?.values?.count || 0;
   const failedRuns = data.metrics.failed_runs?.values?.count || 0;
   const totalRuns = successfulRuns + failedRuns;
+  const serverCpus = countByCpu(probeServerCpus(BASE_URL, LANGSMITH_API_KEY));
+  console.log(`Server CPUs (workers seen): ${JSON.stringify(serverCpus)}`);
 
   const summary = {
     startTimestamp: data.setup_data.startTime,
@@ -218,6 +221,8 @@ export function handleSummary(data) {
           : null,
       clientOverheadMs: trendMs(data.metrics.client_overhead),
       httpReqBlockedP95Ms: data.metrics.http_req_blocked?.values?.["p(95)"] ?? null,
+      // In metrics, not beside them, so every report's history carries it without another reader.
+      serverCpus,
       errors: {
         timeout: data.metrics.timeout_errors ? data.metrics.timeout_errors.values.count : 0,
         connection: data.metrics.connection_errors ? data.metrics.connection_errors.values.count : 0,