langgraph-api 0.17.0.dev3__tar.gz → 0.17.0.dev4__tar.gz

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/LICENSE

@@ -90,4 +90,4 @@ these terms.
 
 **use** means anything you do with the software requiring one of your licenses.
 
-**trademark** means trademarks, service marks, and similar rights.
+**trademark** means trademarks, service marks, and similar rights.

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/Makefile

@@ -38,6 +38,10 @@ check-base-imports:
 # test commands
 
 TEST ?= tests/ ../runtime_inmem/tests/
+# Pin rootdir/config to this package. TEST includes ../runtime_inmem/tests/,
+# and in the monorepo that climbs past api/ to the root pyproject.toml which
+# has no [tool.pytest] — dropping pythonpath (e.g. apply_prerelease_version).
+PYTEST_ROOT_ARGS := --rootdir=. -c pyproject.toml
 PYTEST_WORKERS ?= 0
 AUTH_TEST ?= "tests/integration_tests/test_custom_auth.py"
 LANGGRAPH_HTTP ?= {"disable_mcp": false, "disable_a2a": false}
@@ -105,7 +109,7 @@ REVISION ?= $(shell git rev-parse --short HEAD)
 LANGGRAPH_ENCRYPTION ?=
 
 test-license-oss:
-	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_WEBHOOKS='$(LANGGRAPH_WEBHOOKS)' LANGGRAPH_HTTP='$(HTTP_CONFIG)' LANGGRAPH_STORE='$(STORE_CONFIG)' LANGGRAPH_CHECKPOINTER='$(CHECKPOINTER_CONFIG)' LANGGRAPH_ENCRYPTION='$(LANGGRAPH_ENCRYPTION)' REDIS_URI=_FAKE DATABASE_URI=:memory: MIGRATIONS_PATH=__inmem__ uv run pytest -v $(XDIST_ARGS) $(TEST)
+	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_WEBHOOKS='$(LANGGRAPH_WEBHOOKS)' LANGGRAPH_HTTP='$(HTTP_CONFIG)' LANGGRAPH_STORE='$(STORE_CONFIG)' LANGGRAPH_CHECKPOINTER='$(CHECKPOINTER_CONFIG)' LANGGRAPH_ENCRYPTION='$(LANGGRAPH_ENCRYPTION)' REDIS_URI=_FAKE DATABASE_URI=:memory: MIGRATIONS_PATH=__inmem__ uv run pytest -v $(PYTEST_ROOT_ARGS) $(XDIST_ARGS) $(TEST)
 
 test-watch-oss:
 	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_WEBHOOKS='$(LANGGRAPH_WEBHOOKS)' LANGGRAPH_HTTP='$(HTTP_CONFIG)' LANGGRAPH_ENCRYPTION='$(LANGGRAPH_ENCRYPTION)' REDIS_URI=_FAKE DATABASE_URI=:memory: MIGRATIONS_PATH=__inmem__ uv run --no-sync ptw . -- -x -vv --ff --capture=no $(TEST)
@@ -113,13 +117,13 @@ test-watch-oss:
 test: test-license-oss
 test-watch: test-watch-oss
 unit-test:
-	LANGGRAPH_RUNTIME_EDITION="inmem" DATABASE_URI="test" REDIS_URI="test" uv run pytest tests/unit_tests
+	LANGGRAPH_RUNTIME_EDITION="inmem" DATABASE_URI="test" REDIS_URI="test" uv run pytest $(PYTEST_ROOT_ARGS) tests/unit_tests
 
 unit-test-watch:
 	LANGGRAPH_RUNTIME_EDITION="inmem" DATABASE_URI="test" REDIS_URI="test" uv run ptw . -- -x -vv --ff --capture=no tests/unit_tests
 
 test-auth:
-	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_AUTH='{"path": "./tests/graphs/fastapi_jwt_auth.py:auth"}' REDIS_URI=_FAKE DATABASE_URI=:memory: MIGRATIONS_PATH=__inmem__ uv run pytest -v $(XDIST_ARGS) $(AUTH_TEST)
+	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_AUTH='{"path": "./tests/graphs/fastapi_jwt_auth.py:auth"}' REDIS_URI=_FAKE DATABASE_URI=:memory: MIGRATIONS_PATH=__inmem__ uv run pytest -v $(PYTEST_ROOT_ARGS) $(XDIST_ARGS) $(AUTH_TEST)
 
 test-with-server:
 	@bash -euo pipefail -c '$(MAKE) $(START_TARGET) LANGGRAPH_HTTP='\''$(LANGGRAPH_HTTP)'\'' & \
@@ -146,7 +150,7 @@ define FASTAPI_JWT_AUTH_TEST
 	DATABASE_URI=:memory: \
 	MIGRATIONS_PATH=__inmem__ \
 	LANGGRAPH_HTTP='{"app": "./tests/graphs/fastapi_jwt_middleware_ordering.py:app", "middleware_order": "$(1)", "enable_custom_route_auth": $(2)}' \
-	uv run pytest -v $(AUTH_TEST)
+	uv run pytest -v $(PYTEST_ROOT_ARGS) $(AUTH_TEST)
 endef
 
 test-auth-fastapi-jwt--before-custom-middleware--no-custom-route-auth:
@@ -171,7 +175,7 @@ test-auth-langsmith-api-keys:
 	REDIS_URI=_FAKE \
 	DATABASE_URI=:memory: \
 	MIGRATIONS_PATH=__inmem__ \
-	uv run pytest -v tests/integration_tests/test_langsmith_parallel_auth.py
+	uv run pytest -v $(PYTEST_ROOT_ARGS) tests/integration_tests/test_langsmith_parallel_auth.py
 
 # dev commands
 LANGGRAPH_DISABLE_FILE_PERSISTENCE ?= true

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.17.0.dev3
+Version: 0.17.0.dev4
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
@@ -28,7 +28,7 @@ Requires-Dist: opentelemetry-sdk>=0.0.1
 Requires-Dist: orjson>=3.9.7
 Requires-Dist: prometheus-client>=0.0.1
 Requires-Dist: protobuf<7.0.0,>=6.32.1
-Requires-Dist: pyjwt>=2.13.0
+Requires-Dist: pyjwt>=2.15.0
 Requires-Dist: sse-starlette<4.0.0,>=2.1.3
 Requires-Dist: starlette>=1.3.1
 Requires-Dist: structlog<26,>=24.1.0

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/README.md

@@ -129,13 +129,11 @@ by replaying them. That moves cost from write to read, and the read cost grows w
 thread depth. `delta_turns` measures it: one thread per VU, one turn per iteration,
 `turn_duration` tagged by depth bucket.
 
-`benchmark-delta.yml` runs four cells nightly, 20 users for 25 minutes each, 1 KB appended per turn (`CHECKPOINT_SIZE`): plain channel, then
-delta snapshotting every 50 (`benchmark_delta_freq`, the deepagents setting), 500
-(`benchmark_delta_500`) and 1000 (`benchmark_delta`, the library default). `delta-report.js` posts
-one row per cell (write, read and storage at the deepest turn every cell reached) and puts
-the by-depth tables in the thread. The messages and migration cells stay manual. Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
-with depth for different reasons, the control by re-writing a growing value and delta by
-replaying more writes, so the deliverable is which curve is worse and where they cross.
+`benchmark-delta.yml` runs two cells nightly in one k6 run (`DELTA_CELLS`): every VU turns a
+plain-channel thread and a `benchmark_delta_messages` thread (a message list, snapshot every 50,
+the deepagents setting) in rotation, `ITERATIONS` fixed so both reach 1,400 turns.
+`delta-report.js` posts write, read and storage at the last turn and puts the by-depth tables
+in the thread; `REPORTS.md` explains the post. The cells below stay available for manual runs.
 
 | Cell | Env |
 |---|---|
@@ -143,7 +141,7 @@ replaying more writes, so the deliverable is which curve is worse and where they
 | delta, snapshot every 1000 | `GRAPH_ID=benchmark_delta` |
 | delta, snapshot every 50 | `GRAPH_ID=benchmark_delta_freq` |
 | delta, snapshot every 500 | `GRAPH_ID=benchmark_delta_500` |
-| delta on a message list | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
+| delta on a message list, snapshot every 50 | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
 | migration | `GRAPH_ID=benchmark_delta MIGRATE_FROM_GRAPH_ID=benchmark MIGRATE_AFTER=200` |
 
 ```bash
@@ -190,9 +188,9 @@ be compared on the bytes they occupy, which is what the feature exists to reduce
 
 ## CI workflows
 
-- Daily ramp benchmark: `.github/workflows/benchmark.yml`
-- Daily capacity matrix: `.github/workflows/benchmark-capacity-matrix.yml`
-- Daily delta-channel cells: `.github/workflows/benchmark-delta.yml`, reported by `delta-report.js`
+- Daily ramp benchmark: `.github/workflows/langgraph-api-benchmark.yml`
+- Daily capacity matrix: `.github/workflows/langgraph-api-benchmark-capacity-matrix.yml`
+- Daily delta-channel cells: `.github/workflows/langgraph-api-benchmark-delta.yml`, reported by `delta-report.js`
 
 The staging deployments build their graphs from
 [langgraph-server-benchmarks](https://github.com/langchain-ai/langgraph-server-benchmarks), not

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/REPORTS.md

@@ -16,8 +16,8 @@ in the same channel is separate and not covered here.
 - **Failed / Errors / Regressed / Bottlenecks** lines only appear when there is
   something to say, worst first.
 - **Regressed / Improved.** A figure is flagged only when it leaves its own
-  usual range over the recent nights: the 9 before tonight for the daily
-  reports, the last 7 for capacity and delta, where ▲ better and ▼ worse also
+  usual range over the recent nights: the 9 before tonight for the daily and
+  delta reports, the last 7 for capacity, where ▲ better and ▼ worse also
   mark the cell in the table. The usual range is half the spread
   between the best and worst of those nights, and never less than 10%, so a
   series that swings 40% night to night is not flagged for moving 15%.
@@ -101,8 +101,10 @@ model", the way real traffic arrives), and raises it every 60 s: dedicated by
 
 **The table.** Each cell is the highest request rate the tier held for two
 steps while staying on target: at least 99% success, p50 at most 3 s, p95 at
-most 10 s. `0` means no step met the target, `ERR` means the cell crashed,
-`-` means it never ran.
+most 10 s. `70+` means the tier passed the last step, so its real ceiling is
+higher than the test goes. `0` means no step met the target, `ERR` means the cell crashed,
+`-` means it never ran. A crashed cell's error is on the `Why:` line, once per
+distinct error.
 
 **Bottlenecks.** A component (queue, api, database, redis) running at or over
 95% of its CPU request during the steps the tier passed. That is what stops
@@ -120,9 +122,9 @@ that failed. Last, the request rate actually served at the top step.
 
 ## Delta channel
 
-**Question it answers:** as a thread grows to thousands of turns, does
-`DeltaChannel` keep reads and writes cheaper than a plain channel, and how
-much storage does it save?
+**Question it answers:** on a long thread, does `DeltaChannel` at the
+deepagents setting (a message list, snapshot every 50 turns) save faster and
+store less than a plain channel, without making reads slower?
 
 **Background.** A plain channel saves the whole value at every checkpoint. A
 `DeltaChannel` saves only what changed, plus a full snapshot every N turns,
@@ -130,46 +132,32 @@ and rebuilds the value on read by replaying the changes since the last
 snapshot. That moves cost from write to read, and read cost grows with the
 distance to the last snapshot.
 
-**How it runs.** 20 k6 users, one thread each, for 25 minutes. Every
-iteration adds one turn to the user's thread (`POST /runs/wait`), appending
-1 KB to its state, so threads reach about 1,500 turns. Every 50 turns each user also does a cold state read
-(`GET /threads/{id}/state`) and reads the thread's storage size
-(`GET /threads/{id}/storage`). Four cells run the same graph logic:
-
-| Row | Channel |
-|---|---|
-| `non-delta` | plain channel, the control |
-| `snap 50` | delta, snapshot every 50 turns (the deepagents setting) |
-| `snap 500` | delta, snapshot every 500 turns |
-| `snap 1000` | delta, snapshot every 1000 turns (the library default) |
-
-**The table.** One row per cell at the deepest turn every cell has a cold read
-for, named in the header (`at 1400`). A slower cell gets less deep in 25
-minutes, so the table stops where the slowest one did; the thread says how far
-each cell got. Values are seconds, except `KB`.
-
-- `write`: median checkpoint save at that turn, timed by the worker as the
-  gap between the graph's two steps, with saves made synchronous so nothing
-  overlaps it. `-` with a note means the deployed benchmark graph is too old
-  to report it.
-- `read`: median cold state read at that turn, timed to the server's first
-  byte, so sending the state back is not counted.
-- `KB`: storage the thread occupies at the deepest turn. Missing when the
-  deployment's API predates the storage endpoint.
-
-**Pass bar.** Request errors under 1% and success over 99% per cell. Two
-correctness checks turn the dot yellow on any failure: at turn 2 each cell
-confirms delta is on (or off, for the control), because a too-old langgraph
-silently degrades a `DeltaChannel` to a plain one; and every cold read is
-compared with what the run returned.
-
-**Thread.** Per-cell turn and failure counts against past nights; the plain
-channel and delta-at-50 as two stacked tables every 250 turns, with the
-deepest turn each cell reached; and the snapshot-frequency sweep, whose
-columns are the snapshot interval (50, 500, 1000). The sweep shows read,
-write and storage at the turn before and at each snapshot, where read peaks
-and resets, plus the write at the row's own turn: snapshots land on those
-turns, so it is the full-value write the median hides.
+**How it runs.** One k6 run of 20 users. Each user owns two threads, one plain
+(`base`) and one delta, and takes a turn on each in rotation until both reach
+1,400 turns, so the two are always measured at the same depth seconds apart.
+A turn (`POST /runs/wait`) appends one 1 KB chat message of varied text; text
+repeated every turn would compress away in Postgres and make storage measure
+row overhead. Each user also does a cold state read (`GET /threads/{id}/state`)
+once every 50 turns, at its own offset in that interval, so the reads cover
+the whole climb between two snapshots instead of the moment right after one.
+Storage (`GET /threads/{id}/storage`) is read at turns 350, 700, 1050 and 1400.
+
+**The table.** Both setups at turn 1400: `write` is the median checkpoint
+save, timed by the worker between the graph's two steps; `read` is the median
+cold read, timed to the server's first byte; `MB` is the thread's storage.
+Lower is better for all three. `last 10` is the delta value over tonight and
+the nine nights before it. A metric outside its usual range over five or more
+nights gets a ⚠️ line above the table.
+
+**Pass bar.** Request errors under 1% and success over 99%. Two correctness
+checks turn the dot yellow on any failure: at turn 2 each thread confirms
+delta is on (or off, for `base`), because a too-old langgraph silently
+degrades a `DeltaChannel` to a plain one; and every cold read is compared with
+what the run returned.
+
+**Thread.** Write, read and MB at turns 350, 700, 1050 and 1400, with `gain`:
+how many times better delta is (`same` within 15%, below `1.0x` when delta is
+worse). Then the run's size, failures and correctness checks.
 
 ## Changing a report
 

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/benchmark-runners/delta-turns-lib.test.js

@@ -1,7 +1,9 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import {
+  parseCells,
   planTurn,
+  readOffset,
   deltaGuardFailure,
   digestDrift,
   stateDrift,
@@ -15,6 +17,8 @@ const BASE = {
   migrateAfter: 0,
   expectDelta: true,
   verifyEvery: 50,
+  readOffset: 0,
+  storageEvery: 0,
   bucketSize: 100,
 };
 
@@ -180,3 +184,32 @@ test('a missing digest on either side is reported, not passed', () => {
   assert.equal(digestDrift('', 'a'.repeat(64)), 'run_digest_missing');
   assert.equal(digestDrift('a'.repeat(64), null), 'cold_digest_missing');
 });
+
+test('cold reads spread across the snapshot interval instead of all landing on it', () => {
+  const offsets = Array.from({ length: 20 }, (_, i) => readOffset(i + 1, 20, 50));
+  assert.equal(offsets[0], 0);
+  assert.equal(offsets.at(-1), 47);
+  assert.equal(new Set(offsets).size, 20, 'every VU reads at its own point');
+  assert.equal(planTurn(47, { ...BASE, readOffset: 3 }).verify, true);
+  assert.equal(planTurn(50, { ...BASE, readOffset: 3 }).verify, false);
+});
+
+test('storage is read on the cold-read turn of each bucket that ends a table row', () => {
+  const options = { ...BASE, bucketSize: 50, readOffset: 10, storageEvery: 350 };
+  assert.equal(planTurn(340, options).storage, true, 'bucket 301-350 ends on 350');
+  assert.equal(planTurn(290, options).storage, false, 'a cold read, but bucket 251-300 is not a row');
+  assert.equal(planTurn(341, options).storage, false, 'not a cold-read turn');
+});
+
+test('without a storage interval, storage is read on every cold read', () => {
+  assert.equal(planTurn(50, { ...BASE, storageEvery: 0 }).storage, true);
+  assert.equal(planTurn(51, { ...BASE, storageEvery: 0 }).storage, false);
+});
+
+test('cells parse from name:graph:expectDelta', () => {
+  assert.deepEqual(parseCells('control:benchmark:false, delta_50:benchmark_delta_messages:true'), [
+    { name: 'control', graphId: 'benchmark', expectDelta: false },
+    { name: 'delta_50', graphId: 'benchmark_delta_messages', expectDelta: true },
+  ]);
+  assert.deepEqual(parseCells(''), []);
+});

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/benchmark-runners/delta-turns-lib.ts

@@ -9,9 +9,20 @@ export interface DeltaTurnsOptions {
   expectDelta: boolean;
   /** Cold re-read cadence, in turns. 0 disables. */
   verifyEvery: number;
+  /** Shifts this VU's cold reads off the multiples of `verifyEvery`; see readOffset. */
+  readOffset: number;
+  /** Storage is read on the cold-read turn of every bucket ending on a multiple of this. 0 reads it on every cold read. */
+  storageEvery: number;
   bucketSize: number;
 }
 
+/** One setup the VU turns in rotation, on a thread of its own. */
+export interface DeltaCell {
+  name: string;
+  graphId: string;
+  expectDelta: boolean;
+}
+
 export interface TurnPlan {
   graphId: string;
   /** Cumulative on a reused thread, so turn N asks for N. */
@@ -19,6 +30,7 @@ export interface TurnPlan {
   expectDelta: boolean;
   guard: boolean;
   verify: boolean;
+  storage: boolean;
   depthBucket: string;
 }
 
@@ -32,16 +44,40 @@ export function unusableCheckpointSize(checkpointSize: number | undefined): bool
 export function planTurn(turn: number, options: DeltaTurnsOptions): TurnPlan {
   const migrating = options.migrateFromGraphId !== '' && options.migrateAfter > 0;
   const beforeSwitch = migrating && turn <= options.migrateAfter;
+  const verify = options.verifyEvery > 0 && (turn + options.readOffset) % options.verifyEvery === 0;
   return {
     graphId: beforeSwitch ? options.migrateFromGraphId : options.graphId,
     steps: turn,
     expectDelta: beforeSwitch ? false : options.expectDelta,
     guard: isGuardTurn(turn, options, migrating),
-    verify: options.verifyEvery > 0 && turn % options.verifyEvery === 0,
+    verify,
+    storage: verify && (options.storageEvery <= 0 || bucketEnd(turn, options.bucketSize) % options.storageEvery === 0),
     depthBucket: depthBucket(turn, options.bucketSize),
   };
 }
 
+/** `name:graph:expectDelta`, comma-separated. */
+export function parseCells(spec: string): DeltaCell[] {
+  return spec
+    .split(',')
+    .map((part) => part.trim())
+    .filter(Boolean)
+    .map((part) => {
+      const [name, graphId, expectDelta] = part.split(':');
+      return { name, graphId, expectDelta: expectDelta === 'true' };
+    });
+}
+
+/**
+ * A delta snapshot lands on every multiple of its frequency, so reading every
+ * VU on those turns only ever measures the read right after one. Spreading the
+ * VUs across the interval samples the whole climb between snapshots.
+ */
+export function readOffset(vu: number, vus: number, verifyEvery: number): number {
+  if (verifyEvery <= 0 || vus <= 0) return 0;
+  return Math.floor(((vu - 1) * verifyEvery) / vus) % verifyEvery;
+}
+
 /**
  * Turn 1 has no counters yet, and on a snapshot point Pregel drops the counters
  * entry, so its absence there is correct rather than a fallback.
@@ -51,6 +87,10 @@ function isGuardTurn(turn: number, options: DeltaTurnsOptions, migrating: boolea
   return migrating && turn === options.migrateAfter + 2;
 }
 
+function bucketEnd(turn: number, bucketSize: number): number {
+  return bucketSize <= 0 ? turn : Math.ceil(turn / bucketSize) * bucketSize;
+}
+
 function depthBucket(turn: number, bucketSize: number): string {
   if (bucketSize <= 0) return String(turn);
   const floor = Math.floor((turn - 1) / bucketSize) * bucketSize;

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/benchmark-runners/delta_turns.ts

@@ -13,9 +13,11 @@ import {
   planTurn,
   deltaGuardFailure,
   digestDrift,
+  parseCells,
+  readOffset,
   unusableCheckpointSize,
 } from './delta-turns-lib.js';
-import type { DeltaTurnsOptions, TurnPlan } from './delta-turns-lib.js';
+import type { DeltaCell, DeltaTurnsOptions, TurnPlan } from './delta-turns-lib.js';
 
 const PAYLOAD_CHANNELS = [
   'cumulative_checkpoint',
@@ -35,20 +37,30 @@ const stateReadDuration = new Trend('state_read_duration', true);
 const checkpointSaveDuration = new Trend('checkpoint_save', true);
 const threadStorageBytes = new Trend('thread_storage_bytes', false);
 
-const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
+const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId' | 'readOffset'> = {
   migrateFromGraphId: __ENV.MIGRATE_FROM_GRAPH_ID || '',
   migrateAfter: parseInt(__ENV.MIGRATE_AFTER || '0', 10),
   expectDelta: (__ENV.EXPECT_DELTA || 'true') === 'true',
   verifyEvery: parseInt(__ENV.VERIFY_EVERY || '50', 10),
+  storageEvery: parseInt(__ENV.STORAGE_EVERY || '0', 10),
   // One bucket per cold read, so every row of the report is one sample per thread.
   bucketSize: parseInt(__ENV.DEPTH_BUCKET || __ENV.VERIFY_EVERY || '50', 10),
 };
 
+/**
+ * Every VU turns these in rotation, one thread each, so the cells are measured
+ * at the same depth seconds apart. Empty runs GRAPH_ID alone.
+ */
+const CELLS = parseCells(__ENV.DELTA_CELLS || '');
+
 // Each k6 VU runs this module in its own context, so these are per VU.
-let threadId: string | null = null;
-let turn = 0;
+const threads: Record<string, { threadId: string | null; turn: number }> = {};
+let iteration = 0;
 
 interface DeltaTurnsData {
+  cell: string;
+  turnMs: number;
+  tags: { depth: string; cell: string };
   rawResponse: ReturnType<typeof http.post>;
   plan: TurnPlan;
   guardFailure: string | null;
@@ -69,18 +81,31 @@ export class DeltaTurns extends BenchmarkRunner {
       return failResult('checkpoint_size', responses) as BenchmarkResult<DeltaTurnsData>;
     }
 
-    if (threadId === null) {
-      const thread = http.post(`${baseUrl}/threads`, '{}', requestParams);
+    const cell: DeltaCell = CELLS.length > 0
+      ? CELLS[iteration++ % CELLS.length]
+      : { name: benchmarkGraphOptions.graph_id, graphId: benchmarkGraphOptions.graph_id, expectDelta: TURN_CONFIG.expectDelta };
+    const own = (threads[cell.name] ??= { threadId: null, turn: 0 });
+    if (own.threadId === null) {
+      const ttlMinutes = parseInt(__ENV.THREAD_TTL_MINUTES || '0', 10);
+      const body = ttlMinutes > 0 ? { ttl: { strategy: 'delete', ttl: ttlMinutes } } : {};
+      const thread = http.post(`${baseUrl}/threads`, JSON.stringify(body), requestParams);
       addResponse(responses, 'create_thread', thread);
       if (thread.status !== 200) {
         return failResult('create_thread', responses) as BenchmarkResult<DeltaTurnsData>;
       }
-      threadId = (thread.json() as { thread_id: string }).thread_id;
-      turn = 0;
+      own.threadId = (thread.json() as { thread_id: string }).thread_id;
+      own.turn = 0;
     }
+    const threadId = own.threadId;
 
-    turn += 1;
-    const plan = planTurn(turn, { ...TURN_CONFIG, graphId: benchmarkGraphOptions.graph_id });
+    own.turn += 1;
+    const plan = planTurn(own.turn, {
+      ...TURN_CONFIG,
+      graphId: cell.graphId,
+      expectDelta: cell.expectDelta,
+      readOffset: readOffset(__VU, parseInt(__ENV.LOAD_SIZE || '1', 10), TURN_CONFIG.verifyEvery),
+    });
+    const tags = { depth: plan.depthBucket, cell: cell.name };
     const payload = JSON.stringify({
       assistant_id: plan.graphId,
       input: {},
@@ -97,14 +122,11 @@ export class DeltaTurns extends BenchmarkRunner {
       payload,
       requestParams
     );
-    turnDuration.add(Date.now() - startedAt, { depth: plan.depthBucket });
+    const turnMs = Date.now() - startedAt;
     addResponse(responses, 'wait', response);
     const saveMs = response.status === 200 ? safeJson(response.body)?.save_ms : null;
     if (typeof saveMs === 'number' && saveMs > 0) {
-      // Snapshots land on multiples of the bucket width, so the bucket's last
-      // turn is where a snapshot save shows; a percentile over the bucket hides it.
-      const rowTurn = plan.steps % TURN_CONFIG.bucketSize === 0 ? '1' : '0';
-      checkpointSaveDuration.add(saveMs, { depth: plan.depthBucket, row_turn: rowTurn });
+      checkpointSaveDuration.add(saveMs, tags);
     }
 
     let guardFailure: string | null = null;
@@ -120,7 +142,7 @@ export class DeltaTurns extends BenchmarkRunner {
       // a plain-channel read cost as much as a whole turn. The turn-2 guard
       // read is at depth 2, not the bucket's end, so it is not timed.
       if (state.status === 200 && plan.verify) {
-        stateReadDuration.add(state.timings.waiting, { depth: plan.depthBucket });
+        stateReadDuration.add(state.timings.waiting, tags);
       }
       addResponse(responses, 'state', state);
       const body = state.status === 200 ? safeJson(state.body) : null;
@@ -137,7 +159,7 @@ export class DeltaTurns extends BenchmarkRunner {
           drift = digestDrift(safeJson(response.body)?.payload_sha256, coldDigest(body.values));
         }
       }
-      if (plan.verify) {
+      if (plan.storage) {
         // Servers before 0.16 have no storage endpoint. A 404 there is a missing
         // sample, not a failed request: without this, one 404 per 50 turns is
         // about 2% and trips the 1% error budget on every cell.
@@ -148,12 +170,12 @@ export class DeltaTurns extends BenchmarkRunner {
         addResponse(responses, 'storage', storage);
         const total = storage.status === 200 ? safeJson(storage.body)?.total_bytes : null;
         if (typeof total === 'number') {
-          threadStorageBytes.add(total, { depth: plan.depthBucket, graph: plan.graphId });
+          threadStorageBytes.add(total, { ...tags, graph: plan.graphId });
         }
       }
     }
 
-    return okResult(responses, { rawResponse: response, plan, guardFailure, drift, stateStatus });
+    return okResult(responses, { cell: cell.name, turnMs, tags, rawResponse: response, plan, guardFailure, drift, stateStatus });
   }
 
   static validate(
@@ -203,7 +225,12 @@ export class DeltaTurns extends BenchmarkRunner {
       );
       // The thread is now at an unknown depth, so later turns would be tagged
       // with a depth they do not have.
-      threadId = null;
+      const cell = result.data?.cell;
+      if (cell && threads[cell]) threads[cell].threadId = null;
+    } else if (result.data) {
+      // The report counts these per bucket to tell which depth every user
+      // finished, so only a turn that passed every check counts.
+      turnDuration.add(result.data.turnMs, result.data.tags);
     }
     return success;
   }

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/capacity-ci.js

@@ -315,7 +315,10 @@ function moveLine(row, unit) {
 export function capacityCell(summary) {
   if (!summary) return "-";
   if (summary.error) return "ERR";
-  return String(summary.capacity?.targetVUs ?? 0);
+  const vus = String(summary.capacity?.targetVUs ?? 0);
+  // Passing the last step says only that the server is at least this fast.
+  const lastStep = summary.config?.numSteps;
+  return lastStep != null && summary.capacity?.step === lastStep ? `${vus}+` : vus;
 }
 
 function plural(n, word) {
@@ -396,6 +399,9 @@ export function formatAggregateTable(
   if (failedParts.length > 0) {
     lines.push(`*Failed:* ${failedParts.join(" · ")} · treat the table as incomplete`);
   }
+  // hostRequest appends the response body, which can span lines; the detail is often on a later one.
+  const reasons = new Set(rows.map((row) => row.summary?.error?.replace(/\s+/g, " ").trim()).filter(Boolean));
+  if (reasons.size > 0) lines.push(`*Why:* ${[...reasons].join(" · ")}`);
   if (regressed.length > 0) lines.push(`*Regressed:* ${regressed.map((row) => moveLine(row, unit)).join(" · ")}`);
   if (bottlenecks.length > 0) lines.push(`*Bottlenecks:* ${bottlenecks.join(" · ")}`);
   for (const warning of allWarnings) lines.push(`:warning: ${warning}`);

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/capacity-ci.test.js

@@ -80,6 +80,12 @@ test("the posted cell is the load alone; the served rate lives in the thread", (
   assert.equal(capacityCell({ capacity: { targetVUs: 900, successfulRunsPerSec: 366 } }), "900");
 });
 
+test("a cell that passed the last step reads as a floor, not a ceiling", () => {
+  const config = { numSteps: 13 };
+  assert.equal(capacityCell({ config, capacity: { step: 13, targetVUs: 70 } }), "70+");
+  assert.equal(capacityCell({ config, capacity: { step: 12, targetVUs: 65 } }), "65");
+});
+
 test("a failed cell renders as ERR in the posted table and is named in the Failed line", () => {
   const deployments = [
     { slug: "dep", tiers: ["TIER_A", "TIER_B"] },
@@ -130,6 +136,7 @@ test("an incomplete run says so, a clean run says all quiet", () => {
   assert.match(table, /^🔴 .* · 1 of 3 cells ran · dep TIER_A crashed, 1 never ran$/m);
   assert.match(table, /^\*Failed:\* dep TIER_A stream \(ERR\) · dep TIER_A search never ran · treat the table as incomplete$/m);
   assert.match(table.split("\n")[1], /^\*Failed:\*/, "what went wrong comes right after the verdict");
+  assert.match(table, /^\*Why:\* boom$/m);
 
 
   const clean = formatAggregateTable(deployments, [
@@ -418,3 +425,18 @@ test("the CPU table fits a phone even with every component reporting", () => {
   const view = formatServerView(deployments, summaries);
   assert.ok(widestCodeLine(view) <= PHONE_WIDTH, `widest table line is ${widestCodeLine(view)}`);
 });
+
+test("tiers that crashed for the same reason name it once", () => {
+  const deployments = [{ slug: "dep", tiers: ["TIER_A", "TIER_B"] }];
+  const crash = (computeTier) =>
+    buildErrorSummary({ deploymentSlug: "dep", computeTier, benchmarkType: "wait_write" }, new Error("Deployment failed with status: BUILD_FAILED"));
+  const table = formatAggregateTable(deployments, [crash("TIER_A"), crash("TIER_B")]);
+  assert.match(table, /^\*Why:\* Deployment failed with status: BUILD_FAILED$/m);
+});
+
+test("a crashed cell's multi-line error keeps its detail on the Why line", () => {
+  const deployments = [{ slug: "dep", tiers: ["TIER_A"] }];
+  const error = new Error('PATCH /resource-tiers failed: 409 {\n  "detail": "Unable to resize the deployment database."\n}');
+  const table = formatAggregateTable(deployments, [buildErrorSummary({ deploymentSlug: "dep", computeTier: "TIER_A", benchmarkType: "wait_write" }, error)]);
+  assert.match(table, /^\*Why:\* PATCH \/resource-tiers failed: 409 \{ "detail": "Unable to resize the deployment database\." \}$/m);
+});

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/clean.js

@@ -236,4 +236,3 @@ async function cleanThreads(baseUrl, langsmithApiKey) {
 
     console.log(`Thread cleanup completed. Total threads deleted: ${totalDeleted}`);
 }
-

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/continuous/uv.lock

@@ -549,11 +549,11 @@ wheels = [
 
 [[package]]
 name = "urllib3"
-version = "2.7.0"
+version = "2.8.0"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" },
+    { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" },
 ]
 
 [[package]]

{langgraph_api-0.17.0.dev3 → langgraph_api-0.17.0.dev4}/benchmark/daily-report.js

@@ -217,9 +217,15 @@ function runName(run) {
   return runnerShortName(run.benchmarkType);
 }
 
+function noResults(run) {
+  return run.revisionError
+    ? `no results, the new revision failed: ${run.revisionError}`
+    : `no results produced, step finished as \`${run.stepOutcome}\``;
+}
+
 function failureReason(run) {
   const metrics = run.summary?.metrics;
-  if (!metrics) return `no results produced, step finished as \`${run.stepOutcome}\``;
+  if (!metrics) return noResults(run);
   const crossed = describedThresholds(run.summary.thresholds, metrics).filter((t) => t.ok === false);
   if (crossed.length > 0) return crossed.map((t) => t.breach).join(', ');
   if (!metrics.successfulRuns) return 'no run succeeded';
@@ -356,7 +362,7 @@ function perRunDetail(runs, changesByRun, improved, load) {
   for (const run of runs) {
     const metrics = run.summary?.metrics;
     if (!metrics) {
-      lines.push(`${runName(run)}: no results, step finished as \`${run.stepOutcome}\``);
+      lines.push(`${runName(run)}: ${noResults(run)}`);
       continue;
     }
     const counts = `${count(metrics.totalRuns)} runs, ${count(metrics.failedRuns)} failed`;
@@ -427,6 +433,7 @@ export function collectRuns(resultsDir, historyDir) {
     return {
       benchmarkType,
       stepOutcome: context.stepOutcome ?? (dir ? 'unknown' : 'missing'),
+      revisionError: context.revisionError,
       summary: dir ? readSummary(findLatestSummary(dir)) : null,
       load: { loadSize: context.loadSize, levels: context.levels, plateauDuration: context.plateauDuration },
       history: historyDir ? loadDailyHistory(historyDir, `-${benchmarkType}-`) : [],