langgraph-api 0.15.4__tar.gz → 0.16.0__tar.gz

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/Makefile

@@ -317,6 +317,10 @@ VERSION_KIND ?= patch
 bump-version:
 	uv run --with hatch hatch version $(VERSION_KIND)
 
+sync-runtime-inmem-dep:
+	@python3 -c 'import pathlib,re; v=pathlib.Path("../runtime_inmem/langgraph_runtime_inmem/__init__.py").read_text(); m=re.search(r"__version__\s*=\s*\"(\d+)\.(\d+)", v); major,minor=map(int,m.groups()); p=pathlib.Path("pyproject.toml"); text,n=re.subn(r"\"langgraph-runtime-inmem>=[^\"]+\",", f"\"langgraph-runtime-inmem>={major}.{minor}.0.dev0,<{major}.{minor + 1}.0.dev0\",", p.read_text(), count=1); \
+	assert n==1, "langgraph-runtime-inmem dep not found in pyproject.toml"; p.write_text(text)'
+
 # A2A TCK (Test Compatibility Kit) targets
 # Requires a running server: make start (in another terminal)
 

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.15.4
+Version: 0.16.0
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
@@ -17,7 +17,7 @@ Requires-Dist: jsonschema-rs<0.45,>=0.20.0
 Requires-Dist: langchain-core>=0.3.64
 Requires-Dist: langchain-protocol<0.1,>=0.0.18
 Requires-Dist: langgraph-checkpoint<5,>=3.0.1
-Requires-Dist: langgraph-runtime-inmem<0.36.0.dev0,>=0.35.0.dev0
+Requires-Dist: langgraph-runtime-inmem<0.37.0.dev0,>=0.36.0.dev0
 Requires-Dist: langgraph-sdk>=0.4.4
 Requires-Dist: langgraph<2,>=0.4.10
 Requires-Dist: langsmith[otel]>=0.6.3

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/README.md

@@ -7,7 +7,7 @@ This directory contains k6 + Python/Node benchmark tooling for LangGraph API.
 - `make benchmark-ramp` - Ramps up to a set number of concurrent virtual users, holds plateau load, then ramps down.
   - **Environment variables:**
     - `LOAD_SIZE`, `LEVELS`, `PLATEAU_DURATION`
-    - `P95_RUN_DURATION`, `AVERAGE_RUN_DURATION`
+    - `P95_RUN_DURATION`
   - **Example:**
     - `BENCHMARK_PROFILE=streaming-long BENCHMARK_TYPE=stream_write LOAD_SIZE=300 LEVELS=3 make benchmark-ramp`
 
@@ -25,7 +25,7 @@ This directory contains k6 + Python/Node benchmark tooling for LangGraph API.
 - `make benchmark-reset` - Cleans benchmark server state via `clean-cli.js`.
 - `make benchmark-new-revision` - Triggers a new benchmark deployment revision.
 
-- `make benchmark-capacity-ci` - Runs the daily capacity matrix for one deployment (`DEPLOYMENT` env). Uses `staircase.py` per cell (default 200–2000 VUs, step 100). Config lives in `capacity-ci.js`.
+- `make benchmark-capacity-ci` - Runs the daily capacity matrix for one deployment (`DEPLOYMENT` env). Uses `staircase.py` per cell with k6's `constant-arrival-rate` executor (open model: a fixed request rate per step, so capacity reads as the service rate; requests k6 has to drop count as failures). Geometry is per deployment in `capacity-ci.js` (dedicated 50 to 600 req/s in steps of 50, serverless 10 to 70 in steps of 5). `K6_EXECUTOR=constant-vus` restores the closed model. A cell that never meets SLO still reports `throughputCeiling`, the best runs/sec any step served.
   - **Example:**
     - `DEPLOYMENT=daily-cap-benchmark-dedicated make benchmark-capacity-ci`
 
@@ -48,6 +48,7 @@ The runner contains the actual logic each virtual user will execute during the b
 - `store` - Writes, reads, validates, and deletes a Store item with a configurable payload size.
 - `meta_workload` (randomly delegates to one of the others)
 - `delta_turns` - Keeps one thread per VU and adds a turn to it every iteration, so latency can be read against thread depth. Built for the delta-channel cells; see below.
+- `interrupt_resume` - Starts a run that pauses on an `interrupt()` before step `interrupt_at`, then resumes it at once. Measures the paused checkpoint write and the full state read on resume. Use with the `interrupt` profile.
 
 ## Profiles (`BENCHMARK_PROFILE`)
 
@@ -60,6 +61,9 @@ Profiles are quick ways to configure the benchmark graph for any run depending o
 - `parallel-small` - Stateful high-fanout profile with small checkpoint payloads.
 - `parallel-medium` - Stateful high-fanout profile with medium checkpoint payloads.
 - `delta-turns` - Cheap turns for the delta-channel cells, so a VU reaches the depth where a `snapshot_frequency` of 1000 fires.
+- `wide-deep` - 50 parallel branches held for 20 supersteps: concurrent checkpoint writes and worker spread at the same time.
+- `messages` - State grows as a chat message list (2 KB per step, 10 steps), so each checkpoint serializes structured objects rather than appending bytes.
+- `interrupt` - 5 steps, pauses before step 3. For the `interrupt_resume` runner.
 
 Each profile defines:
 
@@ -85,6 +89,9 @@ Context overrides (merged on top of profile):
 - `EXPAND` - Fanout width (number of parallel branches at start).
 - `STEPS` - Number of graph turns/iterations to execute.
 - `CHECKPOINT_SIZE` - Bytes written per checkpoint operation.
+- `PAYLOAD` - `text` (default; compresses like real state), `random` (incompressible worst case) or `messages` (a chat message of `CHECKPOINT_SIZE` chars per step).
+- `CHANNELS` - Spread the bytes payload across this many channels (1 to 8). Postgres stores one row per changed channel per version, so this costs rows, not only bytes.
+- `INTERRUPT_AT` - Pause the run before this step (1-based) until resumed. 0 never pauses.
 - `STORE_VALUE_SIZE` - Store runner payload size in bytes (defaults to 1 MiB).
 - `LLM_ENABLED` - Enables fake LLM streaming inside the benchmark graph.
 - `STREAM_SIZE` - Normal total streamed payload size (bytes).
@@ -122,7 +129,11 @@ by replaying them. That moves cost from write to read, and the read cost grows w
 thread depth. `delta_turns` measures it: one thread per VU, one turn per iteration,
 `turn_duration` tagged by depth bucket.
 
-Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
+`benchmark-delta.yml` runs four cells nightly, 20 users for 25 minutes each, 1 KB appended per turn (`CHECKPOINT_SIZE`): plain channel, then
+delta snapshotting every 50 (`benchmark_delta_freq`, the deepagents setting), 500
+(`benchmark_delta_500`) and 1000 (`benchmark_delta`, the library default). `delta-report.js` posts
+a headline table (non-delta vs delta at 50: read, write, storage by depth) and one table per metric
+across the three frequencies. The messages and migration cells stay manual. Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
 with depth for different reasons, the control by re-writing a growing value and delta by
 replaying more writes, so the deliverable is which curve is worse and where they cross.
 
@@ -131,12 +142,14 @@ replaying more writes, so the deliverable is which curve is worse and where they
 | control | `GRAPH_ID=benchmark EXPECT_DELTA=false` |
 | delta, snapshot every 1000 | `GRAPH_ID=benchmark_delta` |
 | delta, snapshot every 50 | `GRAPH_ID=benchmark_delta_freq` |
+| delta, snapshot every 500 | `GRAPH_ID=benchmark_delta_500` |
+| delta on a message list | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
 | migration | `GRAPH_ID=benchmark_delta MIGRATE_FROM_GRAPH_ID=benchmark MIGRATE_AFTER=200` |
 
 ```bash
 BASE_URL=... BENCHMARK_TYPE=delta_turns BENCHMARK_PROFILE=delta-turns \
   GRAPH_ID=benchmark_delta LOAD_SIZE=25 LEVELS=1 PLATEAU_DURATION=300 \
-  P95_RUN_DURATION=... AVERAGE_RUN_DURATION=... k6 run --system-tags='[]' ramp.js
+  P95_RUN_DURATION=... k6 run --system-tags='[]' ramp.js
 ```
 
 Knobs beyond the common ones:
@@ -146,21 +159,23 @@ Knobs beyond the common ones:
 - `MIGRATE_FROM_GRAPH_ID` / `MIGRATE_AFTER` - run the first N turns on another graph, then
   switch on the same thread.
 - `VERIFY_EVERY` - cold re-read cadence in turns. Default 50, `0` disables.
-- `DEPTH_BUCKET` - width of a depth bucket. Default 100.
+- `DEPTH_BUCKET` - width of a depth bucket. Defaults to `VERIFY_EVERY`, so each bucket holds one cold read per thread.
 
 Two things to know before reading any result:
 
 - **Run at low `LOAD_SIZE`.** Depth and load fight each other: saturate the deployment and
   turn time measures queueing, not reconstruction. Verify the control curve is flat before
   trusting the delta one.
-- **`successful_runs` will fail its threshold.** The count is derived from a duration table
-  that assumes the existing workload. Pass `P95_RUN_DURATION` and `AVERAGE_RUN_DURATION`,
-  and treat these cells as report-only until the numbers settle.
+- **The ramp gates on p95 and error rate only.** Pass a `P95_RUN_DURATION` that fits the
+  cell, and treat these cells as report-only until the numbers settle.
 
 The runner fails a turn rather than reporting a false pass when delta is not actually on:
 the `DeltaChannel` import is version-guarded, and its stub degrades to a plain channel.
-Storage savings, the reason the feature exists, are not measured here; that needs a query
-against the deployment database, which CI cannot reach.
+At every `VERIFY_EVERY` turn it also times the cold `GET /threads/{thread_id}/state` as
+`state_read_duration` (the read path alone; a turn is one save plus the read before it), and
+reads `GET /threads/{thread_id}/storage` and records
+`thread_storage_bytes` tagged by depth bucket and graph, so the control and delta cells can
+be compared on the bytes they occupy, which is what the feature exists to reduce.
 
 ## Outputs
 
@@ -177,6 +192,12 @@ against the deployment database, which CI cannot reach.
 
 - Daily ramp benchmark: `.github/workflows/benchmark.yml`
 - Daily capacity matrix: `.github/workflows/benchmark-capacity-matrix.yml`
+- Daily delta-channel cells: `.github/workflows/benchmark-delta.yml`, reported by `delta-report.js`
+
+The staging deployments build their graphs from
+[langgraph-server-benchmarks](https://github.com/langchain-ai/langgraph-server-benchmarks), not
+from this repo. Its `src/agent/graph.py` is a copy of `api/tests/graphs/benchmark.py`; a graph
+change here reaches the nightly only once that copy is updated.
 
 ### Capacity matrix setup
 
@@ -184,6 +205,19 @@ against the deployment database, which CI cannot reach.
 2. Fill in `deploymentId` and `baseUrl` in `CAPACITY_DEPLOYMENTS` at the top of [`capacity-ci.js`](capacity-ci.js).
 3. Scheduled run: two parallel CI jobs (one per deployment), aggregate job posts Slack summary.
 
+Each cell also records the deployment's own view of its window (queue and API pod counts, CPU peaks) from the control plane's deployment metrics route. The posted message names any component at or over 95% of its CPU request as that tier's bottleneck, and the full per-cell table goes to the message's thread. Every tier starts at one queue pod and autoscales during a cell, so a capacity number is partly a pod-count number; pinning pods at the tier maximum would need the benchmark deployments recreated as `external_docker` sources, the only kind whose `resource_spec` the API accepts.
+
+### Slack reports
+
+Every report follows one shape so the channel reads the same way each morning, on a phone as well as a desktop:
+
+- The first line is the verdict: a dot (green quiet, yellow something regressed or a bottleneck or errors, red something failed), the report name and date, `manual run` when dispatched by hand, the API version, and a clause saying what needs attention. The second line says what the benchmark is, for a reader outside the team.
+- Then `Failed`, `Regressed`, `Improved` and `Bottlenecks` or `Errors` lines, worst first. `Regressed` prints `none` rather than going quiet.
+- One table per deployment, about 40 characters wide so it fits a phone, with `▲ better` / `▼ worse` markers on cells that left their usual range.
+- One legend line (pass rule, arrows, dot colors), then a link to the thread, which holds the per-cell or per-run numbers and the longer *How to read*, and to the run's artifacts.
+
+Both reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`). A figure is flagged only when it leaves that cell's own observed range, and only once five same-kind nights exist; before that the table says how many nights it has. The daily benchmark posts one digest per workflow once its last runner finishes (`daily-report.js`, fed by each runner's artifact), not one message per runner. The delta-channel report (`delta-report.js`) keeps the plain-vs-delta table in the channel and puts the snapshot-frequency sweep in the thread.
+
 Local commands:
 
 ```bash
@@ -201,3 +235,12 @@ node capacity-ci.js --aggregate --summaries-dir ./artifacts --slack
 
 - [k6 docs](https://k6.io/docs/)
 - [k6 JavaScript API](https://k6.io/docs/javascript-api/)
+
+## What a run returns
+
+The graph's output schema is `counter`, `start_time`, `end_time`, `payload_bytes` and
+`payload_sha256`. The payload channels themselves stay out of `/runs/wait` bodies and
+`values` events, so `CHECKPOINT_SIZE` drives database writes only; before this, every
+response carried the whole payload base64-encoded and write cost could not be told from
+egress. `GET /threads/{thread_id}/state` still returns every channel. The delta cells
+verify a cold re-read by hashing it and comparing with `payload_sha256`.

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/benchmark-runners/benchmark_profiles.ts

@@ -13,6 +13,9 @@ export interface BenchmarkContext {
   burst_mode: boolean;
   burst_probability: number;
   burst_size: number;
+  payload?: 'text' | 'random' | 'messages';
+  channels?: number;
+  interrupt_at?: number;
 }
 
 export interface BenchmarkProfileEnv {
@@ -32,6 +35,9 @@ export interface BenchmarkProfileEnv {
   BURST_MODE?: string;
   BURST_PROBABILITY?: string;
   BURST_SIZE?: string;
+  PAYLOAD?: string;
+  CHANNELS?: string;
+  INTERRUPT_AT?: string;
 }
 
 export interface BenchmarkProfile {
@@ -173,6 +179,64 @@ export const BENCHMARK_PROFILES: Record<string, BenchmarkProfile> = {
       burst_size: 0,
     },
   },
+  // Width held for many supersteps: concurrent checkpoint writes and worker
+  // spread at the same time, which the wide-but-one-step profiles never do.
+  'wide-deep': {
+    resumable: false,
+    runMode: 'stateless',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0.2,
+      expand: 50,
+      steps: 20,
+      checkpoint_size: 1_000,
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
+  // State grows as a message list, so each checkpoint serializes structured
+  // objects rather than appending bytes.
+  messages: {
+    resumable: false,
+    runMode: 'stateful',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0.2,
+      expand: 1,
+      steps: 10,
+      checkpoint_size: 2_000,
+      payload: 'messages',
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
+  // For the interrupt_resume runner.
+  interrupt: {
+    resumable: false,
+    runMode: 'stateful',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0.2,
+      expand: 1,
+      steps: 5,
+      checkpoint_size: 1_000,
+      interrupt_at: 3,
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
 };
 
 export function getBenchmarkProfile(profileName?: string): { name: string; profile: BenchmarkProfile } {
@@ -241,6 +305,9 @@ export function get_profile(env: BenchmarkProfileEnv): ParsedBenchmarkProfile {
       ? { burst_probability: parseOptionalFloat(env.BURST_PROBABILITY)! }
       : {}),
     ...(parseOptionalInt(env.BURST_SIZE) !== undefined ? { burst_size: parseOptionalInt(env.BURST_SIZE)! } : {}),
+    ...(env.PAYLOAD ? { payload: env.PAYLOAD as 'text' | 'random' | 'messages' } : {}),
+    ...(parseOptionalInt(env.CHANNELS) !== undefined ? { channels: parseOptionalInt(env.CHANNELS)! } : {}),
+    ...(parseOptionalInt(env.INTERRUPT_AT) !== undefined ? { interrupt_at: parseOptionalInt(env.INTERRUPT_AT)! } : {}),
   });
 
   return {

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/benchmark-runners/benchmarks.ts

@@ -9,6 +9,7 @@ import { ThreadsSearchMetadata } from './threads_search_metadata.js';
 import { Store } from './store.js';
 import { MetaWorkload } from './meta_workload.js';
 import { DeltaTurns } from './delta_turns.js';
+import { InterruptResume } from './interrupt_resume.js';
 import type { BenchmarkRunner } from './benchmark-runner.js';
 
 export class Benchmarks {
@@ -36,6 +37,8 @@ export class Benchmarks {
         return MetaWorkload;
       case DeltaTurns.toString():
         return DeltaTurns;
+      case InterruptResume.toString():
+        return InterruptResume;
       default:
         throw new Error(`Unknown benchmark type: ${type}`);
     }

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/benchmark-runners/delta-turns-lib.test.js

@@ -3,6 +3,7 @@ import assert from 'node:assert/strict';
 import {
   planTurn,
   deltaGuardFailure,
+  digestDrift,
   stateDrift,
   unusableCheckpointSize,
 } from './dist/delta-turns-lib.js';
@@ -167,3 +168,15 @@ test('the delta-turns profile is wired and writes a growing channel', () => {
   assert.equal(profile.context.delay, 0);
   assert.equal(profile.context.steps, 1);
 });
+
+test('matching digests are no drift, and a changed byte anywhere is', () => {
+  const live = 'a'.repeat(64);
+  assert.equal(digestDrift(live, live), null);
+  assert.equal(digestDrift(live, 'b'.repeat(64)), `digest_drift:${'a'.repeat(12)}!=${'b'.repeat(12)}`);
+});
+
+test('a missing digest on either side is reported, not passed', () => {
+  assert.equal(digestDrift(undefined, 'a'.repeat(64)), 'run_digest_missing');
+  assert.equal(digestDrift('', 'a'.repeat(64)), 'run_digest_missing');
+  assert.equal(digestDrift('a'.repeat(64), null), 'cold_digest_missing');
+});

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/benchmark-runners/delta-turns-lib.ts

@@ -100,3 +100,13 @@ function encodeChannel(value: unknown): string {
   if (value == null) return '';
   return typeof value === 'string' ? value : JSON.stringify(value);
 }
+
+/**
+ * The run reports the SHA-256 of its stored payload; the cold read is hashed
+ * the same way. A rebuild that loses or corrupts a write changes the hash.
+ */
+export function digestDrift(liveDigest: unknown, coldDigest: unknown): string | null {
+  if (typeof liveDigest !== 'string' || liveDigest === '') return 'run_digest_missing';
+  if (typeof coldDigest !== 'string' || coldDigest === '') return 'cold_digest_missing';
+  return liveDigest === coldDigest ? null : `digest_drift:${liveDigest.slice(0, 12)}!=${coldDigest.slice(0, 12)}`;
+}

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/benchmark-runners/delta_turns.ts

@@ -1,6 +1,8 @@
 import { BenchmarkRunner } from './benchmark-runner.js';
 import type { ErrorMetrics } from './benchmark-runner.js';
 import { check } from 'k6';
+import crypto from 'k6/crypto';
+import encoding from 'k6/encoding';
 import http from 'k6/http';
 import { Trend } from 'k6/metrics';
 import type { BenchmarkResult, BenchmarkGraphOptions, HttpResponse } from './types.js';
@@ -10,22 +12,36 @@ import { logFailure } from './log-failure.js';
 import {
   planTurn,
   deltaGuardFailure,
-  stateDrift,
+  digestDrift,
   unusableCheckpointSize,
 } from './delta-turns-lib.js';
 import type { DeltaTurnsOptions, TurnPlan } from './delta-turns-lib.js';
 
-const CHANNEL = 'cumulative_checkpoint';
+const PAYLOAD_CHANNELS = [
+  'cumulative_checkpoint',
+  'payload_2',
+  'payload_3',
+  'payload_4',
+  'payload_5',
+  'payload_6',
+  'payload_7',
+  'payload_8',
+] as const;
 
 /** Tagged by depth: VUs ramp in, so clock time is not thread length. */
 const turnDuration = new Trend('turn_duration', true);
+/** The read path alone: a cold GET /state replays the channel without writing. */
+const stateReadDuration = new Trend('state_read_duration', true);
+const checkpointSaveDuration = new Trend('checkpoint_save', true);
+const threadStorageBytes = new Trend('thread_storage_bytes', false);
 
 const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
   migrateFromGraphId: __ENV.MIGRATE_FROM_GRAPH_ID || '',
   migrateAfter: parseInt(__ENV.MIGRATE_AFTER || '0', 10),
   expectDelta: (__ENV.EXPECT_DELTA || 'true') === 'true',
   verifyEvery: parseInt(__ENV.VERIFY_EVERY || '50', 10),
-  bucketSize: parseInt(__ENV.DEPTH_BUCKET || '100', 10),
+  // One bucket per cold read, so every row of the report is one sample per thread.
+  bucketSize: parseInt(__ENV.DEPTH_BUCKET || __ENV.VERIFY_EVERY || '50', 10),
 };
 
 // Each k6 VU runs this module in its own context, so these are per VU.
@@ -37,6 +53,8 @@ interface DeltaTurnsData {
   plan: TurnPlan;
   guardFailure: string | null;
   drift: string | null;
+  /** Non-200 status of the cold state read, which is a failed request, not a correctness failure. */
+  stateStatus: number | null;
 }
 
 export class DeltaTurns extends BenchmarkRunner {
@@ -68,6 +86,9 @@ export class DeltaTurns extends BenchmarkRunner {
       input: {},
       context: { ...benchmarkGraphOptions.context, steps: plan.steps },
       config: { recursion_limit: plan.steps + 2 },
+      // The graph times its own checkpoint save as the gap between its two
+      // steps; a background save would overlap the second step and hide in it.
+      durability: 'sync',
     });
 
     const startedAt = Date.now();
@@ -78,16 +99,34 @@ export class DeltaTurns extends BenchmarkRunner {
     );
     turnDuration.add(Date.now() - startedAt, { depth: plan.depthBucket });
     addResponse(responses, 'wait', response);
+    const saveMs = response.status === 200 ? safeJson(response.body)?.save_ms : null;
+    if (typeof saveMs === 'number' && saveMs > 0) {
+      // Snapshots land on multiples of the bucket width, so the bucket's last
+      // turn is where a snapshot save shows; a percentile over the bucket hides it.
+      const rowTurn = plan.steps % TURN_CONFIG.bucketSize === 0 ? '1' : '0';
+      checkpointSaveDuration.add(saveMs, { depth: plan.depthBucket, row_turn: rowTurn });
+    }
 
     let guardFailure: string | null = null;
     let drift: string | null = null;
+    let stateStatus: number | null = null;
     // A failed body is not state, and comparing against it would report drift
     // on top of the failure that caused it.
     if (response.status === 200 && (plan.guard || plan.verify)) {
       const state = http.get(`${baseUrl}/threads/${threadId}/state`, requestParams);
+      // Time to first byte: the server cannot start answering before it has
+      // rebuilt the value, and the body that follows is the whole state, which
+      // at depth is hundreds of KB the turn itself never sends. Wall time made
+      // a plain-channel read cost as much as a whole turn. The turn-2 guard
+      // read is at depth 2, not the bucket's end, so it is not timed.
+      if (state.status === 200 && plan.verify) {
+        stateReadDuration.add(state.timings.waiting, { depth: plan.depthBucket });
+      }
       addResponse(responses, 'state', state);
       const body = state.status === 200 ? safeJson(state.body) : null;
-      if (body === null) {
+      if (state.status !== 200) {
+        stateStatus = state.status;
+      } else if (body === null) {
         guardFailure = plan.guard ? 'state_unreadable' : guardFailure;
         drift = plan.verify ? 'state_unreadable' : drift;
       } else {
@@ -95,13 +134,26 @@ export class DeltaTurns extends BenchmarkRunner {
           guardFailure = deltaGuardFailure(body.metadata, plan.expectDelta);
         }
         if (plan.verify) {
-          // `/runs/wait` returns the state values themselves; `/state` nests them.
-          drift = stateDrift(safeJson(response.body), body.values, CHANNEL);
+          drift = digestDrift(safeJson(response.body)?.payload_sha256, coldDigest(body.values));
+        }
+      }
+      if (plan.verify) {
+        // Servers before 0.16 have no storage endpoint. A 404 there is a missing
+        // sample, not a failed request: without this, one 404 per 50 turns is
+        // about 2% and trips the 1% error budget on every cell.
+        const storage = http.get(`${baseUrl}/threads/${threadId}/storage`, {
+          ...requestParams,
+          responseCallback: http.expectedStatuses(200, 404),
+        });
+        addResponse(responses, 'storage', storage);
+        const total = storage.status === 200 ? safeJson(storage.body)?.total_bytes : null;
+        if (typeof total === 'number') {
+          threadStorageBytes.add(total, { depth: plan.depthBucket, graph: plan.graphId });
         }
       }
     }
 
-    return okResult(responses, { rawResponse: response, plan, guardFailure, drift });
+    return okResult(responses, { rawResponse: response, plan, guardFailure, drift, stateStatus });
   }
 
   static validate(
@@ -120,6 +172,7 @@ export class DeltaTurns extends BenchmarkRunner {
         'Response does not contain __error__': () => json?.__error__ === undefined,
         'Run advanced the thread by one turn': () =>
           json == null || plan == null || json.counter === plan.steps,
+        'Cold state read succeeded': () => result.data?.stateStatus == null,
         'Delta storage is as expected': () => result.data?.guardFailure == null,
         'Cold re-read matches the run': () => result.data?.drift == null,
       });
@@ -131,6 +184,7 @@ export class DeltaTurns extends BenchmarkRunner {
       const detail = [
         plan ? `turn=${plan.steps} graph=${plan.graphId}` : '',
         res ? `status=${res.status}` : '',
+        result.data?.stateStatus ? `state_status=${result.data.stateStatus}` : '',
         result.data?.guardFailure ? `guard=${result.data.guardFailure}` : '',
         result.data?.drift ? `drift=${result.data.drift}` : '',
       ]
@@ -139,7 +193,9 @@ export class DeltaTurns extends BenchmarkRunner {
       logFailure(DeltaTurns.toString(), result, { extra: detail });
       recordError(
         classifyError({
-          response: res ?? result.responses?.create_thread ?? null,
+          response: result.data?.stateStatus
+            ? result.responses?.state ?? null
+            : res ?? result.responses?.create_thread ?? null,
           hasApiError: json?.__error__ !== undefined,
           missingMessage: result.data?.drift != null || result.data?.guardFailure != null,
         }),
@@ -167,3 +223,22 @@ function safeJson(body: string | null | ArrayBuffer): Record<string, any> | null
     return null;
   }
 }
+
+function coldDigest(values: Record<string, unknown> | undefined): string | null {
+  if (values == null || typeof values !== 'object') return null;
+  // Reproduce the graph's payload_digest: every bytes channel in order, then
+  // the concatenated message contents. Absent channels default to empty, so
+  // they add nothing, exactly as the unwritten bytes channels do server-side.
+  const hasher = crypto.createHash('sha256');
+  for (const name of PAYLOAD_CHANNELS) {
+    const stored = values[name];
+    if (typeof stored === 'string' && stored.length > 0) {
+      hasher.update(encoding.b64decode(stored));
+    }
+  }
+  const messages = values.messages;
+  if (Array.isArray(messages) && messages.length > 0) {
+    hasher.update(messages.map((m) => String((m as { content?: unknown })?.content ?? '')).join(''));
+  }
+  return hasher.digest('hex');
+}

langgraph_api-0.16.0/benchmark/benchmark-runners/interrupt_resume.ts

@@ -0,0 +1,119 @@
+import { BenchmarkRunner } from './benchmark-runner.js';
+import type { ErrorMetrics } from './benchmark-runner.js';
+import { check } from 'k6';
+import http from 'k6/http';
+import type { BenchmarkResult, BenchmarkGraphOptions, HttpResponse } from './types.js';
+import { addResponse, okResult, failResult } from './types.js';
+import { classifyError, recordError } from './classify-error.js';
+import { logFailure } from './log-failure.js';
+
+interface InterruptResumeData {
+  paused: ReturnType<typeof http.post>;
+  resumed: ReturnType<typeof http.post>;
+  expectedSteps: number;
+}
+
+/**
+ * Pause a run mid-way and resume it. The virtual user is the human: it answers
+ * at once. What is measured is the server's work around the pause: writing the
+ * paused checkpoint, and on resume reading the whole thread state back before
+ * continuing, which is the most expensive read a delta channel can face.
+ */
+export class InterruptResume extends BenchmarkRunner {
+  static run(
+    baseUrl: string,
+    requestParams: Record<string, unknown>,
+    benchmarkGraphOptions: BenchmarkGraphOptions
+  ): BenchmarkResult<InterruptResumeData> {
+    const base = benchmarkGraphOptions.context;
+    const context = {
+      ...base,
+      interrupt_at: base.interrupt_at || Math.max(1, Math.ceil(base.steps / 2)),
+    };
+    const config = { recursion_limit: Math.max(context.expand, context.steps) + 4 };
+    const responses: Record<string, HttpResponse> = {};
+
+    const thread = http.post(`${baseUrl}/threads`, '{}', requestParams);
+    addResponse(responses, 'create_thread', thread);
+    if (thread.status !== 200) {
+      return failResult('create_thread', responses) as BenchmarkResult<InterruptResumeData>;
+    }
+    const threadId = (thread.json() as { thread_id: string }).thread_id;
+    const url = `${baseUrl}/threads/${threadId}/runs/wait`;
+
+    const paused = http.post(
+      url,
+      JSON.stringify({ assistant_id: benchmarkGraphOptions.graph_id, input: {}, context, config }),
+      requestParams
+    );
+    addResponse(responses, 'pause', paused);
+    if (paused.status !== 200) {
+      return failResult('pause', responses) as BenchmarkResult<InterruptResumeData>;
+    }
+
+    const resumed = http.post(
+      url,
+      JSON.stringify({
+        assistant_id: benchmarkGraphOptions.graph_id,
+        command: { resume: true },
+        context,
+        config,
+      }),
+      requestParams
+    );
+    addResponse(responses, 'resume', resumed);
+    return okResult(responses, { paused, resumed, expectedSteps: context.steps });
+  }
+
+  static validate(
+    result: BenchmarkResult<InterruptResumeData>,
+    errorMetrics: ErrorMetrics
+  ): boolean {
+    const data = result.data;
+    const pausedJson = safeJson(data?.paused?.body);
+    const resumedJson = safeJson(data?.resumed?.body);
+    let success = false;
+    try {
+      success = check(result, {
+        'Pause request succeeded': () => (data?.paused?.status ?? 0) === 200,
+        'Run paused on an interrupt': () => Array.isArray(pausedJson?.__interrupt__) && pausedJson.__interrupt__.length > 0,
+        'Resume request succeeded': () => (data?.resumed?.status ?? 0) === 200,
+        'Resumed run finished every step': () =>
+          resumedJson != null &&
+          resumedJson.__interrupt__ === undefined &&
+          resumedJson.__error__ === undefined &&
+          resumedJson.counter === data?.expectedSteps,
+      });
+    } catch (error) {
+      console.log(`Unknown error checking result: ${(error as Error).message}`);
+    }
+    if (!success) {
+      logFailure(InterruptResume.toString(), result, {
+        extra: `pause=${data?.paused?.status} resume=${data?.resumed?.status} counter=${resumedJson?.counter} expected=${data?.expectedSteps}`,
+      });
+      recordError(
+        classifyError({
+          response: data?.resumed ?? data?.paused ?? result.responses?.create_thread ?? null,
+          hasApiError: resumedJson?.__error__ !== undefined || pausedJson?.__error__ !== undefined,
+          missingMessage: resumedJson != null && resumedJson.counter !== data?.expectedSteps,
+        }),
+        errorMetrics
+      );
+    }
+    return success;
+  }
+
+  static toString(): string {
+    return 'interrupt_resume';
+  }
+}
+
+function safeJson(body: unknown): Record<string, any> | null {
+  if (typeof body !== 'string') return null;
+  try {
+    const parsed = JSON.parse(body);
+    return parsed != null && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    return null;
+  }
+}

{langgraph_api-0.15.4 → langgraph_api-0.16.0}/benchmark/benchmark-runners/types.ts

@@ -44,6 +44,9 @@ export interface BenchmarkContext {
   burst_mode: boolean;
   burst_probability: number;
   burst_size: number;
+  payload?: 'text' | 'random' | 'messages';
+  channels?: number;
+  interrupt_at?: number;
 }
 
 /**