langgraph-api 0.16.9__tar.gz → 0.17.0__tar.gz

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/LICENSE

@@ -90,4 +90,4 @@ these terms.
 
 **use** means anything you do with the software requiring one of your licenses.
 
-**trademark** means trademarks, service marks, and similar rights.
+**trademark** means trademarks, service marks, and similar rights.

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/Makefile

@@ -43,7 +43,7 @@ TEST ?= tests/ ../runtime_inmem/tests/
 # has no [tool.pytest] — dropping pythonpath (e.g. apply_prerelease_version).
 PYTEST_ROOT_ARGS := --rootdir=. -c pyproject.toml
 PYTEST_WORKERS ?= 0
-AUTH_TEST ?= "tests/integration_tests/test_custom_auth.py"
+AUTH_TEST ?= tests/integration_tests/test_custom_auth.py tests/integration_tests/test_auth_mcp.py
 LANGGRAPH_HTTP ?= {"disable_mcp": false, "disable_a2a": false}
 LANGGRAPH_AES_KEY ?= '1234567890123456'
 BG_JOB_TIMEOUT_SECS ?= 86400
@@ -128,7 +128,13 @@ test-auth:
 test-with-server:
 	@bash -euo pipefail -c '$(MAKE) $(START_TARGET) LANGGRAPH_HTTP='\''$(LANGGRAPH_HTTP)'\'' & \
 	pid=$$!; \
-	cleanup() { kill $$pid >/dev/null 2>&1 || true; }; \
+	cleanup() { \
+		if [ -r /proc/$$pid/winpid ]; then \
+			taskkill //F //T //PID "$$(cat /proc/$$pid/winpid)" >/dev/null 2>&1 || true; \
+		else \
+			kill $$pid >/dev/null 2>&1 || true; \
+		fi; \
+	}; \
 	trap cleanup EXIT; \
 	for i in $$(seq 1 $(SERVER_START_TIMEOUT_SECS)); do \
 		if curl -sf "$(SERVER_HEALTH_URL)" >/dev/null; then \

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.16.9
+Version: 0.17.0
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
@@ -17,7 +17,7 @@ Requires-Dist: jsonschema-rs<0.45,>=0.20.0
 Requires-Dist: langchain-core>=0.3.64
 Requires-Dist: langchain-protocol<0.1,>=0.0.18
 Requires-Dist: langgraph-checkpoint<5,>=3.0.1
-Requires-Dist: langgraph-runtime-inmem<0.37.0.dev0,>=0.36.0.dev0
+Requires-Dist: langgraph-runtime-inmem<0.38.0.dev0,>=0.37.0.dev0
 Requires-Dist: langgraph-sdk>=0.4.4
 Requires-Dist: langgraph<2,>=0.4.10
 Requires-Dist: langsmith[otel]>=0.6.3

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/benchmark/README.md

@@ -129,13 +129,11 @@ by replaying them. That moves cost from write to read, and the read cost grows w
 thread depth. `delta_turns` measures it: one thread per VU, one turn per iteration,
 `turn_duration` tagged by depth bucket.
 
-`benchmark-delta.yml` runs four cells nightly, 20 users for 25 minutes each, 1 KB appended per turn (`CHECKPOINT_SIZE`): plain channel, then
-delta snapshotting every 50 (`benchmark_delta_freq`, the deepagents setting), 500
-(`benchmark_delta_500`) and 1000 (`benchmark_delta`, the library default). `delta-report.js` posts
-a headline table (non-delta vs delta at 50: read, write, storage by depth) and one table per metric
-across the three frequencies. The messages and migration cells stay manual. Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
-with depth for different reasons, the control by re-writing a growing value and delta by
-replaying more writes, so the deliverable is which curve is worse and where they cross.
+`benchmark-delta.yml` runs two cells nightly in one k6 run (`DELTA_CELLS`): every VU turns a
+plain-channel thread and a `benchmark_delta_messages` thread (a message list, snapshot every 50,
+the deepagents setting) in rotation, `ITERATIONS` fixed so both reach 1,400 turns.
+`delta-report.js` posts turn, write, read and data at the last turn and puts the by-depth tables
+in the thread; `REPORTS.md` explains the post. The cells below stay available for manual runs.
 
 | Cell | Env |
 |---|---|
@@ -143,7 +141,7 @@ replaying more writes, so the deliverable is which curve is worse and where they
 | delta, snapshot every 1000 | `GRAPH_ID=benchmark_delta` |
 | delta, snapshot every 50 | `GRAPH_ID=benchmark_delta_freq` |
 | delta, snapshot every 500 | `GRAPH_ID=benchmark_delta_500` |
-| delta on a message list | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
+| delta on a message list, snapshot every 50 | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
 | migration | `GRAPH_ID=benchmark_delta MIGRATE_FROM_GRAPH_ID=benchmark MIGRATE_AFTER=200` |
 
 ```bash
@@ -175,7 +173,7 @@ At every `VERIFY_EVERY` turn it also times the cold `GET /threads/{thread_id}/st
 `state_read_duration` (the read path alone; a turn is one save plus the read before it), and
 reads `GET /threads/{thread_id}/storage` and records
 `thread_storage_bytes` tagged by depth bucket and graph, so the control and delta cells can
-be compared on the bytes they occupy, which is what the feature exists to reduce.
+be compared on the bytes they store, which is what the feature exists to reduce.
 
 ## Outputs
 
@@ -190,9 +188,9 @@ be compared on the bytes they occupy, which is what the feature exists to reduce
 
 ## CI workflows
 
-- Daily ramp benchmark: `.github/workflows/benchmark.yml`
-- Daily capacity matrix: `.github/workflows/benchmark-capacity-matrix.yml`
-- Daily delta-channel cells: `.github/workflows/benchmark-delta.yml`, reported by `delta-report.js`
+- Daily ramp benchmark: `.github/workflows/langgraph-api-benchmark.yml`
+- Daily capacity matrix: `.github/workflows/langgraph-api-benchmark-capacity-matrix.yml`
+- Daily delta-channel cells: `.github/workflows/langgraph-api-benchmark-delta.yml`, reported by `delta-report.js`
 
 The staging deployments build their graphs from
 [langgraph-server-benchmarks](https://github.com/langchain-ai/langgraph-server-benchmarks), not
@@ -209,14 +207,7 @@ Each cell also records the deployment's own view of its window (queue and API po
 
 ### Slack reports
 
-Every report follows one shape so the channel reads the same way each morning, on a phone as well as a desktop:
-
-- The first line is the verdict: a dot (green quiet, yellow something regressed or a bottleneck or errors, red something failed), the report name and date, `manual run` when dispatched by hand, the API version, and a clause saying what needs attention. The second line says what the benchmark is, for a reader outside the team.
-- Then `Failed`, `Regressed`, `Improved` and `Bottlenecks` or `Errors` lines, worst first. `Regressed` prints `none` rather than going quiet.
-- One table per deployment, about 40 characters wide so it fits a phone, with `▲ better` / `▼ worse` markers on cells that left their usual range.
-- One legend line (pass rule, arrows, dot colors), then a link to the thread, which holds the per-cell or per-run numbers and the longer *How to read*, and to the run's artifacts.
-
-Both reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`). A figure is flagged only when it leaves that cell's own observed range, and only once five same-kind nights exist; before that the table says how many nights it has. The daily benchmark posts one digest per workflow once its last runner finishes (`daily-report.js`, fed by each runner's artifact), not one message per runner. The delta-channel report (`delta-report.js`) keeps the plain-vs-delta table in the channel and puts the snapshot-frequency sweep in the thread.
+What each post means, for a reader outside the team, is in [REPORTS.md](REPORTS.md); every post links to its section. The channel message carries only the verdict, the lines that need attention and a small table per deployment (with a p95 trend on the daily ones), each fitting a phone; the numbers behind it go to the thread.
 
 Local commands:
 

langgraph_api-0.17.0/benchmark/REPORTS.md

@@ -0,0 +1,174 @@
+# Reading the benchmark reports
+
+The LGP Benchmark Reporter posts four benchmark reports to #test-reports-lsd.
+Each post keeps to the numbers; this page explains what they mean. Every post
+links to its own section below with "What is this?". The weekly image-pull post
+in the same channel is separate and not covered here.
+
+## Reading any report
+
+- **The dot.** 🟢 nothing to look at. 🟡 look at it: a regression, 10 or more
+  failed requests, a bottleneck, a correctness check or a warning line. 🔴 something failed: a
+  run crossed its pass bar, or never produced results.
+- **The first line** is the verdict: report, date, `manual run` when someone
+  dispatched it by hand, the API version that was tested, and what needs
+  attention.
+- **Failed / Errors / Regressed / Bottlenecks** lines only appear when there is
+  something to say, worst first.
+- **Regressed / Improved.** A figure is flagged only when it leaves its own
+  usual range over the recent nights: the 9 before tonight for the daily and
+  delta reports, the last 7 for capacity, where ▲ better and ▼ worse also
+  mark the cell in the table. The usual range is half the spread
+  between the best and worst of those nights, and never less than 10%, so a
+  series that swings 40% night to night is not flagged for moving 15%.
+  Flags start once five nights of history exist; until then the post says how
+  many it has.
+- **The thread** holds the full numbers: per-run or per-cell detail and
+  improvements with before and after values; the daily threads also give the
+  load that ran.
+- **Run** opens the GitHub Actions run, with the raw k6 summaries and charts.
+- `-` in a cell means there is no number: the run produced no results, nothing
+  succeeded, or no thread reached that depth.
+
+## Daily benchmark
+
+**Question it answers:** did latency or errors change for the three core
+request paths, compared with the recent nights?
+
+**How it runs.** [k6](https://k6.io/docs/) starts virtual users (simulated
+clients). Each one repeats one request path as fast as the server answers: it
+sends a request, waits for the response, then sends the next (a "closed
+model"). Users are added in two steps of 60 s, then held for 5 minutes. Each
+path runs as its own k6 run against the benchmark staging deployment. Every
+scheduled night starts from a fresh revision of that deployment; it runs the
+server version staging is on, which is the API version in the first line.
+
+| Row | Runner | What one iteration does | Users |
+|---|---|---|---|
+| `wait` | `wait_write` | start a run and wait for its result (`POST /runs/wait`): run execution and checkpointing | 1,000 |
+| `stream` | `stream_write` | start a run and read its event stream to the end (`POST /runs/stream`) | 1,000 |
+| `search` | `threads_search` | create two threads, set their metadata, then search for them by it: database and metadata search | 300 |
+
+Tables use the one-word row names so they fit a phone; the lines above the
+table use the runner names.
+
+`threads_search` does not delete the threads it creates; the deployment's
+60-minute thread TTL removes them, so each night searches a table of about
+that run's threads, not a growing history.
+
+**The table.**
+
+- `p95 s`: 95% of iterations finished within this many seconds tonight. When
+  it left its usual range, the Regressed or Improved line says so, with the
+  before and after.
+- `last 10`: p95 over the same 10 nights the flags use, oldest on the left,
+  tonight on the right. Taller is slower. Each row has its own scale, so a full
+  bar means that runner's slowest night of the ten, not a number of seconds:
+  compare a row with itself, not with the row below. The bars never stretch
+  small noise to full height, so a real step stands out.
+
+**Pass bar.** Request errors under 1%, p95 under 13 s, and more than 99% of
+iterations succeeding. Missing any of them fails the run and turns the dot red.
+
+**Errors line.** Failed requests by HTTP status, and how many nights in a row
+that runner has had any, as `N+` when every fetched night had them, since the
+streak then started before the history does. 10 or more in one night turn the dot yellow even when
+the run passes.
+
+**Thread.** Average, p95 and p95 of successful iterations only; run and
+failure counts; every figure against the 9 nights before tonight, including average
+latency and run count (run count follows latency under a fixed user count, so
+it is shown but never flagged).
+
+## Scale to zero
+
+The daily benchmark against a second deployment that scales to zero when
+idle. Scheduled runs redeploy it first, so the first minute includes the cold
+start; a manual run skips the redeploy unless it is asked for. 200 users per
+path instead of the daily's 1,000 (300 for search), and the pass bar is p95
+under 25 s to allow for the cold start. Read it like the [daily benchmark](#daily-benchmark).
+
+## Capacity
+
+**Question it answers:** how much traffic can each deployment size handle
+before it breaks its latency target, and what runs out first?
+
+**How it runs.** Two staging deployments, one dedicated and one serverless,
+are resized to each tier (L, M, S) in turn. For each tier and path, k6 sends a
+fixed request rate regardless of how fast the server answers (an "open
+model", the way real traffic arrives), and raises it every 60 s: dedicated by
+50 req/s starting at 50, serverless by 5 starting at 10.
+
+**The table.** Each cell is the highest request rate the tier held for two
+steps while staying on target: at least 99% success, p50 at most 3 s, p95 at
+most 10 s. `70+` means the tier passed the last step, so its real ceiling is
+higher than the test goes. `0` means no step met the target, `ERR` means the cell crashed,
+`-` means it never ran. A crashed cell's error is on the `Why:` line, once per
+distinct error.
+
+**Bottlenecks.** A component (queue, api, database, redis) running at or over
+95% of its CPU request during the steps the tier passed. That is what stops
+the tier going higher. Over 100% means it is bursting above its request.
+
+**Keep in mind.** The graph is one step with no LLM call, so this measures the
+server's own ceiling, not a customer workload. Every tier starts at one queue
+pod and autoscales during a cell, so a capacity number is partly a pod-count
+number.
+
+**Thread.** Improvements with before and after values, then per deployment
+two small tables grouped by tier: peak CPU as % of the pod's request, and pods
+at the start and end of the cell. Both cover the whole cell, including steps
+that failed. Last, the request rate actually served at the top step.
+
+## Delta channel
+
+**Question it answers:** on a long thread, does `DeltaChannel` at the
+deepagents setting (a message list, snapshot every 50 turns) save faster and
+store less than a plain channel, without making reads slower?
+
+**Background.** A plain channel saves the whole value at every checkpoint. A
+`DeltaChannel` saves only what changed, plus a full snapshot every N turns,
+and rebuilds the value on read by replaying the changes since the last
+snapshot. That moves cost from write to read, and read cost grows with the
+distance to the last snapshot.
+
+**How it runs.** One k6 run of 20 users. Each user owns two threads, one plain
+(`base`) and one delta, and takes a turn on each in rotation until both reach
+1,400 turns, so the two are always measured at the same depth seconds apart.
+A turn (`POST /runs/wait`) appends one 1 KB chat message of varied text; text
+repeated every turn would compress away in Postgres and make storage measure
+row overhead. Each user also does a cold state read (`GET /threads/{id}/state`)
+once every 50 turns, at its own offset in that interval, so the reads cover
+the whole climb between two snapshots instead of the moment right after one.
+Storage (`GET /threads/{id}/storage`) is read at turns 350, 700, 1050 and 1400.
+
+**The table.** Both setups at turn 1400: `turn` is the average successful
+turn (`POST /runs/wait`), what a user waits for; `write` is the average
+checkpoint save, timed by the worker between the graph's two steps. Both are
+averages so that a delta thread's snapshot, written on one turn in 50, counts:
+a median never sees it. `read` is the median cold read, timed to the server's
+first byte. `data` is the thread's stored data in MB as
+`GET /threads/{id}/storage` reports it: the stored size of the checkpoint,
+metadata and blob values (`pg_column_size`), without keys, row headers or
+indexes, so not disk use. Lower is better for every row. `last 10` is the
+delta value over tonight and the nine nights before it. A metric outside its
+usual range over five or more nights gets a ⚠️ line above the table.
+
+**Pass bar.** Request errors under 1% and success over 99%. Two correctness
+checks turn the dot yellow on any failure: at turn 2 each thread confirms
+delta is on (or off, for `base`), because a too-old langgraph silently
+degrades a `DeltaChannel` to a plain one; and every cold read is compared with
+what the run returned.
+
+**Thread.** Turn, write, read and data at turns 350, 700, 1050 and 1400, with
+`gain`: how many times better delta is (`same` within 15%, below `1.0x` when
+delta is worse). Then the run's size, failures and correctness checks.
+
+## Changing a report
+
+The builders are `daily-report.js` (daily and scale to zero),
+`capacity-ci.js --aggregate` and `delta-report.js`, all posting through
+`slack.js`. The daily workflows post one digest after their last runner finishes, built from each runner's artifact. History comes from previous scheduled runs' artifacts via
+`history.js`. Each has a `*.test.js` beside it (`npm test`). A dispatch of the
+workflow with `post_to_slack` reproduces a post: about 30 minutes for the
+daily reports, about 2 hours for delta and capacity.

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/benchmark/benchmark-runners/delta-turns-lib.test.js

@@ -1,7 +1,9 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import {
+  parseCells,
   planTurn,
+  readOffset,
   deltaGuardFailure,
   digestDrift,
   stateDrift,
@@ -15,6 +17,8 @@ const BASE = {
   migrateAfter: 0,
   expectDelta: true,
   verifyEvery: 50,
+  readOffset: 0,
+  storageEvery: 0,
   bucketSize: 100,
 };
 
@@ -180,3 +184,32 @@ test('a missing digest on either side is reported, not passed', () => {
   assert.equal(digestDrift('', 'a'.repeat(64)), 'run_digest_missing');
   assert.equal(digestDrift('a'.repeat(64), null), 'cold_digest_missing');
 });
+
+test('cold reads spread across the snapshot interval instead of all landing on it', () => {
+  const offsets = Array.from({ length: 20 }, (_, i) => readOffset(i + 1, 20, 50));
+  assert.equal(offsets[0], 0);
+  assert.equal(offsets.at(-1), 47);
+  assert.equal(new Set(offsets).size, 20, 'every VU reads at its own point');
+  assert.equal(planTurn(47, { ...BASE, readOffset: 3 }).verify, true);
+  assert.equal(planTurn(50, { ...BASE, readOffset: 3 }).verify, false);
+});
+
+test('storage is read on the cold-read turn of each bucket that ends a table row', () => {
+  const options = { ...BASE, bucketSize: 50, readOffset: 10, storageEvery: 350 };
+  assert.equal(planTurn(340, options).storage, true, 'bucket 301-350 ends on 350');
+  assert.equal(planTurn(290, options).storage, false, 'a cold read, but bucket 251-300 is not a row');
+  assert.equal(planTurn(341, options).storage, false, 'not a cold-read turn');
+});
+
+test('without a storage interval, storage is read on every cold read', () => {
+  assert.equal(planTurn(50, { ...BASE, storageEvery: 0 }).storage, true);
+  assert.equal(planTurn(51, { ...BASE, storageEvery: 0 }).storage, false);
+});
+
+test('cells parse from name:graph:expectDelta', () => {
+  assert.deepEqual(parseCells('control:benchmark:false, delta_50:benchmark_delta_messages:true'), [
+    { name: 'control', graphId: 'benchmark', expectDelta: false },
+    { name: 'delta_50', graphId: 'benchmark_delta_messages', expectDelta: true },
+  ]);
+  assert.deepEqual(parseCells(''), []);
+});

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/benchmark/benchmark-runners/delta-turns-lib.ts

@@ -9,9 +9,20 @@ export interface DeltaTurnsOptions {
   expectDelta: boolean;
   /** Cold re-read cadence, in turns. 0 disables. */
   verifyEvery: number;
+  /** Shifts this VU's cold reads off the multiples of `verifyEvery`; see readOffset. */
+  readOffset: number;
+  /** Storage is read on the cold-read turn of every bucket ending on a multiple of this. 0 reads it on every cold read. */
+  storageEvery: number;
   bucketSize: number;
 }
 
+/** One setup the VU turns in rotation, on a thread of its own. */
+export interface DeltaCell {
+  name: string;
+  graphId: string;
+  expectDelta: boolean;
+}
+
 export interface TurnPlan {
   graphId: string;
   /** Cumulative on a reused thread, so turn N asks for N. */
@@ -19,6 +30,7 @@ export interface TurnPlan {
   expectDelta: boolean;
   guard: boolean;
   verify: boolean;
+  storage: boolean;
   depthBucket: string;
 }
 
@@ -32,16 +44,40 @@ export function unusableCheckpointSize(checkpointSize: number | undefined): bool
 export function planTurn(turn: number, options: DeltaTurnsOptions): TurnPlan {
   const migrating = options.migrateFromGraphId !== '' && options.migrateAfter > 0;
   const beforeSwitch = migrating && turn <= options.migrateAfter;
+  const verify = options.verifyEvery > 0 && (turn + options.readOffset) % options.verifyEvery === 0;
   return {
     graphId: beforeSwitch ? options.migrateFromGraphId : options.graphId,
     steps: turn,
     expectDelta: beforeSwitch ? false : options.expectDelta,
     guard: isGuardTurn(turn, options, migrating),
-    verify: options.verifyEvery > 0 && turn % options.verifyEvery === 0,
+    verify,
+    storage: verify && (options.storageEvery <= 0 || bucketEnd(turn, options.bucketSize) % options.storageEvery === 0),
     depthBucket: depthBucket(turn, options.bucketSize),
   };
 }
 
+/** `name:graph:expectDelta`, comma-separated. */
+export function parseCells(spec: string): DeltaCell[] {
+  return spec
+    .split(',')
+    .map((part) => part.trim())
+    .filter(Boolean)
+    .map((part) => {
+      const [name, graphId, expectDelta] = part.split(':');
+      return { name, graphId, expectDelta: expectDelta === 'true' };
+    });
+}
+
+/**
+ * A delta snapshot lands on every multiple of its frequency, so reading every
+ * VU on those turns only ever measures the read right after one. Spreading the
+ * VUs across the interval samples the whole climb between snapshots.
+ */
+export function readOffset(vu: number, vus: number, verifyEvery: number): number {
+  if (verifyEvery <= 0 || vus <= 0) return 0;
+  return Math.floor(((vu - 1) * verifyEvery) / vus) % verifyEvery;
+}
+
 /**
  * Turn 1 has no counters yet, and on a snapshot point Pregel drops the counters
  * entry, so its absence there is correct rather than a fallback.
@@ -51,6 +87,10 @@ function isGuardTurn(turn: number, options: DeltaTurnsOptions, migrating: boolea
   return migrating && turn === options.migrateAfter + 2;
 }
 
+function bucketEnd(turn: number, bucketSize: number): number {
+  return bucketSize <= 0 ? turn : Math.ceil(turn / bucketSize) * bucketSize;
+}
+
 function depthBucket(turn: number, bucketSize: number): string {
   if (bucketSize <= 0) return String(turn);
   const floor = Math.floor((turn - 1) / bucketSize) * bucketSize;

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/benchmark/benchmark-runners/delta_turns.ts

@@ -13,9 +13,11 @@ import {
   planTurn,
   deltaGuardFailure,
   digestDrift,
+  parseCells,
+  readOffset,
   unusableCheckpointSize,
 } from './delta-turns-lib.js';
-import type { DeltaTurnsOptions, TurnPlan } from './delta-turns-lib.js';
+import type { DeltaCell, DeltaTurnsOptions, TurnPlan } from './delta-turns-lib.js';
 
 const PAYLOAD_CHANNELS = [
   'cumulative_checkpoint',
@@ -35,20 +37,30 @@ const stateReadDuration = new Trend('state_read_duration', true);
 const checkpointSaveDuration = new Trend('checkpoint_save', true);
 const threadStorageBytes = new Trend('thread_storage_bytes', false);
 
-const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
+const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId' | 'readOffset'> = {
   migrateFromGraphId: __ENV.MIGRATE_FROM_GRAPH_ID || '',
   migrateAfter: parseInt(__ENV.MIGRATE_AFTER || '0', 10),
   expectDelta: (__ENV.EXPECT_DELTA || 'true') === 'true',
   verifyEvery: parseInt(__ENV.VERIFY_EVERY || '50', 10),
+  storageEvery: parseInt(__ENV.STORAGE_EVERY || '0', 10),
   // One bucket per cold read, so every row of the report is one sample per thread.
   bucketSize: parseInt(__ENV.DEPTH_BUCKET || __ENV.VERIFY_EVERY || '50', 10),
 };
 
+/**
+ * Every VU turns these in rotation, one thread each, so the cells are measured
+ * at the same depth seconds apart. Empty runs GRAPH_ID alone.
+ */
+const CELLS = parseCells(__ENV.DELTA_CELLS || '');
+
 // Each k6 VU runs this module in its own context, so these are per VU.
-let threadId: string | null = null;
-let turn = 0;
+const threads: Record<string, { threadId: string | null; turn: number }> = {};
+let iteration = 0;
 
 interface DeltaTurnsData {
+  cell: string;
+  turnMs: number;
+  tags: { depth: string; cell: string };
   rawResponse: ReturnType<typeof http.post>;
   plan: TurnPlan;
   guardFailure: string | null;
@@ -69,18 +81,31 @@ export class DeltaTurns extends BenchmarkRunner {
       return failResult('checkpoint_size', responses) as BenchmarkResult<DeltaTurnsData>;
     }
 
-    if (threadId === null) {
-      const thread = http.post(`${baseUrl}/threads`, '{}', requestParams);
+    const cell: DeltaCell = CELLS.length > 0
+      ? CELLS[iteration++ % CELLS.length]
+      : { name: benchmarkGraphOptions.graph_id, graphId: benchmarkGraphOptions.graph_id, expectDelta: TURN_CONFIG.expectDelta };
+    const own = (threads[cell.name] ??= { threadId: null, turn: 0 });
+    if (own.threadId === null) {
+      const ttlMinutes = parseInt(__ENV.THREAD_TTL_MINUTES || '0', 10);
+      const body = ttlMinutes > 0 ? { ttl: { strategy: 'delete', ttl: ttlMinutes } } : {};
+      const thread = http.post(`${baseUrl}/threads`, JSON.stringify(body), requestParams);
       addResponse(responses, 'create_thread', thread);
       if (thread.status !== 200) {
         return failResult('create_thread', responses) as BenchmarkResult<DeltaTurnsData>;
       }
-      threadId = (thread.json() as { thread_id: string }).thread_id;
-      turn = 0;
+      own.threadId = (thread.json() as { thread_id: string }).thread_id;
+      own.turn = 0;
     }
+    const threadId = own.threadId;
 
-    turn += 1;
-    const plan = planTurn(turn, { ...TURN_CONFIG, graphId: benchmarkGraphOptions.graph_id });
+    own.turn += 1;
+    const plan = planTurn(own.turn, {
+      ...TURN_CONFIG,
+      graphId: cell.graphId,
+      expectDelta: cell.expectDelta,
+      readOffset: readOffset(__VU, parseInt(__ENV.LOAD_SIZE || '1', 10), TURN_CONFIG.verifyEvery),
+    });
+    const tags = { depth: plan.depthBucket, cell: cell.name };
     const payload = JSON.stringify({
       assistant_id: plan.graphId,
       input: {},
@@ -97,14 +122,11 @@ export class DeltaTurns extends BenchmarkRunner {
       payload,
       requestParams
     );
-    turnDuration.add(Date.now() - startedAt, { depth: plan.depthBucket });
+    const turnMs = Date.now() - startedAt;
     addResponse(responses, 'wait', response);
     const saveMs = response.status === 200 ? safeJson(response.body)?.save_ms : null;
     if (typeof saveMs === 'number' && saveMs > 0) {
-      // Snapshots land on multiples of the bucket width, so the bucket's last
-      // turn is where a snapshot save shows; a percentile over the bucket hides it.
-      const rowTurn = plan.steps % TURN_CONFIG.bucketSize === 0 ? '1' : '0';
-      checkpointSaveDuration.add(saveMs, { depth: plan.depthBucket, row_turn: rowTurn });
+      checkpointSaveDuration.add(saveMs, tags);
     }
 
     let guardFailure: string | null = null;
@@ -120,7 +142,7 @@ export class DeltaTurns extends BenchmarkRunner {
       // a plain-channel read cost as much as a whole turn. The turn-2 guard
       // read is at depth 2, not the bucket's end, so it is not timed.
       if (state.status === 200 && plan.verify) {
-        stateReadDuration.add(state.timings.waiting, { depth: plan.depthBucket });
+        stateReadDuration.add(state.timings.waiting, tags);
       }
       addResponse(responses, 'state', state);
       const body = state.status === 200 ? safeJson(state.body) : null;
@@ -137,7 +159,7 @@ export class DeltaTurns extends BenchmarkRunner {
           drift = digestDrift(safeJson(response.body)?.payload_sha256, coldDigest(body.values));
         }
       }
-      if (plan.verify) {
+      if (plan.storage) {
         // Servers before 0.16 have no storage endpoint. A 404 there is a missing
         // sample, not a failed request: without this, one 404 per 50 turns is
         // about 2% and trips the 1% error budget on every cell.
@@ -148,12 +170,12 @@ export class DeltaTurns extends BenchmarkRunner {
         addResponse(responses, 'storage', storage);
         const total = storage.status === 200 ? safeJson(storage.body)?.total_bytes : null;
         if (typeof total === 'number') {
-          threadStorageBytes.add(total, { depth: plan.depthBucket, graph: plan.graphId });
+          threadStorageBytes.add(total, { ...tags, graph: plan.graphId });
         }
       }
     }
 
-    return okResult(responses, { rawResponse: response, plan, guardFailure, drift, stateStatus });
+    return okResult(responses, { cell: cell.name, turnMs, tags, rawResponse: response, plan, guardFailure, drift, stateStatus });
   }
 
   static validate(
@@ -203,7 +225,12 @@ export class DeltaTurns extends BenchmarkRunner {
       );
       // The thread is now at an unknown depth, so later turns would be tagged
       // with a depth they do not have.
-      threadId = null;
+      const cell = result.data?.cell;
+      if (cell && threads[cell]) threads[cell].threadId = null;
+    } else if (result.data) {
+      // The report counts these per bucket to tell which depth every user
+      // finished, so only a turn that passed every check counts.
+      turnDuration.add(result.data.turnMs, result.data.tags);
     }
     return success;
   }

{langgraph_api-0.16.9 → langgraph_api-0.17.0}/benchmark/benchmark-runners/threads_search_metadata.ts

@@ -13,11 +13,12 @@ interface ThreadsSearchMetadataData {
 }
 
 /**
- * Delete the threads each iteration creates, or leave them to accumulate.
+ * Delete the threads each iteration creates, or leave them to the deployment.
  *
- * Off by default: letting the table grow is how this benchmark shows whether
- * search degrades as thread count rises, and it is the realistic case since
- * deployments do not delete threads per request.
+ * Off by default, since deployments do not delete threads per request. The
+ * nightly deployments' 60-minute thread TTL (langgraph.json in
+ * langgraph-server-benchmarks) removes them after the run, so the table holds
+ * about one run's threads rather than growing night over night.
  *
  * The capacity staircase turns it on, because there the table grows *while*
  * load is ramping, so a slower step cannot be attributed to load or to size.