langgraph-api 0.16.0.dev8__tar.gz → 0.16.0.dev10__tar.gz

{langgraph_api-0.16.0.dev8 → langgraph_api-0.16.0.dev10}/Makefile

@@ -313,6 +313,10 @@ VERSION_KIND ?= patch
 bump-version:
 	uv run --with hatch hatch version $(VERSION_KIND)
 
+sync-runtime-inmem-dep:
+	@python3 -c 'import pathlib,re; v=pathlib.Path("../runtime_inmem/langgraph_runtime_inmem/__init__.py").read_text(); m=re.search(r"__version__\s*=\s*\"(\d+)\.(\d+)", v); major,minor=map(int,m.groups()); p=pathlib.Path("pyproject.toml"); text,n=re.subn(r"\"langgraph-runtime-inmem>=[^\"]+\",", f"\"langgraph-runtime-inmem>={major}.{minor}.0.dev0,<{major}.{minor + 1}.0.dev0\",", p.read_text(), count=1); \
+	assert n==1, "langgraph-runtime-inmem dep not found in pyproject.toml"; p.write_text(text)'
+
 # A2A TCK (Test Compatibility Kit) targets
 # Requires a running server: make start (in another terminal)
 

{langgraph_api-0.16.0.dev8 → langgraph_api-0.16.0.dev10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.16.0.dev8
+Version: 0.16.0.dev10
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE

{langgraph_api-0.16.0.dev8 → langgraph_api-0.16.0.dev10}/benchmark/README.md

@@ -129,7 +129,11 @@ by replaying them. That moves cost from write to read, and the read cost grows w
 thread depth. `delta_turns` measures it: one thread per VU, one turn per iteration,
 `turn_duration` tagged by depth bucket.
 
-Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
+`benchmark-delta.yml` runs four cells nightly, 20 users for 25 minutes each, 1 KB appended per turn (`CHECKPOINT_SIZE`): plain channel, then
+delta snapshotting every 50 (`benchmark_delta_freq`, the deepagents setting), 500
+(`benchmark_delta_500`) and 1000 (`benchmark_delta`, the library default). `delta-report.js` posts
+a headline table (non-delta vs delta at 50: read, write, storage by depth) and one table per metric
+across the three frequencies. The messages and migration cells stay manual. Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
 with depth for different reasons, the control by re-writing a growing value and delta by
 replaying more writes, so the deliverable is which curve is worse and where they cross.
 
@@ -138,6 +142,7 @@ replaying more writes, so the deliverable is which curve is worse and where they
 | control | `GRAPH_ID=benchmark EXPECT_DELTA=false` |
 | delta, snapshot every 1000 | `GRAPH_ID=benchmark_delta` |
 | delta, snapshot every 50 | `GRAPH_ID=benchmark_delta_freq` |
+| delta, snapshot every 500 | `GRAPH_ID=benchmark_delta_500` |
 | delta on a message list | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
 | migration | `GRAPH_ID=benchmark_delta MIGRATE_FROM_GRAPH_ID=benchmark MIGRATE_AFTER=200` |
 
@@ -154,7 +159,7 @@ Knobs beyond the common ones:
 - `MIGRATE_FROM_GRAPH_ID` / `MIGRATE_AFTER` - run the first N turns on another graph, then
   switch on the same thread.
 - `VERIFY_EVERY` - cold re-read cadence in turns. Default 50, `0` disables.
-- `DEPTH_BUCKET` - width of a depth bucket. Default 100.
+- `DEPTH_BUCKET` - width of a depth bucket. Defaults to `VERIFY_EVERY`, so each bucket holds one cold read per thread.
 
 Two things to know before reading any result:
 
@@ -166,7 +171,9 @@ Two things to know before reading any result:
 
 The runner fails a turn rather than reporting a false pass when delta is not actually on:
 the `DeltaChannel` import is version-guarded, and its stub degrades to a plain channel.
-At every `VERIFY_EVERY` turn it also reads `GET /threads/{thread_id}/storage` and records
+At every `VERIFY_EVERY` turn it also times the cold `GET /threads/{thread_id}/state` as
+`state_read_duration` (the read path alone; a turn is one save plus the read before it), and
+reads `GET /threads/{thread_id}/storage` and records
 `thread_storage_bytes` tagged by depth bucket and graph, so the control and delta cells can
 be compared on the bytes they occupy, which is what the feature exists to reduce.
 
@@ -185,6 +192,12 @@ be compared on the bytes they occupy, which is what the feature exists to reduce
 
 - Daily ramp benchmark: `.github/workflows/benchmark.yml`
 - Daily capacity matrix: `.github/workflows/benchmark-capacity-matrix.yml`
+- Daily delta-channel cells: `.github/workflows/benchmark-delta.yml`, reported by `delta-report.js`
+
+The staging deployments build their graphs from
+[langgraph-server-benchmarks](https://github.com/langchain-ai/langgraph-server-benchmarks), not
+from this repo. Its `src/agent/graph.py` is a copy of `api/tests/graphs/benchmark.py`; a graph
+change here reaches the nightly only once that copy is updated.
 
 ### Capacity matrix setup
 
@@ -203,7 +216,7 @@ Every report follows one shape so the channel reads the same way each morning, o
 - One table per deployment, about 40 characters wide so it fits a phone, with `▲ better` / `▼ worse` markers on cells that left their usual range.
 - One legend line (pass rule, arrows, dot colors), then a link to the thread, which holds the per-cell or per-run numbers and the longer *How to read*, and to the run's artifacts.
 
-Both reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`). A figure is flagged only when it leaves that cell's own observed range, and only once five same-kind nights exist; before that the table says how many nights it has. The daily benchmark posts one digest per workflow once its last runner finishes (`daily-report.js`, fed by each runner's artifact), not one message per runner.
+Both reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`). A figure is flagged only when it leaves that cell's own observed range, and only once five same-kind nights exist; before that the table says how many nights it has. The daily benchmark posts one digest per workflow once its last runner finishes (`daily-report.js`, fed by each runner's artifact), not one message per runner. The delta-channel report (`delta-report.js`) keeps the plain-vs-delta table in the channel and puts the snapshot-frequency sweep in the thread.
 
 Local commands:
 

{langgraph_api-0.16.0.dev8 → langgraph_api-0.16.0.dev10}/benchmark/benchmark-runners/delta_turns.ts

@@ -30,6 +30,8 @@ const PAYLOAD_CHANNELS = [
 
 /** Tagged by depth: VUs ramp in, so clock time is not thread length. */
 const turnDuration = new Trend('turn_duration', true);
+/** The read path alone: a cold GET /state replays the channel without writing. */
+const stateReadDuration = new Trend('state_read_duration', true);
 const threadStorageBytes = new Trend('thread_storage_bytes', false);
 
 const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
@@ -37,7 +39,8 @@ const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
   migrateAfter: parseInt(__ENV.MIGRATE_AFTER || '0', 10),
   expectDelta: (__ENV.EXPECT_DELTA || 'true') === 'true',
   verifyEvery: parseInt(__ENV.VERIFY_EVERY || '50', 10),
-  bucketSize: parseInt(__ENV.DEPTH_BUCKET || '100', 10),
+  // One bucket per cold read, so every row of the report is one sample per thread.
+  bucketSize: parseInt(__ENV.DEPTH_BUCKET || __ENV.VERIFY_EVERY || '50', 10),
 };
 
 // Each k6 VU runs this module in its own context, so these are per VU.
@@ -49,6 +52,8 @@ interface DeltaTurnsData {
   plan: TurnPlan;
   guardFailure: string | null;
   drift: string | null;
+  /** Non-200 status of the cold state read, which is a failed request, not a correctness failure. */
+  stateStatus: number | null;
 }
 
 export class DeltaTurns extends BenchmarkRunner {
@@ -93,13 +98,24 @@ export class DeltaTurns extends BenchmarkRunner {
 
     let guardFailure: string | null = null;
     let drift: string | null = null;
+    let stateStatus: number | null = null;
     // A failed body is not state, and comparing against it would report drift
     // on top of the failure that caused it.
     if (response.status === 200 && (plan.guard || plan.verify)) {
       const state = http.get(`${baseUrl}/threads/${threadId}/state`, requestParams);
+      // Time to first byte: the server cannot start answering before it has
+      // rebuilt the value, and the body that follows is the whole state, which
+      // at depth is hundreds of KB the turn itself never sends. Wall time made
+      // a plain-channel read cost as much as a whole turn. The turn-2 guard
+      // read is at depth 2, not the bucket's end, so it is not timed.
+      if (state.status === 200 && plan.verify) {
+        stateReadDuration.add(state.timings.waiting, { depth: plan.depthBucket });
+      }
       addResponse(responses, 'state', state);
       const body = state.status === 200 ? safeJson(state.body) : null;
-      if (body === null) {
+      if (state.status !== 200) {
+        stateStatus = state.status;
+      } else if (body === null) {
         guardFailure = plan.guard ? 'state_unreadable' : guardFailure;
         drift = plan.verify ? 'state_unreadable' : drift;
       } else {
@@ -111,7 +127,13 @@ export class DeltaTurns extends BenchmarkRunner {
         }
       }
       if (plan.verify) {
-        const storage = http.get(`${baseUrl}/threads/${threadId}/storage`, requestParams);
+        // Servers before 0.16 have no storage endpoint. A 404 there is a missing
+        // sample, not a failed request: without this, one 404 per 50 turns is
+        // about 2% and trips the 1% error budget on every cell.
+        const storage = http.get(`${baseUrl}/threads/${threadId}/storage`, {
+          ...requestParams,
+          responseCallback: http.expectedStatuses(200, 404),
+        });
         addResponse(responses, 'storage', storage);
         const total = storage.status === 200 ? safeJson(storage.body)?.total_bytes : null;
         if (typeof total === 'number') {
@@ -120,7 +142,7 @@ export class DeltaTurns extends BenchmarkRunner {
       }
     }
 
-    return okResult(responses, { rawResponse: response, plan, guardFailure, drift });
+    return okResult(responses, { rawResponse: response, plan, guardFailure, drift, stateStatus });
   }
 
   static validate(
@@ -139,6 +161,7 @@ export class DeltaTurns extends BenchmarkRunner {
         'Response does not contain __error__': () => json?.__error__ === undefined,
         'Run advanced the thread by one turn': () =>
           json == null || plan == null || json.counter === plan.steps,
+        'Cold state read succeeded': () => result.data?.stateStatus == null,
         'Delta storage is as expected': () => result.data?.guardFailure == null,
         'Cold re-read matches the run': () => result.data?.drift == null,
       });
@@ -150,6 +173,7 @@ export class DeltaTurns extends BenchmarkRunner {
       const detail = [
         plan ? `turn=${plan.steps} graph=${plan.graphId}` : '',
         res ? `status=${res.status}` : '',
+        result.data?.stateStatus ? `state_status=${result.data.stateStatus}` : '',
         result.data?.guardFailure ? `guard=${result.data.guardFailure}` : '',
         result.data?.drift ? `drift=${result.data.drift}` : '',
       ]
@@ -158,7 +182,9 @@ export class DeltaTurns extends BenchmarkRunner {
       logFailure(DeltaTurns.toString(), result, { extra: detail });
       recordError(
         classifyError({
-          response: res ?? result.responses?.create_thread ?? null,
+          response: result.data?.stateStatus
+            ? result.responses?.state ?? null
+            : res ?? result.responses?.create_thread ?? null,
           hasApiError: json?.__error__ !== undefined,
           missingMessage: result.data?.drift != null || result.data?.guardFailure != null,
         }),