langgraph-api 0.13.4__tar.gz → 0.14.0__tar.gz

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/Makefile

@@ -1,7 +1,7 @@
-.PHONY: build release lint format test test_watch start start-inmem start-inmem-license-oss start start-js-server build-go-server start-go-server stop-go-server check-version check-base-imports test-a2a-tck test-a2a-tck-mandatory start-test-a2a-tck start-test-a2a-tck-mandatory start-test-a2a-tck-watch test-mcp-conformance start-test-mcp-conformance list-mcp-scenarios test-auth-read-only-assistant
+.PHONY: build release lint format test test_watch start start-inmem start-inmem-license-oss start build-go-server start-go-server stop-go-server check-version check-base-imports test-a2a-tck test-a2a-tck-mandatory start-test-a2a-tck start-test-a2a-tck-mandatory start-test-a2a-tck-watch test-mcp-conformance start-test-mcp-conformance list-mcp-scenarios test-auth-read-only-assistant
 
 # Environment variables
-LANGSERVE_GRAPHS_ALL = '{"agent": {"path": "./tests/graphs/agent.py:graph", "description": "agent"}, "assistant_id_runtime": "./tests/graphs/assistant_id_runtime.py:graph", "custom_lifespan": "./tests/graphs/my_router.py:graph", "single_node": "./tests/graphs/single_node.py:graph", "benchmark": "./tests/graphs/benchmark.py:graph", "config_graph": "./tests/graphs/config_graph.py:graph", "other": "./tests/graphs/other.py:make_graph", "weather": "./tests/graphs/weather.py:mk_weather_graph", "searchy": "./tests/graphs/searchy.py:graph", "agent_simple": "./tests/graphs/agent_simple.py:graph", "simple_runtime": "./tests/graphs/simple_runtime.py:graph", "agent_interrupt": "./tests/graphs/agent_interrupt.py:graph", "agent_parallel_interrupt": "./tests/graphs/agent_parallel_interrupt.py:graph", "message_type_test": "./tests/graphs/message_type_test.py:graph", "remote_subgraph_parent": "./tests/graphs/remote_subgraph_parent.py:graph", "simple_remote": "./tests/graphs/simple_remote.py:graph", "nested_subgraphs": "./tests/graphs/nested_subgraphs.py:graph", "functional_fibonacci": "./tests/graphs/functional_fibonacci.py:fibonacci", "state_graph_fibonacci": "./tests/graphs/state_graph_fibonacci.py:fibonacci", "max_concurrency_graph": "./tests/graphs/max_concurrency_graph.py:graph", "unserializable_subgraph": "./tests/graphs/unserializable_subgraph.py:graph", "agent_interrupt_text": "./tests/graphs/agent_interrupt_text.py:graph", "agent_echo_stream": "./tests/graphs/agent_echo_stream.py:graph", "agent_tool_stream": "./tests/graphs/agent_tool_stream.py:graph", "runtime_graph": "./tests/graphs/runtime_graph.py:graph", "tool_call_chunk_stream": "./tests/graphs/tool_call_chunk_stream.py:graph", "ui_tool_call_stream": "./tests/graphs/ui_tool_call_stream.py:graph", "agent_metrics_stream": "./tests/graphs/agent_metrics_stream.py:graph", "agent_multimodal_stream": "./tests/graphs/agent_multimodal_stream.py:graph", "agent_bedtime_story": "./tests/graphs/agent_bedtime_story.py:graph", "delta_channel": "./tests/graphs/delta_channel.py:graph", "delta_channel_freq": "./tests/graphs/delta_channel_freq.py:graph", "delta_channel_files": "./tests/graphs/delta_channel_files.py:graph", "delta_channel_files_freq": "./tests/graphs/delta_channel_files_freq.py:graph", "delta_channel_legacy": "./tests/graphs/delta_channel_migration.py:graph_legacy", "delta_channel_delta": "./tests/graphs/delta_channel_migration.py:graph_delta", "delta_channel_delta_filter": "./tests/graphs/delta_channel_migration.py:graph_delta_filter", "delta_channel_messages_freq": "./tests/graphs/delta_channel_messages.py:graph_freq", "delta_channel_messages": "./tests/graphs/delta_channel_messages.py:graph", "delta_channel_messages_legacy": "./tests/graphs/delta_channel_messages.py:graph_legacy", "delta_channel_messages_remove": "./tests/graphs/delta_channel_messages.py:graph_remove", "delta_channel_messages_update": "./tests/graphs/delta_channel_messages.py:graph_update", "delta_channel_messages_delta": "./tests/graphs/delta_channel_messages.py:graph_delta", "store_e2e_async": "./tests/graphs/store_e2e.py:graph_async", "store_e2e_sync": "./tests/graphs/store_e2e.py:graph_sync"}'
+LANGSERVE_GRAPHS_ALL = '{"agent": {"path": "./tests/graphs/agent.py:graph", "description": "agent"}, "assistant_id_runtime": "./tests/graphs/assistant_id_runtime.py:graph", "custom_lifespan": "./tests/graphs/my_router.py:graph", "single_node": "./tests/graphs/single_node.py:graph", "benchmark": "./tests/graphs/benchmark.py:graph", "benchmark_delta": "./tests/graphs/benchmark.py:graph_delta", "benchmark_delta_freq": "./tests/graphs/benchmark.py:graph_delta_freq", "config_graph": "./tests/graphs/config_graph.py:graph", "other": "./tests/graphs/other.py:make_graph", "weather": "./tests/graphs/weather.py:mk_weather_graph", "searchy": "./tests/graphs/searchy.py:graph", "agent_simple": "./tests/graphs/agent_simple.py:graph", "simple_runtime": "./tests/graphs/simple_runtime.py:graph", "agent_interrupt": "./tests/graphs/agent_interrupt.py:graph", "agent_parallel_interrupt": "./tests/graphs/agent_parallel_interrupt.py:graph", "message_type_test": "./tests/graphs/message_type_test.py:graph", "remote_subgraph_parent": "./tests/graphs/remote_subgraph_parent.py:graph", "simple_remote": "./tests/graphs/simple_remote.py:graph", "nested_subgraphs": "./tests/graphs/nested_subgraphs.py:graph", "functional_fibonacci": "./tests/graphs/functional_fibonacci.py:fibonacci", "state_graph_fibonacci": "./tests/graphs/state_graph_fibonacci.py:fibonacci", "max_concurrency_graph": "./tests/graphs/max_concurrency_graph.py:graph", "unserializable_subgraph": "./tests/graphs/unserializable_subgraph.py:graph", "agent_interrupt_text": "./tests/graphs/agent_interrupt_text.py:graph", "agent_echo_stream": "./tests/graphs/agent_echo_stream.py:graph", "agent_tool_stream": "./tests/graphs/agent_tool_stream.py:graph", "runtime_graph": "./tests/graphs/runtime_graph.py:graph", "tool_call_chunk_stream": "./tests/graphs/tool_call_chunk_stream.py:graph", "ui_tool_call_stream": "./tests/graphs/ui_tool_call_stream.py:graph", "agent_metrics_stream": "./tests/graphs/agent_metrics_stream.py:graph", "agent_multimodal_stream": "./tests/graphs/agent_multimodal_stream.py:graph", "agent_bedtime_story": "./tests/graphs/agent_bedtime_story.py:graph", "delta_channel": "./tests/graphs/delta_channel.py:graph", "delta_channel_freq": "./tests/graphs/delta_channel_freq.py:graph", "delta_channel_files": "./tests/graphs/delta_channel_files.py:graph", "delta_channel_files_freq": "./tests/graphs/delta_channel_files_freq.py:graph", "delta_channel_legacy": "./tests/graphs/delta_channel_migration.py:graph_legacy", "delta_channel_delta": "./tests/graphs/delta_channel_migration.py:graph_delta", "delta_channel_delta_filter": "./tests/graphs/delta_channel_migration.py:graph_delta_filter", "delta_channel_messages_freq": "./tests/graphs/delta_channel_messages.py:graph_freq", "delta_channel_messages": "./tests/graphs/delta_channel_messages.py:graph", "delta_channel_messages_legacy": "./tests/graphs/delta_channel_messages.py:graph_legacy", "delta_channel_messages_remove": "./tests/graphs/delta_channel_messages.py:graph_remove", "delta_channel_messages_update": "./tests/graphs/delta_channel_messages.py:graph_update", "delta_channel_messages_delta": "./tests/graphs/delta_channel_messages.py:graph_delta", "store_e2e_async": "./tests/graphs/store_e2e.py:graph_async", "store_e2e_sync": "./tests/graphs/store_e2e.py:graph_sync"}'
 LANGSERVE_GRAPHS_AUTH = '{"agent": {"path": "./tests/graphs/agent.py:graph", "description": "agent"}, "assistant_id_runtime": "./tests/graphs/assistant_id_runtime.py:graph", "config_graph": "./tests/graphs/config_graph.py:graph", "other": "./tests/graphs/other.py:make_graph", "weather": "./tests/graphs/weather.py:mk_weather_graph", "searchy": "./tests/graphs/searchy.py:graph", "agent_simple": "./tests/graphs/agent_simple.py:graph", "simple_runtime": "./tests/graphs/simple_runtime.py:graph", "agent_parallel_interrupt": "./tests/graphs/agent_parallel_interrupt.py:graph", "functional_fibonacci": "./tests/graphs/functional_fibonacci.py:fibonacci", "state_graph_fibonacci": "./tests/graphs/state_graph_fibonacci.py:fibonacci", "runtime_graph": "./tests/graphs/runtime_graph.py:graph"}'
 
 # Go server management
@@ -48,6 +48,7 @@ BG_JOB_TIMEOUT_SECS ?= 86400
 BG_JOB_ISOLATED_LOOPS ?= false
 CRON_SCHEDULER_SLEEP_TIME ?= 0
 RELOAD_ARGS ?= --reload --reload-dir langgraph_api --reload-dir ../runtime_inmem
+LANGGRAPH_AUTH_LANGSMITH ?= {"path": "tests/graphs/jwt_auth.py:auth", "allow_langsmith_api_keys": true}
 
 ifdef CI
 RELOAD_ARGS :=
@@ -166,6 +167,14 @@ test-auth-fastapi-jwt--after-custom-middleware--custom-route-auth:
 test-auth-watch:
 	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_AUTH='{"path": "./tests/graphs/fastapi_jwt_auth.py:auth"}' REDIS_URI=_FAKE DATABASE_URI=:memory: MIGRATIONS_PATH=__inmem__ uv run ptw . -- -x -vv --ff --capture=no $(AUTH_TEST)
 
+test-auth-langsmith-api-keys:
+	LANGGRAPH_RUNTIME_EDITION=inmem \
+	LANGGRAPH_AUTH='$(LANGGRAPH_AUTH_LANGSMITH)' \
+	REDIS_URI=_FAKE \
+	DATABASE_URI=:memory: \
+	MIGRATIONS_PATH=__inmem__ \
+	uv run pytest -v tests/integration_tests/test_langsmith_parallel_auth.py
+
 # dev commands
 LANGGRAPH_DISABLE_FILE_PERSISTENCE ?= true
 LANGSMITH_TRACING ?= false
@@ -223,6 +232,8 @@ start-encrypt:
 		--no-access-log
 
 
+LANGGRAPH_AUTH ?= {"path": "tests/graphs/jwt_auth.py:auth"}
+
 start-auth-jwt:
 	LANGGRAPH_RUNTIME_EDITION=inmem LANGGRAPH_HTTP='$(HTTP_CONFIG)' \
 	LANGGRAPH_AES_KEY='$(LANGGRAPH_AES_KEY)' \
@@ -230,7 +241,7 @@ start-auth-jwt:
 	LANGSERVE_GRAPHS=$(LANGSERVE_GRAPHS_AUTH) \
 	LANGGRAPH_STORE='$(STORE_CONFIG)' \
 	LANGGRAPH_CHECKPOINTER='$(CHECKPOINTER_CONFIG)' \
-	LANGGRAPH_AUTH='{"path": "tests/graphs/jwt_auth.py:auth"}' \
+	LANGGRAPH_AUTH='$(LANGGRAPH_AUTH)' \
 	LANGSMITH_LANGGRAPH_API_VARIANT=test \
 	REDIS_URI=fake \
 	DATABASE_URI=:memory: \
@@ -258,6 +269,13 @@ start-auth-fastapi-jwt:
 		$(RELOAD_ARGS) \
 		--no-access-log
 
+start-auth-langsmith-api-keys:
+	LANGGRAPH_AUTH_TYPE=langsmith \
+	LANGSMITH_AUTH_ENDPOINT=http://localhost:9124 \
+	LANGSMITH_TENANT_ID=8a1f2c4e-5b6d-4f70-9a3b-2c1d0e4f5a6b \
+	$(MAKE) start-auth-jwt \
+		LANGGRAPH_AUTH='$(LANGGRAPH_AUTH_LANGSMITH)'
+
 define RUN_FASTAPI_JWT_MW_ORDERING
 	LANGGRAPH_RUNTIME_EDITION=inmem \
 	LANGGRAPH_HTTP='{"app": "./tests/graphs/fastapi_jwt_middleware_ordering.py:app", "middleware_order": "$(1)", "enable_custom_route_auth": $(2)}' \
@@ -292,22 +310,6 @@ start-auth-fastapi-jwt--after-custom-middleware--custom-route-auth:
 	$(call RUN_FASTAPI_JWT_MW_ORDERING,middleware_first,true)
 
 
-start-js-server:
-	@echo "Building and starting Go gRPC server..."
-	$(MAKE) build-go-server
-	$(MAKE) start-go-server
-	@trap '$(MAKE) -C $(CURDIR) stop-go-server' INT TERM EXIT; \
-	echo "Installing JS server dependencies..."; \
-	cd ../public-api-server-js && yarn install; \
-	echo "Building JS server..."; \
-	cd ../public-api-server-js && yarn run build; \
-	echo "Starting JS server on port 9123..."; \
-	cd ../public-api-server-js && \
-	LANGSERVE_GRAPHS=$(LANGSERVE_GRAPHS_ALL) \
-	LANGGRAPH_CONFIG='{"agent": {"configurable": {"model_name": "openai"}}}' \
-	FF_USE_JS_API=true \
-	PORT=9123 yarn start
-
 VERSION_KIND ?= patch
 
 bump-version:

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.13.4
+Version: 0.14.0
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
@@ -16,8 +16,8 @@ Requires-Dist: jsonschema-rs<0.45,>=0.20.0
 Requires-Dist: langchain-core>=0.3.64
 Requires-Dist: langchain-protocol<0.1,>=0.0.18
 Requires-Dist: langgraph-checkpoint<5,>=3.0.1
-Requires-Dist: langgraph-runtime-inmem<0.34.0.dev0,>=0.33.0.dev0
-Requires-Dist: langgraph-sdk>=0.3.5
+Requires-Dist: langgraph-runtime-inmem<0.35.0.dev0,>=0.34.0.dev0
+Requires-Dist: langgraph-sdk>=0.4.4
 Requires-Dist: langgraph<2,>=0.4.10
 Requires-Dist: langsmith[otel]>=0.6.3
 Requires-Dist: opentelemetry-api>=0.0.1

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/benchmark/Makefile

@@ -19,6 +19,14 @@ benchmark-new-revision:
 benchmark-staircase:
 	python staircase.py
 
+benchmark-capacity-ci:
+	npm install
+	node capacity-ci.js
+
+benchmark-capacity-matrix:
+	npm install
+	node capacity-ci.js --all-deployments
+
 benchmark-staircase-reset:
 	python ../../scripts/reset_deployment.py -a $(BENCHMARK_APP_NAME) -n $(BENCHMARK_NAMESPACE) -db $(BENCHMARK_DB_INSTANCE)
 	python staircase.py

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/benchmark/README.md

@@ -25,6 +25,14 @@ This directory contains k6 + Python/Node benchmark tooling for LangGraph API.
 - `make benchmark-reset` - Cleans benchmark server state via `clean-cli.js`.
 - `make benchmark-new-revision` - Triggers a new benchmark deployment revision.
 
+- `make benchmark-capacity-ci` - Runs the daily capacity matrix for one deployment (`DEPLOYMENT` env). Uses `staircase.py` per cell (default 200–2000 VUs, step 100). Config lives in `capacity-ci.js`.
+  - **Example:**
+    - `DEPLOYMENT=daily-cap-benchmark-dedicated make benchmark-capacity-ci`
+
+- `make benchmark-capacity-matrix` - Runs the full local capacity matrix for all deployments in `capacity-ci.js` (sequential).
+
+Host API calls use [`host-v2.js`](host-v2.js) (v2 revisions + resource-tiers). [`update-revision.js`](update-revision.js) uses the same module.
+
 ## Benchmark runner types (`BENCHMARK_TYPE`)
 
 The runner contains the actual logic each virtual user will execute during the benchmark and steps to validate the logic worked correctly. Options are:
@@ -39,6 +47,7 @@ The runner contains the actual logic each virtual user will execute during the b
 - `threads_search_metadata` - Validates thread metadata search/filter behavior.
 - `store` - Writes, reads, validates, and deletes a Store item with a configurable payload size.
 - `meta_workload` (randomly delegates to one of the others)
+- `delta_turns` - Keeps one thread per VU and adds a turn to it every iteration, so latency can be read against thread depth. Built for the delta-channel cells; see below.
 
 ## Profiles (`BENCHMARK_PROFILE`)
 
@@ -50,6 +59,7 @@ Profiles are quick ways to configure the benchmark graph for any run depending o
 - `stateless-parallel-medium` - Stateless high-fanout profile with medium checkpoint payloads.
 - `parallel-small` - Stateful high-fanout profile with small checkpoint payloads.
 - `parallel-medium` - Stateful high-fanout profile with medium checkpoint payloads.
+- `delta-turns` - Cheap turns for the delta-channel cells, so a VU reaches the depth where a `snapshot_frequency` of 1000 fires.
 
 Each profile defines:
 
@@ -105,6 +115,53 @@ make benchmark-charts
 make benchmark-clean
 ```
 
+## Delta-channel cells
+
+A `DeltaChannel` stores changes rather than the value, and rebuilds the value on read
+by replaying them. That moves cost from write to read, and the read cost grows with
+thread depth. `delta_turns` measures it: one thread per VU, one turn per iteration,
+`turn_duration` tagged by depth bucket.
+
+Run the cells one at a time and compare `turn_duration` by depth. Both sides get slower
+with depth for different reasons, the control by re-writing a growing value and delta by
+replaying more writes, so the deliverable is which curve is worse and where they cross.
+
+| Cell | Env |
+|---|---|
+| control | `GRAPH_ID=benchmark EXPECT_DELTA=false` |
+| delta, snapshot every 1000 | `GRAPH_ID=benchmark_delta` |
+| delta, snapshot every 50 | `GRAPH_ID=benchmark_delta_freq` |
+| migration | `GRAPH_ID=benchmark_delta MIGRATE_FROM_GRAPH_ID=benchmark MIGRATE_AFTER=200` |
+
+```bash
+BASE_URL=... BENCHMARK_TYPE=delta_turns BENCHMARK_PROFILE=delta-turns \
+  GRAPH_ID=benchmark_delta LOAD_SIZE=25 LEVELS=1 PLATEAU_DURATION=300 \
+  P95_RUN_DURATION=... AVERAGE_RUN_DURATION=... k6 run --system-tags='[]' ramp.js
+```
+
+Knobs beyond the common ones:
+
+- `GRAPH_ID` - graph under test (any runner; defaults to `benchmark`).
+- `EXPECT_DELTA` - whether `GRAPH_ID` is delta-backed. Default `true`.
+- `MIGRATE_FROM_GRAPH_ID` / `MIGRATE_AFTER` - run the first N turns on another graph, then
+  switch on the same thread.
+- `VERIFY_EVERY` - cold re-read cadence in turns. Default 50, `0` disables.
+- `DEPTH_BUCKET` - width of a depth bucket. Default 100.
+
+Two things to know before reading any result:
+
+- **Run at low `LOAD_SIZE`.** Depth and load fight each other: saturate the deployment and
+  turn time measures queueing, not reconstruction. Verify the control curve is flat before
+  trusting the delta one.
+- **`successful_runs` will fail its threshold.** The count is derived from a duration table
+  that assumes the existing workload. Pass `P95_RUN_DURATION` and `AVERAGE_RUN_DURATION`,
+  and treat these cells as report-only until the numbers settle.
+
+The runner fails a turn rather than reporting a false pass when delta is not actually on:
+the `DeltaChannel` import is version-guarded, and its stub degrades to a plain channel.
+Storage savings, the reason the feature exists, are not measured here; that needs a query
+against the deployment database, which CI cannot reach.
+
 ## Outputs
 
 - Ramp:
@@ -113,11 +170,32 @@ make benchmark-clean
   - `summary_<timestamp>.json`
   - `*_chart_*.png` (when chart generation is run)
 - Staircase:
-  - `staircase_summary.json`
+  - `staircase_summary.json` (or `SUMMARY_PATH` when set by capacity CI)
+  - `staircase_summary_<deployment>_<tier>_<runner>.json` (capacity matrix)
 
 ## CI workflows
 
 - Daily ramp benchmark: `.github/workflows/benchmark.yml`
+- Daily capacity matrix: `.github/workflows/benchmark-capacity-matrix.yml`
+
+### Capacity matrix setup
+
+1. Create staging deployments `daily-cap-benchmark-dedicated` and `daily-cap-benchmark-serverless`.
+2. Fill in `deploymentId` and `baseUrl` in `CAPACITY_DEPLOYMENTS` at the top of [`capacity-ci.js`](capacity-ci.js).
+3. Scheduled run: two parallel CI jobs (one per deployment), aggregate job posts Slack summary.
+
+Local commands:
+
+```bash
+# Single deployment (9 cells: 3 tiers × 3 runners)
+DEPLOYMENT=daily-cap-benchmark-dedicated make benchmark-capacity-ci
+
+# All deployments sequentially (18 cells)
+make benchmark-capacity-matrix
+
+# Aggregate downloaded CI artifacts
+node capacity-ci.js --aggregate --summaries-dir ./artifacts --slack
+```
 
 ## References
 

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/benchmark/benchmark-runners/benchmark-runner.ts

@@ -3,6 +3,7 @@ import type { BenchmarkResult, BenchmarkGraphOptions } from './types.js';
 export interface ErrorMetrics {
   server_errors: { add(n: number): void };
   timeout_errors: { add(n: number): void };
+  connection_errors?: { add(n: number): void };
   api_errors?: { add(n: number): void };
   missing_message_errors?: { add(n: number): void };
   other_errors: { add(n: number): void };

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/benchmark/benchmark-runners/benchmark_profiles.ts

@@ -68,6 +68,26 @@ export const BENCHMARK_PROFILES: Record<string, BenchmarkProfile> = {
       burst_size: 0,
     },
   },
+  // Delta-channel cells: cheap turns, so a VU reaches the depth where a
+  // snapshot_frequency of 1000 finally fires. Run these at a low LOAD_SIZE;
+  // saturation turns the measurement into queue time.
+  'delta-turns': {
+    resumable: false,
+    runMode: 'stateful',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0,
+      expand: 1,
+      steps: 1,
+      checkpoint_size: 256,
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
   'streaming-long': {
     resumable: true,
     runMode: 'stateful',

{langgraph_api-0.13.4 → langgraph_api-0.14.0}/benchmark/benchmark-runners/benchmarks.ts

@@ -8,6 +8,7 @@ import { ThreadRunsMetadataSearch } from './thread_runs_metadata_search.js';
 import { ThreadsSearchMetadata } from './threads_search_metadata.js';
 import { Store } from './store.js';
 import { MetaWorkload } from './meta_workload.js';
+import { DeltaTurns } from './delta_turns.js';
 import type { BenchmarkRunner } from './benchmark-runner.js';
 
 export class Benchmarks {
@@ -33,6 +34,8 @@ export class Benchmarks {
         return Store;
       case MetaWorkload.toString():
         return MetaWorkload;
+      case DeltaTurns.toString():
+        return DeltaTurns;
       default:
         throw new Error(`Unknown benchmark type: ${type}`);
     }

langgraph_api-0.14.0/benchmark/benchmark-runners/classify-error.test.js

@@ -0,0 +1,116 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+// Compiled output: the source is TypeScript, and `npm test` runs after
+// `npm run build` both locally and in CI.
+import { classifyError, recordError } from './dist/classify-error.js';
+
+test('5xx is a server error', () => {
+  assert.equal(classifyError({ response: { status: 500 } }), 'server');
+  assert.equal(classifyError({ response: { status: 503 } }), 'server');
+});
+
+test('408 and timeout error strings are timeouts', () => {
+  assert.equal(classifyError({ response: { status: 408 } }), 'timeout');
+  assert.equal(
+    classifyError({ response: { status: 0, error: 'request timeout' } }),
+    'timeout'
+  );
+});
+
+// The bucket that had never once been written to. A refused connection has no
+// HTTP status, so every runner used to file it under something else.
+test('an unreachable deployment is a connection error', () => {
+  assert.equal(
+    classifyError({ response: { status: 0, error: 'dial: connection refused' } }),
+    'connection'
+  );
+  assert.equal(classifyError({ response: { status: 0 } }), 'connection');
+});
+
+test('connection beats the runner-specific fallbacks', () => {
+  // Before this classifier, stream_write reported exactly this case as
+  // missing_message and wait_write as other.
+  const unreachable = { status: 0, error: 'dial: connection refused' };
+  assert.equal(
+    classifyError({ response: unreachable, missingMessage: true }),
+    'connection'
+  );
+  assert.equal(
+    classifyError({ response: unreachable, hasApiError: true }),
+    'connection'
+  );
+});
+
+test('a 200 that came back wrong falls to the runner-specific buckets', () => {
+  const ok = { status: 200 };
+  assert.equal(classifyError({ response: ok, hasApiError: true }), 'api');
+  assert.equal(classifyError({ response: ok, missingMessage: true }), 'missing_message');
+  assert.equal(classifyError({ response: ok }), 'other');
+});
+
+test('4xx that is not 408 is not mistaken for anything else', () => {
+  assert.equal(classifyError({ response: { status: 422 } }), 'other');
+  assert.equal(classifyError({ response: { status: 404 } }), 'other');
+});
+
+test('a missing response is not reported as unreachable', () => {
+  // No response object at all means the runner never got that far; that is not
+  // evidence about the network, so it must not claim `connection`.
+  assert.equal(classifyError({}), 'other');
+  assert.equal(classifyError({ response: null, missingMessage: true }), 'missing_message');
+});
+
+// classifyError picks the bucket; recordError is what actually increments a
+// counter, and it was previously untested. A mutation that unwired
+// connection_errors passed the whole suite.
+function spyMetrics(omit = []) {
+  const names = [
+    'server_errors',
+    'timeout_errors',
+    'connection_errors',
+    'api_errors',
+    'missing_message_errors',
+    'other_errors',
+  ];
+  const counts = {};
+  const metrics = {};
+  for (const name of names) {
+    counts[name] = 0;
+    if (!omit.includes(name)) metrics[name] = { add: (n) => (counts[name] += n) };
+  }
+  return { metrics, counts };
+}
+
+test('recordError increments the counter the bucket names', async (t) => {
+  const cases = [
+    ['server', 'server_errors'],
+    ['timeout', 'timeout_errors'],
+    ['connection', 'connection_errors'],
+    ['api', 'api_errors'],
+    ['missing_message', 'missing_message_errors'],
+    ['other', 'other_errors'],
+  ];
+  for (const [bucket, counter] of cases) {
+    await t.test(`${bucket} -> ${counter}`, () => {
+      const { metrics, counts } = spyMetrics();
+      recordError(bucket, metrics);
+      assert.equal(counts[counter], 1, `${bucket} did not increment ${counter}`);
+      const strays = Object.entries(counts).filter(([k, v]) => k !== counter && v !== 0);
+      assert.deepEqual(strays, [], `${bucket} also incremented ${strays.map(([k]) => k)}`);
+    });
+  }
+});
+
+test('recordError falls back to other_errors for counters the entrypoint omits', () => {
+  // Preserves what the runners open-coded for api_errors and
+  // missing_message_errors, and covers older entrypoints without connection.
+  for (const [bucket, counter] of [
+    ['connection', 'connection_errors'],
+    ['api', 'api_errors'],
+    ['missing_message', 'missing_message_errors'],
+  ]) {
+    const { metrics, counts } = spyMetrics([counter]);
+    recordError(bucket, metrics);
+    assert.equal(counts.other_errors, 1, `${bucket} did not fall back to other_errors`);
+  }
+});

langgraph_api-0.14.0/benchmark/benchmark-runners/classify-error.ts

@@ -0,0 +1,86 @@
+import type { ErrorMetrics } from './benchmark-runner.js';
+
+/**
+ * Which error counter a failed run belongs in.
+ *
+ * `wait_write` and `stream_write` each grew their own copy of this ladder, and
+ * both had the same hole: neither could ever produce `connection`. That bucket
+ * is declared in ramp.js and staircase_step_k6.js, wired into the metrics map,
+ * and printed in every report - but no runner has ever written to it, so it has
+ * read 0 in all 32 scheduled runs on record. A refused connection has no HTTP
+ * status, so it fell through to `missing_message` (stream) or `other` (wait):
+ * "the server answered but left something out" when the truth was "the server
+ * never answered".
+ */
+export type ErrorBucket =
+  | 'connection'
+  | 'timeout'
+  | 'server'
+  | 'api'
+  | 'missing_message'
+  | 'other';
+
+/**
+ * Only the two fields the ladder reads. Structurally satisfied by both k6's
+ * RefinedResponse and our normalized HttpResponse, so runners can pass either
+ * without converting.
+ */
+export interface ClassifiableResponse {
+  status?: number;
+  error?: string;
+}
+
+export interface ClassifyInput {
+  /** The response the run hinges on, if there was one. */
+  response?: ClassifiableResponse | null;
+  /** Runner-specific: the response body carried an `__error__` key. */
+  hasApiError?: boolean;
+  /** Runner-specific: the expected message/counter never arrived. */
+  missingMessage?: boolean;
+}
+
+/** k6 reports a status of 0 when no HTTP response was received at all. */
+function isUnreachable(response?: ClassifiableResponse | null): boolean {
+  if (response == null) return false;
+  const status = response.status;
+  if (status != null && status > 0) return false;
+  return status === 0 || (response.error ?? '') !== '';
+}
+
+function isTimeout(response?: ClassifiableResponse | null): boolean {
+  if (response?.status === 408) return true;
+  return (response?.error ?? '').includes('timeout');
+}
+
+/**
+ * Order matters: a timeout is reported by k6 as an error string on a status-0
+ * response, so it has to be tested before the general unreachable case.
+ */
+export function classifyError(input: ClassifyInput): ErrorBucket {
+  const { response, hasApiError, missingMessage } = input;
+  const status = response?.status;
+
+  if (status != null && status >= 500) return 'server';
+  if (isTimeout(response)) return 'timeout';
+  if (isUnreachable(response)) return 'connection';
+  if (hasApiError) return 'api';
+  if (missingMessage) return 'missing_message';
+  return 'other';
+}
+
+/**
+ * Add one to the counter for `bucket`, falling back to `other_errors` when the
+ * entrypoint did not supply that counter. Preserves the behaviour the runners
+ * open-coded for `api_errors` and `missing_message_errors`.
+ */
+export function recordError(bucket: ErrorBucket, errorMetrics: ErrorMetrics): void {
+  const counters: Record<ErrorBucket, { add(n: number): void } | undefined> = {
+    server: errorMetrics.server_errors,
+    timeout: errorMetrics.timeout_errors,
+    connection: errorMetrics.connection_errors,
+    api: errorMetrics.api_errors,
+    missing_message: errorMetrics.missing_message_errors,
+    other: errorMetrics.other_errors,
+  };
+  (counters[bucket] ?? errorMetrics.other_errors).add(1);
+}