langgraph-api 0.16.0.dev5__tar.gz → 0.16.0.dev7__tar.gz

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.16.0.dev5
+Version: 0.16.0.dev7
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/README.md

@@ -7,7 +7,7 @@ This directory contains k6 + Python/Node benchmark tooling for LangGraph API.
 - `make benchmark-ramp` - Ramps up to a set number of concurrent virtual users, holds plateau load, then ramps down.
   - **Environment variables:**
     - `LOAD_SIZE`, `LEVELS`, `PLATEAU_DURATION`
-    - `P95_RUN_DURATION`, `AVERAGE_RUN_DURATION`
+    - `P95_RUN_DURATION`
   - **Example:**
     - `BENCHMARK_PROFILE=streaming-long BENCHMARK_TYPE=stream_write LOAD_SIZE=300 LEVELS=3 make benchmark-ramp`
 
@@ -25,7 +25,7 @@ This directory contains k6 + Python/Node benchmark tooling for LangGraph API.
 - `make benchmark-reset` - Cleans benchmark server state via `clean-cli.js`.
 - `make benchmark-new-revision` - Triggers a new benchmark deployment revision.
 
-- `make benchmark-capacity-ci` - Runs the daily capacity matrix for one deployment (`DEPLOYMENT` env). Uses `staircase.py` per cell (default 200–2000 VUs, step 100). Config lives in `capacity-ci.js`.
+- `make benchmark-capacity-ci` - Runs the daily capacity matrix for one deployment (`DEPLOYMENT` env). Uses `staircase.py` per cell with k6's `constant-arrival-rate` executor (open model: a fixed request rate per step, so capacity reads as the service rate; requests k6 has to drop count as failures). Geometry is per deployment in `capacity-ci.js` (dedicated 50 to 600 req/s in steps of 50, serverless 10 to 70 in steps of 5). `K6_EXECUTOR=constant-vus` restores the closed model. A cell that never meets SLO still reports `throughputCeiling`, the best runs/sec any step served.
   - **Example:**
     - `DEPLOYMENT=daily-cap-benchmark-dedicated make benchmark-capacity-ci`
 
@@ -48,6 +48,7 @@ The runner contains the actual logic each virtual user will execute during the b
 - `store` - Writes, reads, validates, and deletes a Store item with a configurable payload size.
 - `meta_workload` (randomly delegates to one of the others)
 - `delta_turns` - Keeps one thread per VU and adds a turn to it every iteration, so latency can be read against thread depth. Built for the delta-channel cells; see below.
+- `interrupt_resume` - Starts a run that pauses on an `interrupt()` before step `interrupt_at`, then resumes it at once. Measures the paused checkpoint write and the full state read on resume. Use with the `interrupt` profile.
 
 ## Profiles (`BENCHMARK_PROFILE`)
 
@@ -60,6 +61,9 @@ Profiles are quick ways to configure the benchmark graph for any run depending o
 - `parallel-small` - Stateful high-fanout profile with small checkpoint payloads.
 - `parallel-medium` - Stateful high-fanout profile with medium checkpoint payloads.
 - `delta-turns` - Cheap turns for the delta-channel cells, so a VU reaches the depth where a `snapshot_frequency` of 1000 fires.
+- `wide-deep` - 50 parallel branches held for 20 supersteps: concurrent checkpoint writes and worker spread at the same time.
+- `messages` - State grows as a chat message list (2 KB per step, 10 steps), so each checkpoint serializes structured objects rather than appending bytes.
+- `interrupt` - 5 steps, pauses before step 3. For the `interrupt_resume` runner.
 
 Each profile defines:
 
@@ -85,6 +89,9 @@ Context overrides (merged on top of profile):
 - `EXPAND` - Fanout width (number of parallel branches at start).
 - `STEPS` - Number of graph turns/iterations to execute.
 - `CHECKPOINT_SIZE` - Bytes written per checkpoint operation.
+- `PAYLOAD` - `text` (default; compresses like real state), `random` (incompressible worst case) or `messages` (a chat message of `CHECKPOINT_SIZE` chars per step).
+- `CHANNELS` - Spread the bytes payload across this many channels (1 to 8). Postgres stores one row per changed channel per version, so this costs rows, not only bytes.
+- `INTERRUPT_AT` - Pause the run before this step (1-based) until resumed. 0 never pauses.
 - `STORE_VALUE_SIZE` - Store runner payload size in bytes (defaults to 1 MiB).
 - `LLM_ENABLED` - Enables fake LLM streaming inside the benchmark graph.
 - `STREAM_SIZE` - Normal total streamed payload size (bytes).
@@ -131,12 +138,13 @@ replaying more writes, so the deliverable is which curve is worse and where they
 | control | `GRAPH_ID=benchmark EXPECT_DELTA=false` |
 | delta, snapshot every 1000 | `GRAPH_ID=benchmark_delta` |
 | delta, snapshot every 50 | `GRAPH_ID=benchmark_delta_freq` |
+| delta on a message list | `GRAPH_ID=benchmark_delta_messages PAYLOAD=messages` |
 | migration | `GRAPH_ID=benchmark_delta MIGRATE_FROM_GRAPH_ID=benchmark MIGRATE_AFTER=200` |
 
 ```bash
 BASE_URL=... BENCHMARK_TYPE=delta_turns BENCHMARK_PROFILE=delta-turns \
   GRAPH_ID=benchmark_delta LOAD_SIZE=25 LEVELS=1 PLATEAU_DURATION=300 \
-  P95_RUN_DURATION=... AVERAGE_RUN_DURATION=... k6 run --system-tags='[]' ramp.js
+  P95_RUN_DURATION=... k6 run --system-tags='[]' ramp.js
 ```
 
 Knobs beyond the common ones:
@@ -153,14 +161,14 @@ Two things to know before reading any result:
 - **Run at low `LOAD_SIZE`.** Depth and load fight each other: saturate the deployment and
   turn time measures queueing, not reconstruction. Verify the control curve is flat before
   trusting the delta one.
-- **`successful_runs` will fail its threshold.** The count is derived from a duration table
-  that assumes the existing workload. Pass `P95_RUN_DURATION` and `AVERAGE_RUN_DURATION`,
-  and treat these cells as report-only until the numbers settle.
+- **The ramp gates on p95 and error rate only.** Pass a `P95_RUN_DURATION` that fits the
+  cell, and treat these cells as report-only until the numbers settle.
 
 The runner fails a turn rather than reporting a false pass when delta is not actually on:
 the `DeltaChannel` import is version-guarded, and its stub degrades to a plain channel.
-Storage savings, the reason the feature exists, are not measured here; that needs a query
-against the deployment database, which CI cannot reach.
+At every `VERIFY_EVERY` turn it also reads `GET /threads/{thread_id}/storage` and records
+`thread_storage_bytes` tagged by depth bucket and graph, so the control and delta cells can
+be compared on the bytes they occupy, which is what the feature exists to reduce.
 
 ## Outputs
 
@@ -184,6 +192,10 @@ against the deployment database, which CI cannot reach.
 2. Fill in `deploymentId` and `baseUrl` in `CAPACITY_DEPLOYMENTS` at the top of [`capacity-ci.js`](capacity-ci.js).
 3. Scheduled run: two parallel CI jobs (one per deployment), aggregate job posts Slack summary.
 
+Each cell also records the deployment's own view of its window (queue and API pod counts, CPU peaks) from the control plane's deployment metrics route, and the posted table lists it under each deployment. Every tier starts at one queue pod and autoscales during a cell, so a capacity number is partly a pod-count number; pinning pods at the tier maximum would need the benchmark deployments recreated as `external_docker` sources, the only kind whose `resource_spec` the API accepts.
+
+Both the daily and the capacity reports compare tonight with the previous scheduled nights (`history.js`, artifacts fetched with `gh`), flagging a figure only when it leaves that runner's own observed range.
+
 Local commands:
 
 ```bash
@@ -201,3 +213,12 @@ node capacity-ci.js --aggregate --summaries-dir ./artifacts --slack
 
 - [k6 docs](https://k6.io/docs/)
 - [k6 JavaScript API](https://k6.io/docs/javascript-api/)
+
+## What a run returns
+
+The graph's output schema is `counter`, `start_time`, `end_time`, `payload_bytes` and
+`payload_sha256`. The payload channels themselves stay out of `/runs/wait` bodies and
+`values` events, so `CHECKPOINT_SIZE` drives database writes only; before this, every
+response carried the whole payload base64-encoded and write cost could not be told from
+egress. `GET /threads/{thread_id}/state` still returns every channel. The delta cells
+verify a cold re-read by hashing it and comparing with `payload_sha256`.

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/benchmark-runners/benchmark_profiles.ts

@@ -13,6 +13,9 @@ export interface BenchmarkContext {
   burst_mode: boolean;
   burst_probability: number;
   burst_size: number;
+  payload?: 'text' | 'random' | 'messages';
+  channels?: number;
+  interrupt_at?: number;
 }
 
 export interface BenchmarkProfileEnv {
@@ -32,6 +35,9 @@ export interface BenchmarkProfileEnv {
   BURST_MODE?: string;
   BURST_PROBABILITY?: string;
   BURST_SIZE?: string;
+  PAYLOAD?: string;
+  CHANNELS?: string;
+  INTERRUPT_AT?: string;
 }
 
 export interface BenchmarkProfile {
@@ -173,6 +179,64 @@ export const BENCHMARK_PROFILES: Record<string, BenchmarkProfile> = {
       burst_size: 0,
     },
   },
+  // Width held for many supersteps: concurrent checkpoint writes and worker
+  // spread at the same time, which the wide-but-one-step profiles never do.
+  'wide-deep': {
+    resumable: false,
+    runMode: 'stateless',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0.2,
+      expand: 50,
+      steps: 20,
+      checkpoint_size: 1_000,
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
+  // State grows as a message list, so each checkpoint serializes structured
+  // objects rather than appending bytes.
+  messages: {
+    resumable: false,
+    runMode: 'stateful',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0.2,
+      expand: 1,
+      steps: 10,
+      checkpoint_size: 2_000,
+      payload: 'messages',
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
+  // For the interrupt_resume runner.
+  interrupt: {
+    resumable: false,
+    runMode: 'stateful',
+    context: {
+      delay: 0,
+      delay_jitter_ratio: 0.2,
+      expand: 1,
+      steps: 5,
+      checkpoint_size: 1_000,
+      interrupt_at: 3,
+      llm_enabled: false,
+      stream_size: 0,
+      chunk_size: 1,
+      burst_mode: false,
+      burst_probability: 0,
+      burst_size: 0,
+    },
+  },
 };
 
 export function getBenchmarkProfile(profileName?: string): { name: string; profile: BenchmarkProfile } {
@@ -241,6 +305,9 @@ export function get_profile(env: BenchmarkProfileEnv): ParsedBenchmarkProfile {
       ? { burst_probability: parseOptionalFloat(env.BURST_PROBABILITY)! }
       : {}),
     ...(parseOptionalInt(env.BURST_SIZE) !== undefined ? { burst_size: parseOptionalInt(env.BURST_SIZE)! } : {}),
+    ...(env.PAYLOAD ? { payload: env.PAYLOAD as 'text' | 'random' | 'messages' } : {}),
+    ...(parseOptionalInt(env.CHANNELS) !== undefined ? { channels: parseOptionalInt(env.CHANNELS)! } : {}),
+    ...(parseOptionalInt(env.INTERRUPT_AT) !== undefined ? { interrupt_at: parseOptionalInt(env.INTERRUPT_AT)! } : {}),
   });
 
   return {

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/benchmark-runners/benchmarks.ts

@@ -9,6 +9,7 @@ import { ThreadsSearchMetadata } from './threads_search_metadata.js';
 import { Store } from './store.js';
 import { MetaWorkload } from './meta_workload.js';
 import { DeltaTurns } from './delta_turns.js';
+import { InterruptResume } from './interrupt_resume.js';
 import type { BenchmarkRunner } from './benchmark-runner.js';
 
 export class Benchmarks {
@@ -36,6 +37,8 @@ export class Benchmarks {
         return MetaWorkload;
       case DeltaTurns.toString():
         return DeltaTurns;
+      case InterruptResume.toString():
+        return InterruptResume;
       default:
         throw new Error(`Unknown benchmark type: ${type}`);
     }

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/benchmark-runners/delta-turns-lib.test.js

@@ -3,6 +3,7 @@ import assert from 'node:assert/strict';
 import {
   planTurn,
   deltaGuardFailure,
+  digestDrift,
   stateDrift,
   unusableCheckpointSize,
 } from './dist/delta-turns-lib.js';
@@ -167,3 +168,15 @@ test('the delta-turns profile is wired and writes a growing channel', () => {
   assert.equal(profile.context.delay, 0);
   assert.equal(profile.context.steps, 1);
 });
+
+test('matching digests are no drift, and a changed byte anywhere is', () => {
+  const live = 'a'.repeat(64);
+  assert.equal(digestDrift(live, live), null);
+  assert.equal(digestDrift(live, 'b'.repeat(64)), `digest_drift:${'a'.repeat(12)}!=${'b'.repeat(12)}`);
+});
+
+test('a missing digest on either side is reported, not passed', () => {
+  assert.equal(digestDrift(undefined, 'a'.repeat(64)), 'run_digest_missing');
+  assert.equal(digestDrift('', 'a'.repeat(64)), 'run_digest_missing');
+  assert.equal(digestDrift('a'.repeat(64), null), 'cold_digest_missing');
+});

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/benchmark-runners/delta-turns-lib.ts

@@ -100,3 +100,13 @@ function encodeChannel(value: unknown): string {
   if (value == null) return '';
   return typeof value === 'string' ? value : JSON.stringify(value);
 }
+
+/**
+ * The run reports the SHA-256 of its stored payload; the cold read is hashed
+ * the same way. A rebuild that loses or corrupts a write changes the hash.
+ */
+export function digestDrift(liveDigest: unknown, coldDigest: unknown): string | null {
+  if (typeof liveDigest !== 'string' || liveDigest === '') return 'run_digest_missing';
+  if (typeof coldDigest !== 'string' || coldDigest === '') return 'cold_digest_missing';
+  return liveDigest === coldDigest ? null : `digest_drift:${liveDigest.slice(0, 12)}!=${coldDigest.slice(0, 12)}`;
+}

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/benchmark-runners/delta_turns.ts

@@ -1,6 +1,8 @@
 import { BenchmarkRunner } from './benchmark-runner.js';
 import type { ErrorMetrics } from './benchmark-runner.js';
 import { check } from 'k6';
+import crypto from 'k6/crypto';
+import encoding from 'k6/encoding';
 import http from 'k6/http';
 import { Trend } from 'k6/metrics';
 import type { BenchmarkResult, BenchmarkGraphOptions, HttpResponse } from './types.js';
@@ -10,15 +12,25 @@ import { logFailure } from './log-failure.js';
 import {
   planTurn,
   deltaGuardFailure,
-  stateDrift,
+  digestDrift,
   unusableCheckpointSize,
 } from './delta-turns-lib.js';
 import type { DeltaTurnsOptions, TurnPlan } from './delta-turns-lib.js';
 
-const CHANNEL = 'cumulative_checkpoint';
+const PAYLOAD_CHANNELS = [
+  'cumulative_checkpoint',
+  'payload_2',
+  'payload_3',
+  'payload_4',
+  'payload_5',
+  'payload_6',
+  'payload_7',
+  'payload_8',
+] as const;
 
 /** Tagged by depth: VUs ramp in, so clock time is not thread length. */
 const turnDuration = new Trend('turn_duration', true);
+const threadStorageBytes = new Trend('thread_storage_bytes', false);
 
 const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
   migrateFromGraphId: __ENV.MIGRATE_FROM_GRAPH_ID || '',
@@ -95,8 +107,15 @@ export class DeltaTurns extends BenchmarkRunner {
           guardFailure = deltaGuardFailure(body.metadata, plan.expectDelta);
         }
         if (plan.verify) {
-          // `/runs/wait` returns the state values themselves; `/state` nests them.
-          drift = stateDrift(safeJson(response.body), body.values, CHANNEL);
+          drift = digestDrift(safeJson(response.body)?.payload_sha256, coldDigest(body.values));
+        }
+      }
+      if (plan.verify) {
+        const storage = http.get(`${baseUrl}/threads/${threadId}/storage`, requestParams);
+        addResponse(responses, 'storage', storage);
+        const total = storage.status === 200 ? safeJson(storage.body)?.total_bytes : null;
+        if (typeof total === 'number') {
+          threadStorageBytes.add(total, { depth: plan.depthBucket, graph: plan.graphId });
         }
       }
     }
@@ -167,3 +186,22 @@ function safeJson(body: string | null | ArrayBuffer): Record<string, any> | null
     return null;
   }
 }
+
+function coldDigest(values: Record<string, unknown> | undefined): string | null {
+  if (values == null || typeof values !== 'object') return null;
+  // Reproduce the graph's payload_digest: every bytes channel in order, then
+  // the concatenated message contents. Absent channels default to empty, so
+  // they add nothing, exactly as the unwritten bytes channels do server-side.
+  const hasher = crypto.createHash('sha256');
+  for (const name of PAYLOAD_CHANNELS) {
+    const stored = values[name];
+    if (typeof stored === 'string' && stored.length > 0) {
+      hasher.update(encoding.b64decode(stored));
+    }
+  }
+  const messages = values.messages;
+  if (Array.isArray(messages) && messages.length > 0) {
+    hasher.update(messages.map((m) => String((m as { content?: unknown })?.content ?? '')).join(''));
+  }
+  return hasher.digest('hex');
+}

langgraph_api-0.16.0.dev7/benchmark/benchmark-runners/interrupt_resume.ts

@@ -0,0 +1,119 @@
+import { BenchmarkRunner } from './benchmark-runner.js';
+import type { ErrorMetrics } from './benchmark-runner.js';
+import { check } from 'k6';
+import http from 'k6/http';
+import type { BenchmarkResult, BenchmarkGraphOptions, HttpResponse } from './types.js';
+import { addResponse, okResult, failResult } from './types.js';
+import { classifyError, recordError } from './classify-error.js';
+import { logFailure } from './log-failure.js';
+
+interface InterruptResumeData {
+  paused: ReturnType<typeof http.post>;
+  resumed: ReturnType<typeof http.post>;
+  expectedSteps: number;
+}
+
+/**
+ * Pause a run mid-way and resume it. The virtual user is the human: it answers
+ * at once. What is measured is the server's work around the pause: writing the
+ * paused checkpoint, and on resume reading the whole thread state back before
+ * continuing, which is the most expensive read a delta channel can face.
+ */
+export class InterruptResume extends BenchmarkRunner {
+  static run(
+    baseUrl: string,
+    requestParams: Record<string, unknown>,
+    benchmarkGraphOptions: BenchmarkGraphOptions
+  ): BenchmarkResult<InterruptResumeData> {
+    const base = benchmarkGraphOptions.context;
+    const context = {
+      ...base,
+      interrupt_at: base.interrupt_at || Math.max(1, Math.ceil(base.steps / 2)),
+    };
+    const config = { recursion_limit: Math.max(context.expand, context.steps) + 4 };
+    const responses: Record<string, HttpResponse> = {};
+
+    const thread = http.post(`${baseUrl}/threads`, '{}', requestParams);
+    addResponse(responses, 'create_thread', thread);
+    if (thread.status !== 200) {
+      return failResult('create_thread', responses) as BenchmarkResult<InterruptResumeData>;
+    }
+    const threadId = (thread.json() as { thread_id: string }).thread_id;
+    const url = `${baseUrl}/threads/${threadId}/runs/wait`;
+
+    const paused = http.post(
+      url,
+      JSON.stringify({ assistant_id: benchmarkGraphOptions.graph_id, input: {}, context, config }),
+      requestParams
+    );
+    addResponse(responses, 'pause', paused);
+    if (paused.status !== 200) {
+      return failResult('pause', responses) as BenchmarkResult<InterruptResumeData>;
+    }
+
+    const resumed = http.post(
+      url,
+      JSON.stringify({
+        assistant_id: benchmarkGraphOptions.graph_id,
+        command: { resume: true },
+        context,
+        config,
+      }),
+      requestParams
+    );
+    addResponse(responses, 'resume', resumed);
+    return okResult(responses, { paused, resumed, expectedSteps: context.steps });
+  }
+
+  static validate(
+    result: BenchmarkResult<InterruptResumeData>,
+    errorMetrics: ErrorMetrics
+  ): boolean {
+    const data = result.data;
+    const pausedJson = safeJson(data?.paused?.body);
+    const resumedJson = safeJson(data?.resumed?.body);
+    let success = false;
+    try {
+      success = check(result, {
+        'Pause request succeeded': () => (data?.paused?.status ?? 0) === 200,
+        'Run paused on an interrupt': () => Array.isArray(pausedJson?.__interrupt__) && pausedJson.__interrupt__.length > 0,
+        'Resume request succeeded': () => (data?.resumed?.status ?? 0) === 200,
+        'Resumed run finished every step': () =>
+          resumedJson != null &&
+          resumedJson.__interrupt__ === undefined &&
+          resumedJson.__error__ === undefined &&
+          resumedJson.counter === data?.expectedSteps,
+      });
+    } catch (error) {
+      console.log(`Unknown error checking result: ${(error as Error).message}`);
+    }
+    if (!success) {
+      logFailure(InterruptResume.toString(), result, {
+        extra: `pause=${data?.paused?.status} resume=${data?.resumed?.status} counter=${resumedJson?.counter} expected=${data?.expectedSteps}`,
+      });
+      recordError(
+        classifyError({
+          response: data?.resumed ?? data?.paused ?? result.responses?.create_thread ?? null,
+          hasApiError: resumedJson?.__error__ !== undefined || pausedJson?.__error__ !== undefined,
+          missingMessage: resumedJson != null && resumedJson.counter !== data?.expectedSteps,
+        }),
+        errorMetrics
+      );
+    }
+    return success;
+  }
+
+  static toString(): string {
+    return 'interrupt_resume';
+  }
+}
+
+function safeJson(body: unknown): Record<string, any> | null {
+  if (typeof body !== 'string') return null;
+  try {
+    const parsed = JSON.parse(body);
+    return parsed != null && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    return null;
+  }
+}

{langgraph_api-0.16.0.dev5 → langgraph_api-0.16.0.dev7}/benchmark/benchmark-runners/types.ts

@@ -44,6 +44,9 @@ export interface BenchmarkContext {
   burst_mode: boolean;
   burst_probability: number;
   burst_size: number;
+  payload?: 'text' | 'random' | 'messages';
+  channels?: number;
+  interrupt_at?: number;
 }
 
 /**