langgraph-api 0.16.0.dev11__tar.gz → 0.16.0.dev12__tar.gz

{langgraph_api-0.16.0.dev11 → langgraph_api-0.16.0.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.16.0.dev11
+Version: 0.16.0.dev12
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE

{langgraph_api-0.16.0.dev11 → langgraph_api-0.16.0.dev12}/benchmark/benchmark-runners/delta_turns.ts

@@ -32,6 +32,7 @@ const PAYLOAD_CHANNELS = [
 const turnDuration = new Trend('turn_duration', true);
 /** The read path alone: a cold GET /state replays the channel without writing. */
 const stateReadDuration = new Trend('state_read_duration', true);
+const checkpointSaveDuration = new Trend('checkpoint_save', true);
 const threadStorageBytes = new Trend('thread_storage_bytes', false);
 
 const TURN_CONFIG: Omit<DeltaTurnsOptions, 'graphId'> = {
@@ -85,6 +86,9 @@ export class DeltaTurns extends BenchmarkRunner {
       input: {},
       context: { ...benchmarkGraphOptions.context, steps: plan.steps },
       config: { recursion_limit: plan.steps + 2 },
+      // The graph times its own checkpoint save as the gap between its two
+      // steps; a background save would overlap the second step and hide in it.
+      durability: 'sync',
     });
 
     const startedAt = Date.now();
@@ -95,6 +99,13 @@ export class DeltaTurns extends BenchmarkRunner {
     );
     turnDuration.add(Date.now() - startedAt, { depth: plan.depthBucket });
     addResponse(responses, 'wait', response);
+    const saveMs = response.status === 200 ? safeJson(response.body)?.save_ms : null;
+    if (typeof saveMs === 'number' && saveMs > 0) {
+      // Snapshots land on multiples of the bucket width, so the bucket's last
+      // turn is where a snapshot save shows; a percentile over the bucket hides it.
+      const rowTurn = plan.steps % TURN_CONFIG.bucketSize === 0 ? '1' : '0';
+      checkpointSaveDuration.add(saveMs, { depth: plan.depthBucket, row_turn: rowTurn });
+    }
 
     let guardFailure: string | null = null;
     let drift: string | null = null;

{langgraph_api-0.16.0.dev11 → langgraph_api-0.16.0.dev12}/benchmark/delta-report.js

@@ -29,7 +29,7 @@ export const HEADLINE = ['control', 'delta_50'];
 /** Metric key, label, printer. All three are lower-is-better. */
 const METRICS = [
   ['readP50Ms', 'read', (v) => (isNumber(v) ? (v / 1000).toFixed(2) : '-')],
-  ['saveEstMs', 'write', (v) => (isNumber(v) ? (v / 1000).toFixed(2) : '-')],
+  ['saveMs', 'write', (v) => (isNumber(v) ? (v / 1000).toFixed(2) : '-')],
   ['bytes', 'KB', (v) => (isNumber(v) ? String(Math.round(v / 1024)) : '-')],
 ];
 
@@ -93,33 +93,30 @@ export function sweepRows(depths, frequencies) {
   return sorted.filter((d) => wanted.has(bucketEnd(d)));
 }
 
-/**
- * A turn is read + step + save and cannot be split from outside, so the save
- * is what is left after the cold read. It still carries the run's fixed
- * overhead (queue, worker), which is the same in every cell, so it compares
- * across cells and is not an absolute.
- */
-export function saveEstimate(turnP50Ms, readP50Ms) {
-  if (!isNumber(turnP50Ms) || !isNumber(readP50Ms)) return null;
-  return Math.max(0, turnP50Ms - readP50Ms);
-}
-
 /** Bytes keep the largest reading in the bucket: the thread only grows. */
 export function aggregateSamples(points) {
   const turns = {};
   const reads = {};
+  const saves = {};
+  const rowSaves = {};
   const bytes = {};
   for (const point of points) {
     const depth = point.tags?.depth;
     if (depth == null || !isNumber(point.value)) continue;
     if (point.metric === 'turn_duration') (turns[depth] ??= []).push(point.value);
     else if (point.metric === 'state_read_duration') (reads[depth] ??= []).push(point.value);
+    else if (point.metric === 'checkpoint_save') {
+      (saves[depth] ??= []).push(point.value);
+      if (point.tags?.row_turn === '1') (rowSaves[depth] ??= []).push(point.value);
+    }
     else if (point.metric === 'thread_storage_bytes') bytes[depth] = Math.max(bytes[depth] ?? 0, point.value);
   }
   const buckets = {};
-  for (const depth of new Set([...Object.keys(turns), ...Object.keys(reads), ...Object.keys(bytes)])) {
+  for (const depth of new Set([...Object.keys(turns), ...Object.keys(reads), ...Object.keys(saves), ...Object.keys(bytes)])) {
     const sorted = (turns[depth] ?? []).sort((a, b) => a - b);
     const sortedReads = (reads[depth] ?? []).sort((a, b) => a - b);
+    const sortedSaves = (saves[depth] ?? []).sort((a, b) => a - b);
+    const sortedRowSaves = (rowSaves[depth] ?? []).sort((a, b) => a - b);
     const p50Ms = percentile(sorted, 0.5);
     // One cold read per thread per VERIFY_EVERY turns is few samples, so the median only.
     const readP50Ms = percentile(sortedReads, 0.5);
@@ -128,7 +125,8 @@ export function aggregateSamples(points) {
       p50Ms,
       p95Ms: percentile(sorted, 0.95),
       readP50Ms,
-      saveEstMs: saveEstimate(p50Ms, readP50Ms),
+      saveMs: percentile(sortedSaves, 0.5),
+      rowSaveMs: percentile(sortedRowSaves, 0.5),
       bytes: bytes[depth] ?? null,
     };
   }
@@ -146,7 +144,7 @@ async function readSamples(file) {
     } catch {
       continue;
     }
-    if (!['turn_duration', 'state_read_duration', 'thread_storage_bytes'].includes(entry.metric)) continue;
+    if (!['turn_duration', 'state_read_duration', 'checkpoint_save', 'thread_storage_bytes'].includes(entry.metric)) continue;
     points.push({ metric: entry.metric, value: entry.data?.value, tags: entry.data?.tags });
   }
   return points;
@@ -374,13 +372,18 @@ export function sweepTables(cells) {
   const depths = sweepRows(sharedDepths(deltas, new Set(deltas.flatMap((c) => Object.keys(c.buckets ?? {})))), deltas.map((c) => c.snapshot));
   if (depths.length === 0) return [];
   const w = 10;
-  const titles = { read: 'read, median seconds', write: 'write, seconds', KB: 'storage, KB at that turn' };
+  const titles = { read: 'read, median seconds', write: 'write, median / at the row\'s turn, seconds', KB: 'storage, KB at that turn' };
+  const cellText = (b, key, format) => {
+    if (!(b?.turns > 0)) return '-';
+    if (key === 'saveMs' && isNumber(b.rowSaveMs)) return `${format(b[key])}/${format(b.rowSaveMs)}`;
+    return format(b[key]);
+  };
   return METRICS.map(([key, label, format]) => {
     const lines = [`*${titles[label]}*`, '```'];
     lines.push(['turn'.padEnd(6), ...deltas.map((c) => `snap ${c.column}`.padStart(w))].join(''));
     for (const depth of depths) {
       lines.push(
-        [depthLabel(depth).padEnd(6), ...deltas.map((c) => (c.buckets?.[depth]?.turns > 0 ? format(c.buckets[depth][key]) : '-').padStart(w))].join('')
+        [depthLabel(depth).padEnd(6), ...deltas.map((c) => cellText(c.buckets?.[depth], key, format).padStart(w))].join('')
       );
     }
     lines.push('```');
@@ -392,6 +395,16 @@ function cellName(cell) {
   return cell.cell === 'control' ? 'non-delta' : `delta/${cell.column}`;
 }
 
+/**
+ * Cells whose successful runs returned no `save_ms`: the deployed graph predates
+ * the save timing. A cell with no successful run is already reported as failed.
+ */
+function cellsWithoutSaves(cells) {
+  return cells.filter(
+    (c) => c.summary?.metrics?.successfulRuns > 0 && !Object.values(c.buckets ?? {}).some((b) => isNumber(b.saveMs))
+  );
+}
+
 function anyBytes(cells) {
   return cells.some((c) => Object.values(c.buckets ?? {}).some((b) => isNumber(b.bytes)));
 }
@@ -464,6 +477,12 @@ export function buildReport(cells, context = {}) {
   );
   lines.push(...headlineTable(cells, changesByCell));
   lines.push(depthLine(cells, cells.find((c) => c.load?.plateauDuration)?.load.plateauDuration));
+  const unsaved = cellsWithoutSaves(cells);
+  if (unsaved.length > 0) {
+    lines.push(
+      `_write reads \`-\` for ${unsaved.map(cellName).join(', ')}: its runs returned no \`save_ms\`, so the deployed benchmark graph predates langgraph-server-benchmarks#22._`
+    );
+  }
   if (!anyBytes(cells)) {
     lines.push('_KB reads `-`: the deployment runs an API without `GET /threads/{id}/storage` yet._');
   }
@@ -523,7 +542,7 @@ function howToRead(cells) {
     withLoad ? `• *Load:* ${count(Number(withLoad.load.loadSize))} users, one thread each${held ? `, held ${held}` : ''}, one turn per iteration, no ramp.` : null,
     '• *Turn:* one `POST /runs/wait` on the thread, which reads the last checkpoint, runs one step and saves a new one. The row label is the thread depth the cold read was taken at.',
     '• *Read:* a cold `GET /threads/{id}/state` every 50 turns, timed to the server\'s first byte, so sending the state back is not counted. The server rebuilds the value from storage with nothing to write. With delta that is the replay of every diff since the last snapshot.',
-    '• *Write:* the turn\'s median latency minus the cold read, so what saving one more checkpoint costs. It still includes the fixed cost of queueing and running the one-node step, identical in every cell, so compare columns; it is not a checkpoint-write latency on its own.',
+    '• *Write:* the checkpoint save, timed by the worker as the gap between the graph\'s two steps, with saves made synchronous so nothing overlaps it. The sweep also shows the save at the row\'s own turn: snapshots fall on those turns, so at a snapshot row it is the full-value write the median hides.',
     '• *Storage:* `GET /threads/{id}/storage` every 50 turns, the bytes the thread occupies at that turn.',
     `• *Regressed / Improved:* read, write or storage at the deepest turn every night reached, outside its own usual range over the last ${MIN_HISTORY_NIGHTS}+ nights.`,
     `• *Errors:* failed requests by HTTP status. ${ERROR_ATTENTION} or more turn the dot yellow even when the cell passes.`,

{langgraph_api-0.16.0.dev11 → langgraph_api-0.16.0.dev12}/benchmark/delta-report.test.js

@@ -1,6 +1,6 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
-import { CELLS, aggregateSamples, buildReport, cellChanges, headlineRows, saveEstimate, sweepRows, sweepTables } from './delta-report.js';
+import { CELLS, aggregateSamples, buildReport, cellChanges, headlineRows, sweepRows, sweepTables } from './delta-report.js';
 
 const point = (metric, depth, value) => ({ metric, value, tags: { depth } });
 
@@ -15,13 +15,8 @@ test('aggregates turn latency, cold read, derived write and end-of-bucket bytes
     point('state_read_duration', '1-100', 70),
     point('turn_duration', '101-200', 900),
   ]);
-  assert.deepEqual(buckets['1-100'], { turns: 3, p50Ms: 200, p95Ms: 300, readP50Ms: 50, saveEstMs: 150, bytes: 4096 });
-  assert.deepEqual(buckets['101-200'], { turns: 1, p50Ms: 900, p95Ms: 900, readP50Ms: null, saveEstMs: null, bytes: null });
-});
-
-test('write estimate never goes negative when a cold read is slower than the turn', () => {
-  assert.equal(saveEstimate(100, 140), 0);
-  assert.equal(saveEstimate(null, 140), null);
+  assert.deepEqual(buckets['1-100'], { turns: 3, p50Ms: 200, p95Ms: 300, readP50Ms: 50, saveMs: null, rowSaveMs: null, bytes: 4096 });
+  assert.deepEqual(buckets['101-200'], { turns: 1, p50Ms: 900, p95Ms: 900, readP50Ms: null, saveMs: null, rowSaveMs: null, bytes: null });
 });
 
 const bucket = (readMs, writeMs, bytes) => ({
@@ -29,7 +24,7 @@ const bucket = (readMs, writeMs, bytes) => ({
   p50Ms: readMs + writeMs,
   p95Ms: (readMs + writeMs) * 1.5,
   readP50Ms: readMs,
-  saveEstMs: writeMs,
+  saveMs: writeMs,
   bytes,
 });
 
@@ -140,7 +135,7 @@ test('the sweep keeps the first row, the deepest, and the turn before and at eac
 test('a night that stops mid-bucket compares at the last turn with a cold read, not the partial one', () => {
   const past = Array.from({ length: 5 }, () => asHistory(night()));
   const tonight = night({ delta_50: { readMs: 90 } });
-  tonight[1].buckets['101-150'] = { turns: 7, p50Ms: 500, p95Ms: 600, readP50Ms: null, saveEstMs: null, bytes: null };
+  tonight[1].buckets['101-150'] = { turns: 7, p50Ms: 500, p95Ms: 600, readP50Ms: null, saveMs: null, bytes: null };
   const { text } = buildReport(tonight, { ...base, history: past });
   assert.match(text, /^\*Regressed:\* delta\/50 read@turn 100 0\.05 → 0\.11 s \(\+125%\)$/m);
   assert.match(text, /^100\s+.*0\.11▼/m);
@@ -151,11 +146,11 @@ test('tables stop where the slowest cell stopped, and the depth each cell reache
   tonight[0].buckets['101-150'] = bucket(50, 400, 122880);
   tonight[0].buckets['151-200'] = bucket(50, 400, 163840);
   // delta/50 got nine turns into the next bucket, with no cold read there yet.
-  tonight[1].buckets['101-150'] = { turns: 9, p50Ms: 450, p95Ms: 600, readP50Ms: null, saveEstMs: null, bytes: null };
+  tonight[1].buckets['101-150'] = { turns: 9, p50Ms: 450, p95Ms: 600, readP50Ms: null, saveMs: null, bytes: null };
   // delta/500's one thread there had its state read fail; only the storage probe answered.
-  tonight[2].buckets['101-150'] = { turns: 1, p50Ms: 450, p95Ms: 450, readP50Ms: null, saveEstMs: null, bytes: 1024 };
+  tonight[2].buckets['101-150'] = { turns: 1, p50Ms: 450, p95Ms: 450, readP50Ms: null, saveMs: null, bytes: 1024 };
   // delta/1000's first verify turn had its state read fail, with storage answering: a hole, not the end.
-  tonight[3].buckets['1-50'] = { turns: 500, p50Ms: 450, p95Ms: 600, readP50Ms: null, saveEstMs: null, bytes: 4096 };
+  tonight[3].buckets['1-50'] = { turns: 500, p50Ms: 450, p95Ms: 600, readP50Ms: null, saveMs: null, bytes: 4096 };
   const { text, thread } = buildReport(tonight, base);
   assert.match(text, /^50\s+/m);
   assert.match(text, /^100\s+/m);
@@ -167,8 +162,30 @@ test('tables stop where the slowest cell stopped, and the depth each cell reache
 
 test('a deepest bucket with turns but no cold read yet is not a row of dashes', () => {
   const tonight = night();
-  for (const c of tonight) c.buckets['101-150'] = { turns: 9, p50Ms: 450, p95Ms: 600, readP50Ms: null, saveEstMs: null, bytes: null };
+  for (const c of tonight) c.buckets['101-150'] = { turns: 9, p50Ms: 450, p95Ms: 600, readP50Ms: null, saveMs: null, bytes: null };
   const { text } = buildReport(tonight, base);
   assert.match(text, /^100\s+/m);
   assert.doesNotMatch(text, /^150\s+/m);
 });
+
+test('write is the measured save, and the sweep shows the save at the row turn where snapshots land', () => {
+  const save = (value, rowTurn) => ({ metric: 'checkpoint_save', value, tags: { depth: '1-50', row_turn: rowTurn } });
+  const buckets = aggregateSamples([save(20, '0'), save(30, '0'), save(25, '0'), save(900, '1'), save(1100, '1')]);
+  assert.equal(buckets['1-50'].saveMs, 30);
+  assert.equal(buckets['1-50'].rowSaveMs, 900);
+  const tonight = night();
+  for (const c of tonight) c.buckets['1-50'] = { ...c.buckets['1-50'], saveMs: 30, rowSaveMs: 900 };
+  const [, write] = sweepTables(tonight).map((t) => t.split('\n'));
+  assert.match(write[0], /^\*write, median \/ at the row's turn, seconds\*/);
+  assert.match(write[3], /^50\s+0\.03\/0\.90\s+0\.03\/0\.90\s+0\.03\/0\.90$/);
+});
+
+test('a cell whose runs returned no save_ms says why write is blank', () => {
+  const tonight = night();
+  for (const b of Object.values(tonight[1].buckets)) b.saveMs = null;
+  const { text } = buildReport(tonight, base);
+  assert.match(text, /^_write reads `-` for delta\/50: its runs returned no `save_ms`/m);
+  assert.doesNotMatch(buildReport(night(), base).text, /returned no `save_ms`/);
+  tonight[1].summary.metrics.successfulRuns = 0;
+  assert.doesNotMatch(buildReport(tonight, base).text, /returned no `save_ms`/);
+});

langgraph_api-0.16.0.dev12/langgraph_api/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.16.0.dev12"