langgraph-api 0.14.0.dev8__tar.gz → 0.14.0.dev9__tar.gz

{langgraph_api-0.14.0.dev8 → langgraph_api-0.14.0.dev9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.14.0.dev8
+Version: 0.14.0.dev9
 Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE

{langgraph_api-0.14.0.dev8 → langgraph_api-0.14.0.dev9}/benchmark/benchmark-runners/benchmark-runner.ts

@@ -3,6 +3,7 @@ import type { BenchmarkResult, BenchmarkGraphOptions } from './types.js';
 export interface ErrorMetrics {
   server_errors: { add(n: number): void };
   timeout_errors: { add(n: number): void };
+  connection_errors?: { add(n: number): void };
   api_errors?: { add(n: number): void };
   missing_message_errors?: { add(n: number): void };
   other_errors: { add(n: number): void };

langgraph_api-0.14.0.dev9/benchmark/benchmark-runners/classify-error.test.js

@@ -0,0 +1,116 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+// Compiled output: the source is TypeScript, and `npm test` runs after
+// `npm run build` both locally and in CI.
+import { classifyError, recordError } from './dist/classify-error.js';
+
+test('5xx is a server error', () => {
+  assert.equal(classifyError({ response: { status: 500 } }), 'server');
+  assert.equal(classifyError({ response: { status: 503 } }), 'server');
+});
+
+test('408 and timeout error strings are timeouts', () => {
+  assert.equal(classifyError({ response: { status: 408 } }), 'timeout');
+  assert.equal(
+    classifyError({ response: { status: 0, error: 'request timeout' } }),
+    'timeout'
+  );
+});
+
+// The bucket that had never once been written to. A refused connection has no
+// HTTP status, so every runner used to file it under something else.
+test('an unreachable deployment is a connection error', () => {
+  assert.equal(
+    classifyError({ response: { status: 0, error: 'dial: connection refused' } }),
+    'connection'
+  );
+  assert.equal(classifyError({ response: { status: 0 } }), 'connection');
+});
+
+test('connection beats the runner-specific fallbacks', () => {
+  // Before this classifier, stream_write reported exactly this case as
+  // missing_message and wait_write as other.
+  const unreachable = { status: 0, error: 'dial: connection refused' };
+  assert.equal(
+    classifyError({ response: unreachable, missingMessage: true }),
+    'connection'
+  );
+  assert.equal(
+    classifyError({ response: unreachable, hasApiError: true }),
+    'connection'
+  );
+});
+
+test('a 200 that came back wrong falls to the runner-specific buckets', () => {
+  const ok = { status: 200 };
+  assert.equal(classifyError({ response: ok, hasApiError: true }), 'api');
+  assert.equal(classifyError({ response: ok, missingMessage: true }), 'missing_message');
+  assert.equal(classifyError({ response: ok }), 'other');
+});
+
+test('4xx that is not 408 is not mistaken for anything else', () => {
+  assert.equal(classifyError({ response: { status: 422 } }), 'other');
+  assert.equal(classifyError({ response: { status: 404 } }), 'other');
+});
+
+test('a missing response is not reported as unreachable', () => {
+  // No response object at all means the runner never got that far; that is not
+  // evidence about the network, so it must not claim `connection`.
+  assert.equal(classifyError({}), 'other');
+  assert.equal(classifyError({ response: null, missingMessage: true }), 'missing_message');
+});
+
+// classifyError picks the bucket; recordError is what actually increments a
+// counter, and it was previously untested. A mutation that unwired
+// connection_errors passed the whole suite.
+function spyMetrics(omit = []) {
+  const names = [
+    'server_errors',
+    'timeout_errors',
+    'connection_errors',
+    'api_errors',
+    'missing_message_errors',
+    'other_errors',
+  ];
+  const counts = {};
+  const metrics = {};
+  for (const name of names) {
+    counts[name] = 0;
+    if (!omit.includes(name)) metrics[name] = { add: (n) => (counts[name] += n) };
+  }
+  return { metrics, counts };
+}
+
+test('recordError increments the counter the bucket names', async (t) => {
+  const cases = [
+    ['server', 'server_errors'],
+    ['timeout', 'timeout_errors'],
+    ['connection', 'connection_errors'],
+    ['api', 'api_errors'],
+    ['missing_message', 'missing_message_errors'],
+    ['other', 'other_errors'],
+  ];
+  for (const [bucket, counter] of cases) {
+    await t.test(`${bucket} -> ${counter}`, () => {
+      const { metrics, counts } = spyMetrics();
+      recordError(bucket, metrics);
+      assert.equal(counts[counter], 1, `${bucket} did not increment ${counter}`);
+      const strays = Object.entries(counts).filter(([k, v]) => k !== counter && v !== 0);
+      assert.deepEqual(strays, [], `${bucket} also incremented ${strays.map(([k]) => k)}`);
+    });
+  }
+});
+
+test('recordError falls back to other_errors for counters the entrypoint omits', () => {
+  // Preserves what the runners open-coded for api_errors and
+  // missing_message_errors, and covers older entrypoints without connection.
+  for (const [bucket, counter] of [
+    ['connection', 'connection_errors'],
+    ['api', 'api_errors'],
+    ['missing_message', 'missing_message_errors'],
+  ]) {
+    const { metrics, counts } = spyMetrics([counter]);
+    recordError(bucket, metrics);
+    assert.equal(counts.other_errors, 1, `${bucket} did not fall back to other_errors`);
+  }
+});

langgraph_api-0.14.0.dev9/benchmark/benchmark-runners/classify-error.ts

@@ -0,0 +1,86 @@
+import type { ErrorMetrics } from './benchmark-runner.js';
+
+/**
+ * Which error counter a failed run belongs in.
+ *
+ * `wait_write` and `stream_write` each grew their own copy of this ladder, and
+ * both had the same hole: neither could ever produce `connection`. That bucket
+ * is declared in ramp.js and staircase_step_k6.js, wired into the metrics map,
+ * and printed in every report - but no runner has ever written to it, so it has
+ * read 0 in all 32 scheduled runs on record. A refused connection has no HTTP
+ * status, so it fell through to `missing_message` (stream) or `other` (wait):
+ * "the server answered but left something out" when the truth was "the server
+ * never answered".
+ */
+export type ErrorBucket =
+  | 'connection'
+  | 'timeout'
+  | 'server'
+  | 'api'
+  | 'missing_message'
+  | 'other';
+
+/**
+ * Only the two fields the ladder reads. Structurally satisfied by both k6's
+ * RefinedResponse and our normalized HttpResponse, so runners can pass either
+ * without converting.
+ */
+export interface ClassifiableResponse {
+  status?: number;
+  error?: string;
+}
+
+export interface ClassifyInput {
+  /** The response the run hinges on, if there was one. */
+  response?: ClassifiableResponse | null;
+  /** Runner-specific: the response body carried an `__error__` key. */
+  hasApiError?: boolean;
+  /** Runner-specific: the expected message/counter never arrived. */
+  missingMessage?: boolean;
+}
+
+/** k6 reports a status of 0 when no HTTP response was received at all. */
+function isUnreachable(response?: ClassifiableResponse | null): boolean {
+  if (response == null) return false;
+  const status = response.status;
+  if (status != null && status > 0) return false;
+  return status === 0 || (response.error ?? '') !== '';
+}
+
+function isTimeout(response?: ClassifiableResponse | null): boolean {
+  if (response?.status === 408) return true;
+  return (response?.error ?? '').includes('timeout');
+}
+
+/**
+ * Order matters: a timeout is reported by k6 as an error string on a status-0
+ * response, so it has to be tested before the general unreachable case.
+ */
+export function classifyError(input: ClassifyInput): ErrorBucket {
+  const { response, hasApiError, missingMessage } = input;
+  const status = response?.status;
+
+  if (status != null && status >= 500) return 'server';
+  if (isTimeout(response)) return 'timeout';
+  if (isUnreachable(response)) return 'connection';
+  if (hasApiError) return 'api';
+  if (missingMessage) return 'missing_message';
+  return 'other';
+}
+
+/**
+ * Add one to the counter for `bucket`, falling back to `other_errors` when the
+ * entrypoint did not supply that counter. Preserves the behaviour the runners
+ * open-coded for `api_errors` and `missing_message_errors`.
+ */
+export function recordError(bucket: ErrorBucket, errorMetrics: ErrorMetrics): void {
+  const counters: Record<ErrorBucket, { add(n: number): void } | undefined> = {
+    server: errorMetrics.server_errors,
+    timeout: errorMetrics.timeout_errors,
+    connection: errorMetrics.connection_errors,
+    api: errorMetrics.api_errors,
+    missing_message: errorMetrics.missing_message_errors,
+    other: errorMetrics.other_errors,
+  };
+  (counters[bucket] ?? errorMetrics.other_errors).add(1);
+}

{langgraph_api-0.14.0.dev8 → langgraph_api-0.14.0.dev9}/benchmark/benchmark-runners/log-failure.ts

@@ -4,6 +4,37 @@ export interface LogFailureOptions {
   extra?: string;
 }
 
+/**
+ * k6's console.log is synchronous, so logging every failure throttles the load
+ * generator exactly when the server starts failing.
+ *
+ * Sampling is keyed on the VU rather than a counter because every k6 VU runs in
+ * its own JavaScript runtime, so a "first N" counter would emit N lines per VU.
+ * Only VU 1 logs, capped at MAX_LOGGED_SAMPLES; the error counters already
+ * carry the volume. BENCHMARK_VERBOSE_FAILURES=1 restores full logging.
+ */
+const VERBOSE = __ENV.BENCHMARK_VERBOSE_FAILURES === '1';
+const MAX_LOGGED_SAMPLES = Number(__ENV.BENCHMARK_FAILURE_SAMPLES || '5');
+let logged = 0;
+let suppressed = 0;
+
+function shouldLog(): boolean {
+  if (VERBOSE) return true;
+  // Only VU 1 narrates. Every other VU counts silently.
+  if (__VU !== 1) return false;
+  if (logged < MAX_LOGGED_SAMPLES) {
+    logged++;
+    return true;
+  }
+  suppressed++;
+  if (suppressed === 1) {
+    console.log(
+      `[benchmark] further failures suppressed after ${MAX_LOGGED_SAMPLES} samples; see error counters for totals`
+    );
+  }
+  return false;
+}
+
 /**
  * Log useful information when a benchmark run or validation fails.
  * Uses the consistent BenchmarkResult shape: ok, step, responses.
@@ -13,6 +44,8 @@ export function logFailure(
   result: Partial<BenchmarkResult<unknown>> | null | undefined,
   options: LogFailureOptions = {}
 ): void {
+  if (!shouldLog()) return;
+
   const parts: string[] = [`[${benchmarkType}] FAIL`];
   if (result?.ok === false) {
     parts.push('result.ok=false');

{langgraph_api-0.14.0.dev8 → langgraph_api-0.14.0.dev9}/benchmark/benchmark-runners/stream_write.ts

@@ -4,7 +4,8 @@ import { check } from 'k6';
 import http from 'k6/http';
 import type { BenchmarkResult, BenchmarkGraphOptions } from './types.js';
 import { parseSSE } from './types.js';
-import { addResponse, okResult } from './types.js';
+import { addResponse, okResult, failResult } from './types.js';
+import { classifyError, recordError } from './classify-error.js';
 import { logFailure } from './log-failure.js';
 import { getExpectedEvents } from './benchmark_profiles.js';
 
@@ -36,13 +37,21 @@ export class StreamWrite extends BenchmarkRunner {
     if (benchmarkGraphOptions.stateful) {
       const thread = http.post(`${baseUrl}/threads`, '{}', requestParams);
       addResponse(responses, 'create_thread', thread);
+      // Check before parsing. `thread.json()` on a failed create throws, and the
+      // throw surfaced as a generic other_error instead of the real cause.
+      if (thread.status !== 200) {
+        return failResult('create_thread', responses) as BenchmarkResult<StreamWriteData>;
+      }
       const threadId = (thread.json() as { thread_id: string }).thread_id;
       url = `${baseUrl}/threads/${threadId}/runs/stream`;
     }
 
     const response = http.post(url, payload, requestParams);
     addResponse(responses, 'stream', response);
-    const events = parseSSE(response.body as string);
+    // Guarded: on a connection failure k6 returns a response whose body is null,
+    // and parseSSE would throw out of run(), discarding the responses map that
+    // the error classifier needs to see the failure for what it was.
+    const events = typeof response.body === 'string' ? parseSSE(response.body) : [];
     return okResult(responses, { events, rawResponse: response });
   }
 
@@ -75,17 +84,13 @@ export class StreamWrite extends BenchmarkRunner {
       logFailure(StreamWrite.toString(), result, {
         extra: `events.length=${events.length} expected_events=${expected_events} expected_steps=${expected_steps} hasExpectedCounter=${hasExpectedCounter}`,
       });
-      const status = result.data?.rawResponse?.status;
-      if (status != null && status >= 500) {
-        errorMetrics.server_errors.add(1);
-      } else if (status === 408) {
-        errorMetrics.timeout_errors.add(1);
-      } else if (!hasExpectedCounter) {
-        if (errorMetrics.missing_message_errors) errorMetrics.missing_message_errors.add(1);
-        else errorMetrics.other_errors.add(1);
-      } else {
-        errorMetrics.other_errors.add(1);
-      }
+      recordError(
+        classifyError({
+          response: result.responses?.stream ?? result.responses?.create_thread ?? null,
+          missingMessage: !hasExpectedCounter,
+        }),
+        errorMetrics
+      );
     }
     return success;
   }

{langgraph_api-0.14.0.dev8 → langgraph_api-0.14.0.dev9}/benchmark/benchmark-runners/threads_search_metadata.ts

@@ -12,6 +12,18 @@ interface ThreadsSearchMetadataData {
   found: Array<{ thread_id: string; metadata?: { scenario?: string; tag?: string } }>;
 }
 
+/**
+ * Delete the threads each iteration creates, or leave them to accumulate.
+ *
+ * Off by default: letting the table grow is how this benchmark shows whether
+ * search degrades as thread count rises, and it is the realistic case since
+ * deployments do not delete threads per request.
+ *
+ * The capacity staircase turns it on, because there the table grows *while*
+ * load is ramping, so a slower step cannot be attributed to load or to size.
+ */
+const DELETE_THREADS = __ENV.DELETE_THREADS === '1';
+
 export class ThreadsSearchMetadata extends BenchmarkRunner {
   static run(
     baseUrl: string,
@@ -23,33 +35,44 @@ export class ThreadsSearchMetadata extends BenchmarkRunner {
     const responses: Record<string, import('./types.js').HttpResponse> = {};
 
     const threadIds: string[] = [];
-    for (let i = 0; i < 2; i++) {
-      const createRes = http.post(`${baseUrl}/threads`, '{}', requestParams);
-      addResponse(responses, `create_thread_${i}`, createRes);
-      if (createRes.status !== 200) {
-        return failResult(`create_thread_${i}`, responses) as BenchmarkResult<ThreadsSearchMetadataData>;
+    try {
+      for (let i = 0; i < 2; i++) {
+        const createRes = http.post(`${baseUrl}/threads`, '{}', requestParams);
+        addResponse(responses, `create_thread_${i}`, createRes);
+        if (createRes.status !== 200) {
+          return failResult(`create_thread_${i}`, responses) as BenchmarkResult<ThreadsSearchMetadataData>;
+        }
+        threadIds.push((createRes.json() as { thread_id: string }).thread_id);
       }
-      threadIds.push((createRes.json() as { thread_id: string }).thread_id);
-    }
 
-    for (let i = 0; i < 2; i++) {
-      const patchPayload = JSON.stringify({ metadata: { ...searchMetadata, index: i } });
-      const patchRes = http.patch(`${baseUrl}/threads/${threadIds[i]}`, patchPayload, requestParams);
-      addResponse(responses, `patch_thread_${i}`, patchRes);
-      if (patchRes.status !== 200) {
-        return failResult(`patch_thread_${i}`, responses) as BenchmarkResult<ThreadsSearchMetadataData>;
+      for (let i = 0; i < 2; i++) {
+        const patchPayload = JSON.stringify({ metadata: { ...searchMetadata, index: i } });
+        const patchRes = http.patch(`${baseUrl}/threads/${threadIds[i]}`, patchPayload, requestParams);
+        addResponse(responses, `patch_thread_${i}`, patchRes);
+        if (patchRes.status !== 200) {
+          return failResult(`patch_thread_${i}`, responses) as BenchmarkResult<ThreadsSearchMetadataData>;
+        }
       }
-    }
 
-    const searchPayload = JSON.stringify({ metadata: searchMetadata, limit: 10 });
-    const searchRes = http.post(`${baseUrl}/threads/search`, searchPayload, requestParams);
-    addResponse(responses, 'search', searchRes);
-    if (searchRes.status !== 200) {
-      return failResult(undefined, responses) as BenchmarkResult<ThreadsSearchMetadataData>;
-    }
-    const found = searchRes.json() as ThreadsSearchMetadataData['found'];
+      const searchPayload = JSON.stringify({ metadata: searchMetadata, limit: 10 });
+      const searchRes = http.post(`${baseUrl}/threads/search`, searchPayload, requestParams);
+      addResponse(responses, 'search', searchRes);
+      if (searchRes.status !== 200) {
+        return failResult(undefined, responses) as BenchmarkResult<ThreadsSearchMetadataData>;
+      }
+      const found = searchRes.json() as ThreadsSearchMetadataData['found'];
 
-    return okResult(responses, { threadIds, searchMetadata, found });
+      return okResult(responses, { threadIds, searchMetadata, found });
+    } finally {
+      // Best effort: a failed delete leaks one thread, which the inter-cell
+      // reset and the deployment's thread TTL both clean up behind us. Failing
+      // the iteration over cleanup would misreport the workload under test.
+      if (DELETE_THREADS) {
+        for (const threadId of threadIds) {
+          http.del(`${baseUrl}/threads/${threadId}`, null, requestParams);
+        }
+      }
+    }
   }
 
   static validate(

{langgraph_api-0.14.0.dev8 → langgraph_api-0.14.0.dev9}/benchmark/benchmark-runners/wait_write.ts

@@ -3,7 +3,8 @@ import type { ErrorMetrics } from './benchmark-runner.js';
 import { check } from 'k6';
 import http from 'k6/http';
 import type { BenchmarkResult, BenchmarkGraphOptions } from './types.js';
-import { addResponse, okResult } from './types.js';
+import { addResponse, okResult, failResult } from './types.js';
+import { classifyError, recordError } from './classify-error.js';
 import { logFailure } from './log-failure.js';
 
 interface WaitWriteData {
@@ -19,6 +20,7 @@ export class WaitWrite extends BenchmarkRunner {
     const context = benchmarkGraphOptions.context;
     const expand = context.expand;
     const steps = context.steps;
+    const responses: Record<string, import('./types.js').HttpResponse> = {};
     let url = `${baseUrl}/runs/wait`;
     const payload = JSON.stringify({
       assistant_id: benchmarkGraphOptions.graph_id,
@@ -29,12 +31,17 @@ export class WaitWrite extends BenchmarkRunner {
 
     if (benchmarkGraphOptions.stateful) {
       const thread = http.post(`${baseUrl}/threads`, '{}', requestParams);
+      addResponse(responses, 'create_thread', thread);
+      // Check before parsing. `thread.json()` on a failed create throws, and the
+      // throw surfaced as a generic other_error instead of the real cause.
+      if (thread.status !== 200) {
+        return failResult('create_thread', responses) as BenchmarkResult<WaitWriteData>;
+      }
       const threadId = (thread.json() as { thread_id: string }).thread_id;
       url = `${baseUrl}/threads/${threadId}/runs/wait`;
     }
 
     const response = http.post(url, payload, requestParams);
-    const responses: Record<string, import('./types.js').HttpResponse> = {};
     addResponse(responses, 'wait', response);
     return okResult(responses, { rawResponse: response });
   }
@@ -76,16 +83,16 @@ export class WaitWrite extends BenchmarkRunner {
 
     if (!success) {
       logFailure(WaitWrite.toString(), result, { extra: `status=${res?.status}` });
-      if (res?.status != null && res.status >= 500) {
-        errorMetrics.server_errors.add(1);
-      } else if (res?.status === 408 || (res as { error?: string }).error?.includes('timeout')) {
-        errorMetrics.timeout_errors.add(1);
-      } else if (json && typeof json === 'object' && (json as Record<string, unknown>).__error__) {
-        if (errorMetrics.api_errors) errorMetrics.api_errors.add(1);
-        else errorMetrics.other_errors.add(1);
-      } else {
-        errorMetrics.other_errors.add(1);
-      }
+      recordError(
+        classifyError({
+          response: res ?? result.responses?.create_thread ?? null,
+          hasApiError:
+            !!json &&
+            typeof json === 'object' &&
+            (json as Record<string, unknown>).__error__ !== undefined,
+        }),
+        errorMetrics
+      );
     }
     return success;
   }