langgraph-api 0.4.47__tar.gz → 0.5.0__tar.gz

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.4
 Name: langgraph-api
-Version: 0.4.47
-Author-email: Nuno Campos <nuno@langchain.dev>, Will Fu-Hinthorn <will@langchain.dev>
+Version: 0.5.0
+Author-email: Will Fu-Hinthorn <will@langchain.dev>, Josh Rogers <josh@langchain.dev>, Parker Rule <parker@langchain.dev>
 License: Elastic-2.0
 License-File: LICENSE
 Requires-Python: >=3.11
@@ -12,10 +12,10 @@ Requires-Dist: grpcio<2.0.0,>=1.75.0
 Requires-Dist: httpx>=0.25.0
 Requires-Dist: jsonschema-rs<0.30,>=0.20.0
 Requires-Dist: langchain-core>=0.3.64
-Requires-Dist: langgraph-checkpoint>=2.0.23
-Requires-Dist: langgraph-runtime-inmem<0.15.0,>=0.14.0
+Requires-Dist: langgraph-checkpoint<4,>=3
+Requires-Dist: langgraph-runtime-inmem<0.16.0,>=0.15.0
 Requires-Dist: langgraph-sdk>=0.2.0
-Requires-Dist: langgraph>=0.4.0
+Requires-Dist: langgraph<2,>=0.4.10
 Requires-Dist: langsmith>=0.3.45
 Requires-Dist: opentelemetry-api>=1.37.0
 Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.37.0

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/benchmark/Makefile

@@ -1,16 +1,15 @@
 # Benchmark commands
-BASE_URL ?= https://ifh-core-api-dr-benchmark-048d364b548f5d9790082d8ba4fb44d8.us.langgraph.app
-RAMP_START ?= 10
+BASE_URL ?= https://benchmark-dr-s-2799835ad04b501a95044223ae72ced7.staging.langgraph.app 
+RAMP_START ?= 40
 RAMP_END ?= 1000
 RAMP_MULTIPLIER ?= 2
 WAIT_SECONDS ?= 60
-SUCCESS_THRESHOLD ?= 0.99
 CLEAR_BETWEEN_STEPS ?= true
 CLEAR_DELAY_SECONDS ?= 5
 DATA_SIZE ?= 1000
 DELAY ?= 0
-EXPAND ?= 10
-STEPS ?= 10
+EXPAND ?= 1
+STEPS ?= 100
 
 benchmark-burst:
 	make benchmark-reset
@@ -21,14 +20,13 @@ benchmark-ramp:
 	k6 run --out json=raw_data_$(shell date +%Y-%m-%dT%H-%M-%S).json --system-tags=[] ramp.js
 
 benchmark-capacity:
-	rm -f capacity_summary_t*.json capacity_report_*.json capacity_raw_t*.json capacity_hist_*.png capacity_pie_*.png
+	rm -f capacity_summary_t*.json capacity_raw_t*.json capacity_histogram_*.json
 	npm install
 	BASE_URL=$(BASE_URL) \
 	RAMP_START=$(RAMP_START) \
 	RAMP_END=$(RAMP_END) \
 	RAMP_MULTIPLIER=$(RAMP_MULTIPLIER) \
 	WAIT_SECONDS=$(WAIT_SECONDS) \
-	SUCCESS_THRESHOLD=$(SUCCESS_THRESHOLD) \
 	CLEAR_BETWEEN_STEPS=$(CLEAR_BETWEEN_STEPS) \
 	CLEAR_DELAY_SECONDS=$(CLEAR_DELAY_SECONDS) \
 	DATA_SIZE=$(DATA_SIZE) \

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/benchmark/benchmark-runners/assistant.js

@@ -9,7 +9,7 @@ export class Assistant extends BenchmarkRunner {
      */
     static run(baseUrl, requestParams) {
         const graph_id = 'benchmark';
-        let metadata = { description: `Test benchmark assistant ${crypto.randomUUID()}` };
+        let metadata = { description: `Test benchmark assistant ${crypto.randomUUID()}`, created_by: 'benchmark' };
 
         // Create an assistant
         const createPayload = JSON.stringify({ graph_id, metadata });
@@ -36,7 +36,8 @@ export class Assistant extends BenchmarkRunner {
         const countResponse = http.post(`${baseUrl}/assistants/count`, countPayload, requestParams);
 
         // Delete the assistant
-        http.del(`${baseUrl}/assistants/${assistantId}`, requestParams);
+        const deleteResponse = http.del(`${baseUrl}/assistants/${assistantId}`, "{}", requestParams);
+
         return {
             assistantId,
             searchResponse,
@@ -44,6 +45,7 @@ export class Assistant extends BenchmarkRunner {
             patchResponse,
             getResponse2,
             countResponse,
+            deleteResponse,
         };
     }
 
@@ -57,22 +59,24 @@ export class Assistant extends BenchmarkRunner {
                 'Patch response contains the correct assistant': (r) => r.patchResponse.json().assistant_id === result.assistantId,
                 'Get response 2 contains the correct assistant': (r) => r.getResponse2.json().assistant_id === result.assistantId,
                 'Get response 2 contains the new description': (r) => r.getResponse2.json().metadata.description != result.getResponse.json().metadata.description && result.getResponse2.json().metadata.description === result.patchResponse.json().metadata.description,
+                'Get response 2 contains the correct created_by': (r) => r.getResponse2.json().metadata.created_by === 'josh',
                 'Count response contains the correct number of assistants': (r) => parseInt(r.countResponse.json()) === 1,
+                'Delete response is successful': (r) => r.deleteResponse.status === 204,
             });
         } catch (error) {
             console.log(`Unknown error checking response: ${error.message}`);
         }
 
         if (!success) {
-            if (result.searchResponse.status >= 500 || result.getResponse.status >= 500 || result.patchResponse.status >= 500 || result.getResponse2.status >= 500 || result.countResponse.status >= 500) {
+            if (result.searchResponse.status >= 500 || result.getResponse.status >= 500 || result.patchResponse.status >= 500 || result.getResponse2.status >= 500 || result.countResponse.status >= 500 || result.deleteResponse.status >= 500) {
                 errorMetrics.server_errors.add(1);
-                console.log(`Server error: ${result.searchResponse.status}, ${result.getResponse.status}, ${result.patchResponse.status}, ${result.getResponse2.status}, ${result.countResponse.status}`);
-            } else if (result.searchResponse.status === 408 || result.getResponse.status === 408 || result.patchResponse.status === 408 || result.getResponse2.status === 408 || result.countResponse.status === 408) {
+                console.log(`Server error: ${result.searchResponse.status}, ${result.getResponse.status}, ${result.patchResponse.status}, ${result.getResponse2.status}, ${result.countResponse.status}, ${result.deleteResponse.status}`);
+            } else if (result.searchResponse.status === 408 || result.getResponse.status === 408 || result.patchResponse.status === 408 || result.getResponse2.status === 408 || result.countResponse.status === 408 || result.deleteResponse.status === 408) {
                 errorMetrics.timeout_errors.add(1);
-                console.log(`Timeout error: ${result.searchResponse.error}, ${result.getResponse.error}, ${result.patchResponse.error}, ${result.getResponse2.error}, ${result.countResponse.error}`);
+                console.log(`Timeout error: ${result.searchResponse.error}, ${result.getResponse.error}, ${result.patchResponse.error}, ${result.getResponse2.error}, ${result.countResponse.error}, ${result.deleteResponse.error}`);
             } else {
                 errorMetrics.other_errors.add(1);
-                console.log(`Other error: ${result.searchResponse.body}, ${result.getResponse.body}, ${result.patchResponse.body}, ${result.getResponse2.body}, ${result.countResponse.body}`);
+                console.log(`Other error: ${result.searchResponse.body}, ${result.getResponse.body}, ${result.patchResponse.body}, ${result.getResponse2.body}, ${result.countResponse.body}, ${result.deleteResponse.body}`);
             }
         }
         return success;

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/benchmark/capacity_k6.js

@@ -19,7 +19,6 @@ const LANGSMITH_API_KEY = __ENV.LANGSMITH_API_KEY;
 
 const TARGET = parseInt(__ENV.TARGET || '10');
 const WAIT_SECONDS = parseInt(__ENV.WAIT_SECONDS || '60');
-const SUCCESS_THRESHOLD = parseFloat(__ENV.SUCCESS_THRESHOLD || '0.99');
 
 // Agent params
 const DATA_SIZE = parseInt(__ENV.DATA_SIZE || '1000');
@@ -37,9 +36,6 @@ export const options = {
       maxDuration: `${Math.max(WAIT_SECONDS + 120, 150)}s`,
     },
   },
-  thresholds: {
-    'capacity_success_rate': [`rate>=${SUCCESS_THRESHOLD}`],
-  },
 };
 
 function headers() {
@@ -171,7 +167,12 @@ export function handleSummary(data) {
   const successRate = total > 0 ? (succ / total) * 100 : 0;
 
   function withStats(metric) {
-    if (!data.metrics[metric]?.values) return {};
+    if (!data.metrics[metric]?.values) return {
+      avg: null,
+      p50: null,
+      p95: null,
+      max: null,
+    };
     const vals = data.metrics[metric].values;
     return {
       avg: vals.avg ? vals.avg / 1000 : null,

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/benchmark/capacity_runner.mjs

@@ -32,6 +32,11 @@ function envFloat(name, def) {
   return Number.isFinite(n) ? n : def;
 }
 
+function slugifyTimestamp(ts) {
+  const raw = ts ?? new Date().toISOString();
+  return raw.replace(/:/g, '-').replace(/\..+/, '');
+}
+
 const BASE_URL = process.env.BASE_URL;
 const LANGSMITH_API_KEY = process.env.LANGSMITH_API_KEY;
 
@@ -39,7 +44,6 @@ const RAMP_START = envInt('RAMP_START', 10);
 const RAMP_END = envInt('RAMP_END', 1000);
 const RAMP_MULTIPLIER = envFloat('RAMP_MULTIPLIER', 10);
 const WAIT_SECONDS = envInt('WAIT_SECONDS', 60);
-const SUCCESS_THRESHOLD = envFloat('SUCCESS_THRESHOLD', 0.99);
 const CLEAR_BETWEEN_STEPS = envBool('CLEAR_BETWEEN_STEPS', true);
 const CLEAR_DELAY_SECONDS = envInt('CLEAR_DELAY_SECONDS', 5);
 
@@ -107,7 +111,6 @@ function runK6(target) {
     LANGSMITH_API_KEY,
     TARGET: String(target),
     WAIT_SECONDS: String(WAIT_SECONDS),
-    SUCCESS_THRESHOLD: String(SUCCESS_THRESHOLD),
     DATA_SIZE: String(DATA_SIZE),
     DELAY: String(DELAY),
     EXPAND: String(EXPAND),
@@ -140,89 +143,95 @@ function loadSummaryForTarget(target) {
 
 async function main() {
   let n = RAMP_START;
-  let lastSuccess = null; // { target, avgDurationSeconds, successRate }
-  let failedStep = null; // { target, successRate }
 
   while (n <= RAMP_END) {
-    console.log(`\n=== Capacity step: N=${n} ===`);
+    const currentTarget = n;
+    console.log(`\n=== Capacity step: N=${currentTarget} ===`);
     if (CLEAR_BETWEEN_STEPS) {
       console.log('Clearing threads before step...');
       await cleanThreads();
     }
 
+    let stageTimestamp = null;
+    let runDurationHistogram = null;
+    let histogramSampleCount = 0;
+
     try {
-      const { rawOut, ts } = runK6(n);
+      const { rawOut, ts } = runK6(currentTarget);
+      stageTimestamp = ts;
       try {
-        await generateHistogramsForStage(rawOut, n, ts);
+        const histogramResult = await generateHistogramsForStage(rawOut);
+        runDurationHistogram = histogramResult?.runDurationHistogramSeconds ?? null;
+        histogramSampleCount = histogramResult?.sampleCount ?? 0;
       } catch (e) {
-        console.error(`Failed to generate histograms for N=${n}:`, e?.message || e);
+        console.error(`Failed to generate histograms for N=${currentTarget}:`, e?.message || e);
       }
     } catch (e) {
-      console.error(`k6 run failed at N=${n}:`, e?.message || e);
+      console.error(`k6 run failed at N=${currentTarget}:`, e?.message || e);
       // Treat as failure for this step
     }
 
-    let successRate = 0;
-    let avgDurationSeconds = null;
+    let summary = null;
     try {
-      const summary = loadSummaryForTarget(n);
-      const s = summary?.metrics?.successRate; // percent
-      successRate = Number.isFinite(s) ? s / 100 : 0;
-      avgDurationSeconds = summary?.metrics?.runDuration?.avg ?? null;
-      console.log(`Step N=${n} successRate=${(successRate * 100).toFixed(2)}% avgDur=${avgDurationSeconds ?? 'n/a'}s`);
+      summary = loadSummaryForTarget(currentTarget);
     } catch (e) {
-      console.error(`Failed to read summary for N=${n}:`, e?.message || e);
+      console.error(`Failed to read summary for N=${currentTarget}:`, e?.message || e);
     }
 
-    if (successRate >= SUCCESS_THRESHOLD) {
-      lastSuccess = { target: n, avgDurationSeconds, successRate };
-      // next n
-      const next = Math.floor(n * RAMP_MULTIPLIER);
-      if (next <= n) {
-        console.log('Next ramp value would not increase; stopping.');
-        break;
-      }
-      n = next;
+    const metrics = summary?.metrics ?? {};
+    const successRatePercent = metrics?.successRate ?? null;
+    const avgDurationSeconds = metrics?.runDuration?.avg ?? null;
+
+    const hasSuccessRate = Number.isFinite(successRatePercent);
+    const hasAvgDuration = Number.isFinite(avgDurationSeconds);
+    if (hasSuccessRate && hasAvgDuration) {
+      console.log(`Step N=${currentTarget} successRate=${successRatePercent.toFixed(2)}% avgDur=${avgDurationSeconds.toFixed(3)}s`);
     } else {
-      failedStep = { target: n, avgDurationSeconds, successRate };
+      console.log(`Step N=${currentTarget} summary incomplete`);
+    }
+
+    const effectiveTimestamp = summary?.timestamp ?? stageTimestamp ?? slugifyTimestamp();
+
+    if (Array.isArray(runDurationHistogram) && runDurationHistogram.length > 0) {
+      const histogramLog = {
+        timestamp: effectiveTimestamp,
+        baseUrl: BASE_URL,
+        baseUrlName: baseUrlToBaseUrlName[BASE_URL],
+        target: currentTarget,
+        waitSeconds: WAIT_SECONDS,
+        dataSize: DATA_SIZE,
+        delay: DELAY,
+        expand: EXPAND,
+        steps: STEPS,
+        metric: 'run_duration_seconds',
+        histogram: {
+          unit: 'seconds',
+          buckets: runDurationHistogram,
+          sampleCount: histogramSampleCount,
+        },
+      };
+      const histogramFile = `capacity_histogram_t${currentTarget}_${slugifyTimestamp(effectiveTimestamp)}.json`;
+      writeFileSync(join(process.cwd(), histogramFile), JSON.stringify(histogramLog, null, 2));
+      console.log(JSON.stringify(histogramLog));
+    }
+
+    if (currentTarget >= RAMP_END) {
+      console.log('Reached final ramp target; stopping.');
       break;
     }
-  }
 
-  const ts = new Date().toISOString().replace(/:/g, '-').replace(/\..+/, '');
-  const report = {
-    baseUrl: BASE_URL,
-    baseUrlName: baseUrlToBaseUrlName[BASE_URL],
-    timestamp: ts,
-    ramp: { start: RAMP_START, end: RAMP_END, multiplier: RAMP_MULTIPLIER },
-    waitSeconds: WAIT_SECONDS,
-    threshold: SUCCESS_THRESHOLD,
-    last_success_target: lastSuccess?.target ?? 0,
-    last_success_avg_duration_seconds: lastSuccess?.avgDurationSeconds ?? null,
-    last_success_rate: lastSuccess?.successRate ?? null,
-    failed_target: failedStep?.target ?? null,
-    failed_success_rate: failedStep?.successRate ?? null,
-    failed_avg_duration_seconds: failedStep?.avgDurationSeconds ?? null,
-  };
-  const fname = `capacity_report_${ts}.json`;
-  writeFileSync(join(process.cwd(), fname), JSON.stringify(report, null, 2));
+    const next = Math.floor(currentTarget * RAMP_MULTIPLIER);
+    if (next > RAMP_END) {
+      n = RAMP_END;
+    } else {
+      n = next;
+    }
 
-  // Export selected fields as GitHub Action outputs if available
-  if (process.env.GITHUB_OUTPUT) {
-    const out = [
-      `last_success_target=${report.last_success_target}`,
-      `last_success_avg_duration_seconds=${report.last_success_avg_duration_seconds}`,
-      `failed_target=${report.failed_target}`,
-      `failed_success_rate=${report.failed_success_rate}`,
-      `failed_avg_duration_seconds=${report.failed_avg_duration_seconds}`,
-    ].join('\n');
-    writeFileSync(process.env.GITHUB_OUTPUT, `${out}\n`, { flag: 'a' });
+    if (n <= currentTarget) {
+      console.log('Next ramp value would not increase; stopping.');
+      break;
+    }
   }
-
-  console.log('=== Capacity Benchmark Report ===');
-  console.log(`Last successful step: ${report.last_success_target}`);
-  console.log(`Average duration (s) at success: ${report.last_success_avg_duration_seconds}`);
-  console.log(`Failed step: ${report.failed_target} with success rate: ${report.failed_success_rate}`);
 }
 
 main().catch((e) => {
@@ -230,8 +239,47 @@ main().catch((e) => {
   process.exit(1);
 });
 
+function buildHistogramBuckets(valuesSeconds, bucketCount = 12) {
+  if (!Array.isArray(valuesSeconds) || valuesSeconds.length === 0) return [];
+  const min = Math.min(...valuesSeconds);
+  const max = Math.max(...valuesSeconds);
+  if (!Number.isFinite(min) || !Number.isFinite(max)) return [];
+  if (min === max) {
+    return [{
+      start: Number(min.toFixed(6)),
+      end: Number(max.toFixed(6)),
+      count: valuesSeconds.length,
+    }];
+  }
+  const width = (max - min) / bucketCount;
+  if (width === 0) {
+    return [{
+      start: Number(min.toFixed(6)),
+      end: Number(max.toFixed(6)),
+      count: valuesSeconds.length,
+    }];
+  }
+  const buckets = Array.from({ length: bucketCount }, (_, i) => ({
+    start: min + i * width,
+    end: i === bucketCount - 1 ? max : min + (i + 1) * width,
+    count: 0,
+  }));
+  for (const v of valuesSeconds) {
+    if (!Number.isFinite(v)) continue;
+    let idx = Math.floor((v - min) / width);
+    if (idx < 0) idx = 0;
+    if (idx >= bucketCount) idx = bucketCount - 1;
+    buckets[idx].count += 1;
+  }
+  return buckets.map((b, i) => ({
+    start: Number(b.start.toFixed(6)),
+    end: Number(b.end.toFixed(6)),
+    count: b.count,
+  }));
+}
+
 // Build and save histogram charts for one stage from raw K6 JSON
-async function generateHistogramsForStage(rawFile, target, ts) {
+async function generateHistogramsForStage(rawFile) {
   // Parse streaming JSONL from k6 --out json
   const metrics = {
     run_duration: [],
@@ -261,52 +309,9 @@ async function generateHistogramsForStage(rawFile, target, ts) {
     rl.on('error', reject);
   });
 
-  // Build pie chart for component breakdown based on average seconds
-  const avg = (arr) => (arr.length ? arr.reduce((a, b) => a + b, 0) / arr.length : 0);
-  const avgInsertionS = avg(metrics.run_insertion_duration) / 1000;
-  const avgPickupS = avg(metrics.run_pickup_duration) / 1000;
-  const avgOssS = avg(metrics.run_oss_duration) / 1000;
-  const avgReturnS = avg(metrics.run_return_duration) / 1000;
-  const parts = [avgInsertionS, avgPickupS, avgOssS, avgReturnS];
-  const totalParts = parts.reduce((a, b) => a + b, 0);
-  if (totalParts > 0) {
-    const chart = new QuickChart();
-    chart.setWidth(700);
-    chart.setHeight(500);
-    chart.setFormat('png');
-    chart.setConfig({
-      type: 'pie',
-      data: {
-        labels: ['Insertion', 'Pickup', 'OSS', 'Return'],
-        datasets: [{
-          label: `Breakdown of Run Duration (N=${target})`,
-          data: parts.map((v) => Number(v.toFixed(4))),
-          backgroundColor: [
-            'rgba(255, 99, 132, 0.6)',
-            'rgba(54, 162, 235, 0.6)',
-            'rgba(255, 206, 86, 0.6)',
-            'rgba(75, 192, 192, 0.6)',
-          ],
-          borderColor: [
-            'rgba(255, 99, 132, 1)',
-            'rgba(54, 162, 235, 1)',
-            'rgba(255, 206, 86, 1)',
-            'rgba(75, 192, 192, 1)',
-          ],
-          borderWidth: 1,
-        }],
-      },
-      options: {
-        plugins: {
-          title: { display: true, text: `Run Duration Breakdown (s) — N=${target}` },
-          legend: { position: 'right' },
-          tooltip: { callbacks: { label: (ctx) => `${ctx.label}: ${ctx.parsed.toFixed(3)}s` } },
-        },
-      },
-    });
-    const pieBuf = await chart.toBinary();
-    const piePath = join(process.cwd(), `capacity_pie_breakdown_t${target}_${ts}.png`);
-    writeFileSync(piePath, pieBuf);
-    console.log(`Saved pie chart: ${piePath}`);
-  }
+  const runDurationSeconds = metrics.run_duration.map((v) => v / 1000);
+  return {
+    runDurationHistogramSeconds: buildHistogramBuckets(runDurationSeconds),
+    sampleCount: runDurationSeconds.length,
+  };
 }

langgraph_api-0.5.0/benchmark/capacity_urls.mjs

@@ -0,0 +1,8 @@
+export const baseUrlToBaseUrlName = {
+  'https://benchmark-s-6d17010be8a25023a9eab0a688d29c05.staging.langgraph.app': 's',
+  'https://benchmark-m-fd2e770a72f55324b6c59f2664a56d32.staging.langgraph.app': 'm',
+  'https://benchmark-l-d655833b3cba5fc8a703c95f20045831.staging.langgraph.app': 'l',
+  'https://benchmark-dr-s-2799835ad04b501a95044223ae72ced7.staging.langgraph.app': 'dr-s',
+  'https://benchmark-dr-m-ec079ea9f20e5655ab35a4ebc1a0ade8.staging.langgraph.app': 'dr-m',
+  'https://benchmark-dr-l-e996bbdcfbf15c9e8f547ab74fae93d2.staging.langgraph.app': 'dr-l'
+};

langgraph_api-0.5.0/benchmark/clean.js

@@ -0,0 +1,150 @@
+/*
+ * Delete all threads and runs from the last benchmark run for consistent tests
+ * The default benchmark server has a thread TTL of one hour that should clean things up too so this doesn't run too long.
+ */
+
+// URL of your LangGraph server
+const BASE_URL = process.env.BASE_URL || 'http://localhost:9123';
+// LangSmith API key only needed with a custom server endpoint
+const LANGSMITH_API_KEY = process.env.LANGSMITH_API_KEY;
+
+async function clean() {
+    try {
+        await cleanAssistants();
+        await cleanThreads();
+    } catch (error) {
+        console.error('Fatal error during cleanup:', error.message);
+        process.exit(1);
+    }
+}
+
+async function cleanAssistants() {
+    const headers = { 'Content-Type': 'application/json' };
+    if (LANGSMITH_API_KEY) {
+        headers['x-api-key'] = LANGSMITH_API_KEY;
+    }
+
+    const searchUrl = `${BASE_URL}/assistants/search`;
+    let totalDeleted = 0;
+
+    console.log('Starting assistant cleanup...');
+    
+    while (true) {
+        // Get the next page of assistants
+        console.log('Searching for assistants...');
+        const searchResponse = await fetch(searchUrl, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                limit: 1000,
+                metadata: {
+                    created_by: 'benchmark' // NOTE: Super important to not clean up the assistants created by the system
+                }
+             })
+        });
+
+        if (!searchResponse.ok) {
+            throw new Error(`Search request failed: ${searchResponse.status} ${searchResponse.statusText}`);
+        }
+
+        const assistants = await searchResponse.json();
+        
+        // If no assistants found, we're done
+        if (!assistants || assistants.length === 0) {
+            console.log('No more assistants found.');
+            break;
+        }
+
+        console.log(`Found ${assistants.length} assistants to delete`);
+
+        // Delete each assistant
+        for (const assistant of assistants) {
+            try {
+                const deleteUrl = `${BASE_URL}/assistants/${assistant.assistant_id}`;
+                const deleteResponse = await fetch(deleteUrl, {
+                    method: 'DELETE',
+                    headers
+                });
+
+                if (!deleteResponse.ok) {
+                    console.error(`Failed to delete assistant ${assistant.assistant_id}: ${deleteResponse.status} ${deleteResponse.statusText}`);
+                } else {
+                    totalDeleted++;
+                }
+            } catch (deleteError) {
+                console.error(`Error deleting assistant ${assistant.assistant_id}:`, deleteError.message);
+            }
+        }
+
+        console.log(`Deleted ${assistants.length} assistants in this batch`);
+    }
+
+    console.log(`Assistant cleanup completed. Total assistants deleted: ${totalDeleted}`);
+}
+
+
+async function cleanThreads() {
+    const headers = { 'Content-Type': 'application/json' };
+    if (LANGSMITH_API_KEY) {
+        headers['x-api-key'] = LANGSMITH_API_KEY;
+    }
+
+    const searchUrl = `${BASE_URL}/threads/search`;
+    let totalDeleted = 0;
+
+    console.log('Starting thread cleanup...');
+    
+    while (true) {
+        // Get the next page of threads
+        console.log('Searching for threads...');
+        const searchResponse = await fetch(searchUrl, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                limit: 1000
+            })
+        });
+
+        if (!searchResponse.ok) {
+            throw new Error(`Search request failed: ${searchResponse.status} ${searchResponse.statusText}`);
+        }
+
+        const threads = await searchResponse.json();
+        
+        // If no threads found, we're done
+        if (!threads || threads.length === 0) {
+            console.log('No more threads found.');
+            break;
+        }
+
+        console.log(`Found ${threads.length} threads to delete`);
+
+        // Delete each thread
+        for (const thread of threads) {
+            try {
+                const deleteUrl = `${BASE_URL}/threads/${thread.thread_id}`;
+                const deleteResponse = await fetch(deleteUrl, {
+                    method: 'DELETE',
+                    headers
+                });
+
+                if (!deleteResponse.ok) {
+                    console.error(`Failed to delete thread ${thread.thread_id}: ${deleteResponse.status} ${deleteResponse.statusText}`);
+                } else {
+                    totalDeleted++;
+                }
+            } catch (deleteError) {
+                console.error(`Error deleting thread ${thread.thread_id}:`, deleteError.message);
+            }
+        }
+
+        console.log(`Deleted ${threads.length} threads in this batch`);
+    }
+
+    console.log(`Thread cleanup completed. Total threads deleted: ${totalDeleted}`);
+}
+
+clean().catch(error => {
+    console.error('Unhandled error:', error.message);
+    process.exit(1);
+});

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/benchmark/ramp.js

@@ -31,7 +31,8 @@ const LOAD_SIZE = parseInt(__ENV.LOAD_SIZE || '500');
 const LEVELS = parseInt(__ENV.LEVELS || '2');
 const PLATEAU_DURATION = parseInt(__ENV.PLATEAU_DURATION || '300');
 const BENCHMARK_TYPE = __ENV.BENCHMARK_TYPE || 'wait_write';
-const STATEFUL = __ENV.STATEFUL === 'true';
+const STATEFUL = __ENV.STATEFUL === 'true'; // Should the runner be stateful if possible?
+const P95_RUN_DURATION = __ENV.P95_RUN_DURATION; // Expected P95 run duration in milliseconds
 
 // Params for the agent
 const DATA_SIZE = parseInt(__ENV.DATA_SIZE || '1000');
@@ -53,6 +54,15 @@ const p95_run_duration = {
   'single': 1500,
 }
 
+function getP95RunDuration(mode) {
+  return P95_RUN_DURATION ? parseInt(P95_RUN_DURATION) : p95_run_duration[mode];
+}
+
+function getSuccessfulRunsThreshold(mode) {
+  // Number of expected successful runs per user worst case during plateau * max number of users * 2 cause that feels about right
+  return (PLATEAU_DURATION / (getP95RunDuration(mode) / 1000)) * LOAD_SIZE * LEVELS * 2;
+}
+
 // Test configuration
 export let options = {
   scenarios: {
@@ -64,8 +74,8 @@ export let options = {
     },
   },
   thresholds: {
-    'run_duration': [`p(95)<${p95_run_duration[MODE]}`],
-    'successful_runs': [`count>${(PLATEAU_DURATION / (p95_run_duration[MODE] / 1000)) * LOAD_SIZE * LEVELS * 2}`],  // Number of expected successful runs per user worst case during plateau * max number of users * 2 cause that feels about right
+    'run_duration': [`p(95)<${getP95RunDuration(MODE)}`],
+    'successful_runs': [`count>${getSuccessfulRunsThreshold(MODE)}`],
     'http_req_failed': ['rate<0.01'],   // Error rate should be less than 1%
   },
 };
@@ -129,6 +139,7 @@ export function setup() {
   console.log(`Running on pod: ${__ENV.POD_NAME || 'local'}`);
   console.log(`Running with the following ramp config: load size ${LOAD_SIZE}, levels ${LEVELS}, plateau duration ${PLATEAU_DURATION}, stateful ${STATEFUL}`);
   console.log(`Running with the following agent config: data size ${DATA_SIZE}, delay ${DELAY}, expand ${EXPAND}, mode ${MODE}`);
+  console.log(`Running with the following thresholds: p95 run duration ${getP95RunDuration(MODE)}ms, successful runs threshold ${getSuccessfulRunsThreshold(MODE)}, error rate < 1%`);
 
   return { startTime: new Date().toISOString().replace(/:/g, '-').replace(/\..+/, '') };
 }

{langgraph_api-0.4.47 → langgraph_api-0.5.0}/benchmark/reporting/dd_reporting.py

@@ -34,7 +34,7 @@ def send_benchmark_results(
         benchmark_data["labels"] = common_labels
 
     log_item = HTTPLogItem(
-        ddsource="benchmark-3",
+        ddsource="benchmark-6",
         ddtags="env:benchmarking" if common_labels else "env:benchmarking",
         hostname=os.getenv("HOSTNAME", "localhost"),
         message=json.dumps(benchmark_data),
@@ -77,11 +77,12 @@ if __name__ == "__main__":
 
     DD_SITE = os.getenv("DD_SITE", "us5.datadoghq.com")
 
-    common_labels = {"base_url": os.getenv("BASE_URL")}
-    if "report" in args.benchmark_file:
-        common_labels["type"] = "summary"
-    else:
-        common_labels["type"] = "individual_run"
-
     for file in glob.glob(args.benchmark_file):
-        process_benchmark_file(file, common_labels, DD_SITE, DD_API_KEY)
+        labels = {"base_url": os.getenv("BASE_URL")}
+        basename = os.path.basename(file)
+        lowered = basename.lower()
+        if "histogram" in lowered:
+            labels["type"] = "histogram"
+        else:
+            labels["type"] = "individual_run"
+        process_benchmark_file(file, labels, DD_SITE, DD_API_KEY)