streamtrust 0.1.1 → 0.3.0

package/examples/vercel-ai/reference-app/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "streamtrust-vercel-ai-reference-app",
+  "private": true,
+  "description": "Minimal Next.js + AI SDK chat used as the StreamTrust vercel-ai preset's application under test",
+  "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "build": "next build",
+    "start": "next start"
+  },
+  "dependencies": {
+    "@ai-sdk/openai-compatible": "3.0.68",
+    "@ai-sdk/react": "4.0.141",
+    "ai": "7.0.138",
+    "next": "16.4.3",
+    "react": "19.3.0",
+    "react-dom": "19.3.0"
+  }
+}

package/examples/vercel-ai/setup.mjs

@@ -0,0 +1,27 @@
+import { assertIsolatedCheckout } from './config.mjs';
+import { execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+const root = resolve(
+  process.env.STREAMTRUST_VERCEL_AI_ROOT ||
+    fileURLToPath(new URL('./reference-app', import.meta.url)),
+);
+if (Number(process.versions.node.split('.')[0]) < 22)
+  throw new Error('The vercel-ai preset requires Node.js 22+ (AI SDK 7)');
+if (!existsSync(resolve(root, 'package.json')))
+  throw new Error(`No package.json in ${root}`);
+assertIsolatedCheckout(root);
+const run = (cmd, args) =>
+  execFileSync(cmd, args, {
+    cwd: root,
+    stdio: 'inherit',
+    env: { ...process.env, NEXT_TELEMETRY_DISABLED: '1' },
+  });
+// The bundled shrinkwrap pins the fixture; external checkouts retain their own lockfile.
+run('npm', [
+  existsSync(resolve(root, 'npm-shrinkwrap.json')) || existsSync(resolve(root, 'package-lock.json')) ? 'ci' : 'install',
+  '--no-audit',
+  '--no-fund',
+]);
+run('npm', ['run', 'build']);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "streamtrust",
-  "version": "0.1.1",
+  "version": "0.3.0",
   "description": "Adapter-driven CI verification for streaming application lifecycle guarantees",
   "type": "module",
   "license": "MIT",
@@ -14,13 +14,28 @@
     ".": {
       "types": "./src/index.d.ts",
       "import": "./src/index.js"
+    },
+    "./checks": {
+      "types": "./src/checks.d.ts",
+      "import": "./src/checks.js"
     }
   },
   "types": "src/index.d.ts",
   "files": [
     "bin/",
     "src/",
-    "examples/",
+    "examples/http.adapter.mjs",
+    "examples/demo-app.mjs",
+    "examples/librechat/*.mjs",
+    "examples/librechat/README.md",
+    "examples/vercel-ai/*.mjs",
+    "examples/vercel-ai/README.md",
+    "examples/vercel-ai/reference-app/package.json",
+    "examples/vercel-ai/reference-app/npm-shrinkwrap.json",
+    "examples/vercel-ai/reference-app/next.config.mjs",
+    "examples/vercel-ai/reference-app/lib/*.js",
+    "examples/vercel-ai/reference-app/app/**/*.js",
+    "examples/vercel-ai/reference-app/app/**/*.jsx",
     "README.md",
     "LICENSE"
   ],

package/src/checks.d.ts

@@ -0,0 +1,36 @@
+import type { ExternalObservation } from './index.js';
+export type LifecycleScenario =
+  | 'persistence'
+  | 'cancellation'
+  | 'regeneration'
+  | 'error-retry';
+export const LIFECYCLE_SCENARIOS: readonly LifecycleScenario[];
+export interface LifecycleObservation {
+  scenario: LifecycleScenario;
+  expected: string;
+  original: string;
+  previousPrefix: string;
+  beforeReload: string;
+  reloaded: string;
+  messages: Array<{
+    id: string;
+    parentId: string;
+    text: string;
+    assistant: boolean;
+    unfinished: boolean;
+    error: boolean;
+  }>;
+  callsBeforeReload: number;
+  callsAfterReload: number;
+  cancellation?: {
+    acknowledged: boolean;
+    providerClosed: boolean;
+    providerFinished: boolean;
+    lateCompletionRejected: boolean;
+  };
+  failure?: { saved: boolean; visibleAfterReload: boolean };
+}
+/** Evaluate in-memory observations; returns only fixed check names and allowlisted numeric evidence. */
+export function evaluateLifecycle(
+  observation: LifecycleObservation,
+): ExternalObservation;

package/src/checks.js

@@ -0,0 +1,105 @@
+// Raw observations remain in memory. Returned evidence is deliberately allowlisted.
+export const LIFECYCLE_SCENARIOS = Object.freeze([
+  'persistence',
+  'cancellation',
+  'regeneration',
+  'error-retry',
+]);
+export function evaluateLifecycle(o) {
+  const text = (v) => typeof v === 'string' && v.length > 0;
+  const bool = (v) => typeof v === 'boolean';
+  if (
+    !o ||
+    !LIFECYCLE_SCENARIOS.includes(o.scenario) ||
+    !text(o.expected) ||
+    !text(o.original) ||
+    !text(o.previousPrefix) ||
+    typeof o.beforeReload !== 'string' ||
+    typeof o.reloaded !== 'string' ||
+    !Array.isArray(o.messages) ||
+    o.messages.some(
+      (m) =>
+        !m ||
+        !text(m.id) ||
+        typeof m.parentId !== 'string' ||
+        typeof m.text !== 'string' ||
+        !bool(m.assistant) ||
+        !bool(m.unfinished) ||
+        !bool(m.error),
+    ) ||
+    !Number.isSafeInteger(o.callsBeforeReload) ||
+    o.callsBeforeReload < 1 ||
+    !Number.isSafeInteger(o.callsAfterReload) ||
+    o.callsAfterReload < 0
+  )
+    throw new Error('Invalid lifecycle observation');
+  const checks = [];
+  const check = (name, passed) => checks.push({ name, passed });
+  const matching = o.messages.filter(
+    (m) => m.assistant && m.text.trim() === o.expected.trim(),
+  );
+  check('saved.answer-present', matching.length > 0);
+  check('saved.single-matching-answer', matching.length === 1);
+  check('ui.answer-visible', o.beforeReload.includes(o.expected));
+  check(
+    'saved.lifecycle-state',
+    matching.length === 1 &&
+      (o.scenario === 'cancellation'
+        ? matching[0].unfinished === true
+        : matching[0].unfinished === false && !matching[0].error),
+  );
+  check('reload.answer-preserved', o.reloaded.includes(o.expected));
+  check('reload.no-new-generation', o.callsAfterReload === o.callsBeforeReload);
+  if (o.scenario === 'cancellation') {
+    const c = o.cancellation;
+    if (
+      !c ||
+      [
+        'acknowledged',
+        'providerClosed',
+        'providerFinished',
+        'lateCompletionRejected',
+      ].some((k) => !bool(c[k]))
+    )
+      throw new Error('Invalid cancellation observation');
+    check('cancel.server-acknowledged', c.acknowledged);
+    check('cancel.provider-aborted', c.providerClosed && !c.providerFinished);
+    check('cancel.late-completion-rejected', c.lateCompletionRejected);
+  }
+  if (o.scenario === 'regeneration') {
+    const original = o.messages.filter(
+      (m) => m.assistant && m.text === o.original,
+    );
+    check('retry.original-branch-preserved', original.length === 1);
+    check(
+      'retry.distinct-sibling',
+      original.length === 1 &&
+        matching.length === 1 &&
+        original[0].id !== matching[0].id &&
+        original[0].parentId === matching[0].parentId,
+    );
+  }
+  if (o.scenario === 'error-retry') {
+    const f = o.failure;
+    if (!f || !bool(f.saved) || !bool(f.visibleAfterReload))
+      throw new Error('Invalid failure observation');
+    check('error.saved-before-retry', f.saved);
+    check('error.visible-after-reload', f.visibleAfterReload);
+  }
+  if (['regeneration', 'error-retry'].includes(o.scenario))
+    check(
+      'retry.previous-answer-not-active',
+      !o.reloaded.includes(o.previousPrefix),
+    );
+  return {
+    checks,
+    evidence: {
+      policy: 'summary-only-v1',
+      checkVersion: 1,
+      savedMessageCount: o.messages.length,
+      matchingAnswerCount: matching.length,
+      providerCallsBeforeReload: o.callsBeforeReload,
+      providerCallsAfterReload: o.callsAfterReload,
+    },
+  };
+}

package/src/cli.d.ts

@@ -1 +1,3 @@
 export function main(args: string[]): Promise<number>;
+/** Preset names accepted by `streamtrust init --preset`; `http` is the default. */
+export function presets(): string[];

package/src/cli.js

@@ -1,11 +1,29 @@
 import { fork } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { resolve } from 'node:path';
+import { resolve, join, dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { mkdir, copyFile, open, unlink, readFile } from 'node:fs/promises';
 import { constants } from 'node:fs';
 import { VERSION, failureReport } from './index.js';
-const HELP = `StreamTrust ${VERSION}\n\nUsage:\n  streamtrust init [directory]\n  streamtrust verify --adapter ./streamtrust/adapter.mjs [options]\n\nOptions:\n  --output <directory>    Reports (default: reports/streamtrust)\n  --repetitions <1-1000>   Repetitions per scenario (default: 3)\n  --timeout-ms <integer>  Total run deadline including setup/cleanup (default: 300000)\n  --help                 Show help\n  --version              Show version\n\nExit codes: 0 all pass; 1 failed/unsupported checks; 2 setup, timeout or harness error.\nAdapters execute trusted local code. JavaScript ESM (.mjs) works without dependencies.\n`;
+const EXAMPLES = fileURLToPath(new URL('../examples/', import.meta.url));
+// Explicit source inventory: never copy local builds, dependencies or configuration.
+const PRESETS = {
+  http: [['http.adapter.mjs', 'adapter.mjs'], ['demo-app.mjs', 'demo-app.mjs']],
+  librechat: ['adapter.mjs', 'provider.mjs', 'setup.mjs', 'README.md'].map(
+    (name) => [`librechat/${name}`, name]),
+  'vercel-ai': [
+    'adapter.mjs', 'provider.mjs', 'setup.mjs', 'config.mjs', 'README.md',
+    'reference-app/package.json', 'reference-app/npm-shrinkwrap.json',
+    'reference-app/next.config.mjs', 'reference-app/lib/store.js',
+    'reference-app/app/layout.jsx', 'reference-app/app/page.jsx',
+    'reference-app/app/api/chat/route.js', 'reference-app/app/chat/[id]/chat.jsx',
+    'reference-app/app/chat/[id]/page.jsx',
+  ].map((name) => [`vercel-ai/${name}`, name]),
+};
+export function presets() { return Object.keys(PRESETS); }
+function presetFiles(preset) { return PRESETS[preset]; }
+const HELP = () =>
+  `StreamTrust ${VERSION}\n\nUsage:\n  streamtrust init [directory] [--preset <name>]\n  streamtrust verify --adapter ./streamtrust/adapter.mjs [options]\n\nOptions:\n  --output <directory>    Reports (default: reports/streamtrust)\n  --repetitions <1-1000>   Repetitions per scenario (default: 3)\n  --timeout-ms <integer>  Total run deadline including setup/cleanup (default: 300000)\n  --help                 Show help\n  --version              Show version\n\nPresets: ${presets().join(', ')} (default: http)\n\nExit codes: 0 all pass; 1 failed/unsupported checks; 2 setup, timeout or harness error.\nAdapters execute trusted local code. JavaScript ESM (.mjs) works without dependencies.\n`;
 export async function main(args) {
   let outputDir;
   try {
@@ -14,24 +32,33 @@ export async function main(args) {
       return 0;
     }
     if (!args.length || args.includes('--help')) {
-      console.log(HELP);
+      console.log(HELP());
       return 0;
     }
     if (args[0] === 'init') {
-      if (args.length > 2 || args[1]?.startsWith('-'))
-        throw new Error('Usage: streamtrust init [directory]');
-      const dir = resolve(args[1] ?? 'streamtrust');
+      const initArgs = args.slice(1);
+      const presetIndex = initArgs.indexOf('--preset');
+      let preset = 'http';
+      if (presetIndex !== -1) {
+        preset = initArgs[presetIndex + 1];
+        initArgs.splice(presetIndex, 2);
+        if (!presets().includes(preset))
+          throw new Error(
+            `Unknown init preset; available: ${presets().join(', ')}`,
+          );
+      }
+      if (initArgs.length > 1 || initArgs[0]?.startsWith('-'))
+        throw new Error('Invalid init arguments');
+      const dir = resolve(initArgs[0] ?? 'streamtrust');
       await mkdir(dir, { recursive: true });
       // Exclusive copies preserve existing adapters. No generated config overrides user files.
       const created = [];
       try {
-        for (const [source, target] of [
-          ['http.adapter.mjs', 'adapter.mjs'],
-          ['demo-app.mjs', 'demo-app.mjs'],
-        ]) {
+        for (const [source, target] of presetFiles(preset)) {
           const destination = resolve(dir, target);
+          await mkdir(dirname(destination), { recursive: true });
           await copyFile(
-            new URL(`../examples/${source}`, import.meta.url),
+            join(EXAMPLES, source),
             destination,
             constants.COPYFILE_EXCL,
           );

package/src/index.js

@@ -63,13 +63,15 @@ export async function writeReports(outputDir, report) {
   const rows = report.results;
   const screenshots = new Set();
   await Promise.all(
-    rows.map(async (row) => {
-      const path = `${row.scenario}-${row.repetition}.png`;
-      try {
-        await access(resolve(outputDir, path));
-        screenshots.add(path);
-      } catch {}
-    }),
+    (report.metadata?.evidencePolicy === 'summary-only-v1' ? [] : rows).map(
+      async (row) => {
+        const path = `${row.scenario}-${row.repetition}.png`;
+        try {
+          await access(resolve(outputDir, path));
+          screenshots.add(path);
+        } catch {}
+      },
+    ),
   );
   const suiteCases = report.suiteErrors
     .map(