testdriverai 7.8.0-test.60 → 7.8.0-test.62

package/agent/lib/logger.js

@@ -7,7 +7,7 @@
  */
 
 const useStderr = process.env.TD_STDIO === 'stderr';
-const isDebug = process.env.DEBUG === 'true';
+const isDebug = process.env.TD_DEBUG === 'true' || process.env.VERBOSE === 'true';
 
 /**
  * Log a message - uses stdout by default, stderr if TD_STDIO=stderr

package/agent/lib/sandbox.js

@@ -49,6 +49,10 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       );
     }
 
+    getPublishCount() {
+      return this._publishCount;
+    }
+
     async _initAbly(ablyToken, channelName) {
       if (this._ably) {
         try {
@@ -177,7 +181,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
               return rid + '(' + (e && e.message ? e.message.type : '?') + ')';
             }).join(', ')
             : 'none';
-          logger.warn(
+          logger.debug(
             '[realtime] No pending promise for requestId=' + (message.requestId || 'null') +
             ' | response type=' + (message.type || 'unknown') +
             ' | error=' + (message.error ? (message.errorMessage || 'true') : 'false') +
@@ -192,7 +196,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           var pendingAge = pendingEntry && pendingEntry.startTime
             ? ((Date.now() - pendingEntry.startTime) / 1000).toFixed(1) + 's'
             : '?';
-          logger.warn(
+          logger.debug(
             '[realtime] Promise REJECTED: requestId=' + message.requestId +
             ' | type=' + (pendingMessage ? pendingMessage.type : 'unknown') +
             ' | age=' + pendingAge +
@@ -305,11 +309,11 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       });
 
       this._ably.connection.on("suspended", function () {
-        logger.warn("[realtime] Connection: suspended - connection lost for extended period, will keep retrying");
+        logger.debug("[realtime] Connection: suspended - connection lost for extended period, will keep retrying");
       });
 
       this._ably.connection.on("failed", function () {
-        logger.error("[realtime] Connection: failed");
+        logger.debug("[realtime] Connection: failed");
         self.apiSocketConnected = false;
         self.instanceSocketConnected = false;
         emitter.emit(events.error.sandbox, "Realtime connection failed");
@@ -326,7 +330,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
         var reasonMsg = reason ? (reason.message || reason.code || String(reason)) : '';
 
         if (current === 'attached' && stateChange.resumed === false && previous === 'attached') {
-          logger.warn('[realtime] Channel DISCONTINUITY detected (resumed=false)' + (reasonMsg ? ' — ' + reasonMsg : ''));
+          logger.debug('[realtime] Channel DISCONTINUITY detected (resumed=false)' + (reasonMsg ? ' — ' + reasonMsg : ''));
           emitter.emit(events.sandbox.progress, {
             step: 'discontinuity',
             message: 'Recovering missed messages after connection interruption...',
@@ -367,7 +371,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
                   entry.handler(page.items[j]);
                 }
               } catch (replayErr) {
-                logger.error('[realtime] Error replaying recovered message: ' + (replayErr.message || replayErr));
+                logger.debug('[realtime] Error replaying recovered message: ' + (replayErr.message || replayErr));
               }
             }
             page = page.hasNext() ? await page.next() : null;
@@ -375,11 +379,11 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
           totalRecovered += recovered;
           logger.debug('[realtime] Discontinuity recovery: replayed ' + recovered + ' ' + entry.name + ' message(s) from gap');
         } catch (err) {
-          logger.error('[realtime] Discontinuity recovery failed for ' + entry.name + ': ' + (err.message || err));
+          logger.debug('[realtime] Discontinuity recovery failed for ' + entry.name + ': ' + (err.message || err));
         }
       }
       if (totalRecovered > 0) {
-        logger.warn('[realtime] Recovered and replayed ' + totalRecovered + ' message(s) that were missed during connection interruption');
+        logger.debug('[realtime] Recovered and replayed ' + totalRecovered + ' message(s) that were missed during connection interruption');
       } else {
         logger.debug('[realtime] Discontinuity recovery: no missed messages found');
       }
@@ -579,6 +583,10 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
               if (data && data.os && reply.runner) reply.runner.os = data.os;
               if (data && data.ip && reply.runner) reply.runner.ip = data.ip;
               if (data && data.runnerVersion && reply.runner) reply.runner.version = data.runnerVersion;
+              // Persist version metadata for test result reporting
+              self._runnerVersionBefore = reply.imageVersion || null;
+              self._runnerVersionAfter = (data && data.runnerVersion) || reply.imageVersion || null;
+              self._wasUpdated = !!(data && data.runnerVersion && reply.imageVersion && data.runnerVersion !== reply.imageVersion);
               logger.log('Runner agent ready (os=' + ((data && data.os) || 'unknown') + ', runner v' + ((data && data.runnerVersion) || 'unknown') + ')');
               // Show upgrade info: if the runner's npm version differs from the baked image version,
               // the runner was upgraded during provisioning.
@@ -679,6 +687,15 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
             url: url,
             vncPort: noVncPort || undefined,
             runner: reply.runner,
+            // Extra metadata for test result reporting
+            amiId: reply.amiId || null,
+            e2bTemplateId: reply.e2bTemplateId || null,
+            imageVersion: reply.imageVersion || null,
+            runnerVersionBefore: this._runnerVersionBefore || reply.imageVersion || null,
+            runnerVersionAfter: this._runnerVersionAfter || reply.runner?.version || null,
+            wasUpdated: this._wasUpdated || false,
+            vncUrl: url || null,
+            channelName: this._channelName || null,
           },
         };
       }
@@ -1149,7 +1166,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       // Send end-session control message to runner before disconnecting
       if (this._sessionChannel && this._ably?.connection?.state === 'connected') {
         try {
-          logger.log('[realtime] Publishing control: type=end-session');
+          logger.debug('[realtime] Publishing control: type=end-session');
           await this._sessionChannel.publish('control', { type: 'end-session' });
         } catch (e) {
           // Ignore - best effort
@@ -1159,7 +1176,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       // Leave presence on session channel
       if (this._sessionChannel) {
         try {
-          logger.log('[realtime] Leaving presence on session channel');
+          logger.debug('[realtime] Leaving presence on session channel');
           await this._sessionChannel.presence.leave();
         } catch (e) {
           // ignore - best effort, Ably will auto-leave on disconnect
@@ -1167,7 +1184,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
       }
 
       try {
-        logger.log('[realtime] Detaching session channel');
+        logger.debug('[realtime] Detaching session channel');
         if (this._sessionChannel) {
           await this._sessionChannel.detach();
         }
@@ -1177,7 +1194,7 @@ const createSandbox = function (emitter, analytics, sessionInstance) {
 
       if (this._ably) {
         try {
-          logger.log('[realtime] Closing Realtime connection');
+          logger.debug('[realtime] Closing Realtime connection');
           this._ably.close();
         } catch (e) {
           /* ignore */

package/docs/_scripts/extract-example-urls.js

@@ -1,39 +1,33 @@
 #!/usr/bin/env node
 
 /**
- * Extract Example URLs from CI Logs
- * 
- * Parses vitest output to extract TESTDRIVER_EXAMPLE_URL lines
- * and updates the examples-manifest.json file.
- * 
+ * Extract Example URLs from Test Result JSON Files
+ *
+ * Reads per-test-case JSON result files written by the vitest plugin
+ * to .testdriver/results/ and updates examples-manifest.json.
+ *
  * Usage:
- *   cat ci-log.txt | node extract-example-urls.js
- *   node extract-example-urls.js < ci-log.txt
- *   node extract-example-urls.js --file=ci-log.txt
+ *   node extract-example-urls.js --results-dir=.testdriver/results
  */
 
 const fs = require("fs");
 const path = require("path");
-const readline = require("readline");
 
 const MANIFEST_PATH = path.join(__dirname, "../_data/examples-manifest.json");
 
-// Regex to match TESTDRIVER_EXAMPLE_URL::filename::url (handles optional timestamp prefix from CI logs)
-const URL_PATTERN = /TESTDRIVER_EXAMPLE_URL::([^:]+)::(.+)$/;
-
 // Parse command line arguments
 function parseArgs() {
   const args = process.argv.slice(2);
   const options = {
-    file: null,
+    resultsDir: null,
     help: false,
   };
 
   for (const arg of args) {
     if (arg === "--help" || arg === "-h") {
       options.help = true;
-    } else if (arg.startsWith("--file=")) {
-      options.file = arg.slice(7);
+    } else if (arg.startsWith("--results-dir=")) {
+      options.resultsDir = arg.slice(14);
     }
   }
 
@@ -62,40 +56,51 @@ function saveManifest(manifest) {
   fs.writeFileSync(MANIFEST_PATH, JSON.stringify(manifest, null, 2) + "\n", "utf-8");
 }
 
-// Process a single line and extract URL if present
-function processLine(line, manifest, stats) {
-  const match = line.match(URL_PATTERN);
-  if (match) {
-    const [, filename, url] = match;
-    const isNew = !manifest.examples[filename];
-    
-    manifest.examples[filename] = {
-      url: url.trim(),
-      lastUpdated: new Date().toISOString(),
-    };
-    
-    if (isNew) {
-      stats.added++;
-    } else {
-      stats.updated++;
-    }
-    
-    console.log(`${isNew ? "➕" : "🔄"} ${filename}: ${url}`);
-  }
-}
-
-// Process input stream
-async function processInput(inputStream) {
+// Process JSON result files from .testdriver/results/
+function processResultsDir(resultsDir) {
   const manifest = loadManifest();
   const stats = { added: 0, updated: 0 };
 
-  const rl = readline.createInterface({
-    input: inputStream,
-    crlfDelay: Infinity,
-  });
+  // Look for JSON files under examples/ subdirectories
+  const examplesDir = path.join(resultsDir, "examples");
+  if (!fs.existsSync(examplesDir)) {
+    console.log(`\n⚠️  No examples results found in ${examplesDir}`);
+    return stats;
+  }
 
-  for await (const line of rl) {
-    processLine(line, manifest, stats);
+  // Walk example test directories (e.g., examples/assert.test.mjs/)
+  const testDirs = fs.readdirSync(examplesDir, { withFileTypes: true });
+  for (const entry of testDirs) {
+    if (!entry.isDirectory()) continue;
+    const testDir = path.join(examplesDir, entry.name);
+    const jsonFiles = fs.readdirSync(testDir).filter(f => f.endsWith(".json"));
+
+    for (const jsonFile of jsonFiles) {
+      try {
+        const content = fs.readFileSync(path.join(testDir, jsonFile), "utf-8");
+        const result = JSON.parse(content);
+        const testFileName = path.basename(result.test?.file || result.testFile || entry.name);
+        const url = result.urls?.testRun || result.testRunLink;
+
+        if (!url) continue;
+
+        const isNew = !manifest.examples[testFileName];
+        manifest.examples[testFileName] = {
+          url: url,
+          lastUpdated: result.date || new Date().toISOString(),
+        };
+
+        if (isNew) {
+          stats.added++;
+        } else {
+          stats.updated++;
+        }
+
+        console.log(`${isNew ? "➕" : "🔄"} ${testFileName}: ${url}`);
+      } catch (err) {
+        console.warn(`⚠️  Failed to read ${jsonFile}: ${err.message}`);
+      }
+    }
   }
 
   if (stats.added > 0 || stats.updated > 0) {
@@ -103,7 +108,7 @@ async function processInput(inputStream) {
     console.log(`\n✨ Manifest updated: ${stats.added} added, ${stats.updated} updated`);
     console.log(`📂 Written to: ${MANIFEST_PATH}`);
   } else {
-    console.log("\n⚠️  No TESTDRIVER_EXAMPLE_URL entries found in input");
+    console.log("\n⚠️  No example URLs found in result files");
   }
 
   return stats;
@@ -112,28 +117,24 @@ async function processInput(inputStream) {
 // Show help
 function showHelp() {
   console.log(`
-Extract Example URLs from CI Logs
+Extract Example URLs from Test Result JSON Files
 
 Usage:
-  cat ci-log.txt | node extract-example-urls.js
-  node extract-example-urls.js < ci-log.txt
-  node extract-example-urls.js --file=ci-log.txt
+  node extract-example-urls.js --results-dir=.testdriver/results
 
 Options:
-  --file=<path>   Read from file instead of stdin
-  --help, -h      Show this help message
+  --results-dir=<path>  Path to .testdriver/results directory (required)
+  --help, -h            Show this help message
 
 Description:
-  Parses CI log output looking for lines matching:
-    TESTDRIVER_EXAMPLE_URL::<filename>::<url>
-  
-  Updates docs/_data/examples-manifest.json with the extracted URLs.
+  Reads per-test-case JSON result files from .testdriver/results/examples/
+  and updates docs/_data/examples-manifest.json with the extracted URLs.
   Existing entries are updated, new entries are added.
 `);
 }
 
 // Main function
-async function main() {
+function main() {
   const options = parseArgs();
 
   if (options.help) {
@@ -141,25 +142,19 @@ async function main() {
     process.exit(0);
   }
 
-  console.log("🔍 Extracting example URLs from input...\n");
-
-  let inputStream;
-  if (options.file) {
-    if (!fs.existsSync(options.file)) {
-      console.error(`❌ File not found: ${options.file}`);
-      process.exit(1);
-    }
-    inputStream = fs.createReadStream(options.file);
-  } else {
-    inputStream = process.stdin;
+  if (!options.resultsDir) {
+    console.error("❌ --results-dir is required. Example: --results-dir=.testdriver/results");
+    process.exit(1);
   }
 
-  try {
-    await processInput(inputStream);
-  } catch (error) {
-    console.error(`❌ Error: ${error.message}`);
+  console.log("🔍 Reading example URLs from JSON result files...\n");
+
+  if (!fs.existsSync(options.resultsDir)) {
+    console.error(`❌ Results directory not found: ${options.resultsDir}`);
     process.exit(1);
   }
+
+  processResultsDir(options.resultsDir);
 }
 
 main();

package/docs/docs.json

@@ -87,7 +87,8 @@
             "pages": [
               "/v7/running-tests",
               "/v7/caching",
-              "/v7/ci-cd"
+              "/v7/ci-cd",
+              "/v7/test-results-json"
             ]
           },
           {

package/docs/v7/test-results-json.mdx

@@ -0,0 +1,258 @@
+---
+title: "Test Result JSON"
+sidebarTitle: "Test Result JSON"
+description: "Per-test JSON result files with metadata, versions, and infrastructure details"
+icon: "file-code"
+---
+
+## Overview
+
+TestDriver automatically writes a JSON result file for each test case after it finishes. These files contain comprehensive metadata about the test run, including SDK and runner versions, infrastructure details, interaction statistics, and links to recordings.
+
+Result files are written to:
+
+```
+.testdriver/results/<testFile>/<testName>.json
+```
+
+For example, a test file `tests/login.test.mjs` with a test named `"should log in"` produces:
+
+```
+.testdriver/results/tests/login.test.mjs/should_log_in.json
+```
+
+<Note>
+  Test names are sanitized for filesystem use — special characters are replaced with underscores and names are truncated to 200 characters.
+</Note>
+
+## Enabling
+
+No configuration is required. The JSON files are written automatically by the TestDriver Vitest reporter plugin whenever tests run.
+
+## JSON Schema
+
+Each result file is organized into logical groups:
+
+### `versions`
+
+| Field | Type | Description |
+|---|---|---|
+| `versions.sdk` | `string \| null` | TestDriver SDK version (e.g. `"7.8.0"`) |
+| `versions.vitest` | `string \| null` | Vitest version used to run the test |
+| `versions.api` | `string \| null` | TestDriver API server version |
+| `versions.runnerBefore` | `string \| null` | Runner version at sandbox start |
+| `versions.runnerAfter` | `string \| null` | Runner version after auto-update |
+| `versions.runnerWasUpdated` | `boolean` | Whether the runner was auto-updated during provisioning |
+
+### `test`
+
+| Field | Type | Description |
+|---|---|---|
+| `test.file` | `string \| null` | Relative path to the test file |
+| `test.name` | `string \| null` | Name of the test case |
+| `test.suite` | `string \| null` | Name of the parent `describe` block |
+| `test.passed` | `boolean` | Whether the test passed |
+| `test.caseId` | `string \| null` | Database ID for this test case |
+| `test.runId` | `string \| null` | Database ID for the overall test run |
+| `test.error` | `string \| null` | Error message if the test failed |
+| `test.errorStack` | `string \| null` | Error stack trace if the test failed |
+
+### `urls`
+
+| Field | Type | Description |
+|---|---|---|
+| `urls.api` | `string \| null` | API root URL used for this test |
+| `urls.console` | `string \| null` | TestDriver console base URL |
+| `urls.vnc` | `string \| null` | VNC URL for the sandbox |
+| `urls.testRun` | `string \| null` | Direct link to this test case in the console |
+
+### `replay`
+
+The `replay` object contains the recording replay URL and derived embed links. The `gifUrl` and `embedUrl` are generated automatically from the replay URL.
+
+| Field | Type | Description |
+|---|---|---|
+| `replay.url` | `string \| null` | Recording replay URL |
+| `replay.gifUrl` | `string \| null` | Animated GIF thumbnail of the recording |
+| `replay.embedUrl` | `string \| null` | Embeddable replay URL (appends `&embed=true`) |
+| `replay.markdown` | `string \| null` | Ready-to-use Markdown embed with GIF linking to the replay |
+
+The `replay.markdown` field produces a clickable GIF badge you can paste directly into PR comments, README files, or issue descriptions:
+
+```markdown
+[![Test Recording](https://api.testdriver.ai/replay/abc123/gif?shareKey=xyz)](https://console.testdriver.ai/replay/abc123?share=xyz)
+```
+
+### `date`
+
+| Field | Type | Description |
+|---|---|---|
+| `date` | `string` | ISO 8601 timestamp when the test finished |
+
+### `team`
+
+| Field | Type | Description |
+|---|---|---|
+| `team.id` | `string \| null` | Team ID from the sandbox |
+| `team.sessionId` | `string \| null` | SDK session ID |
+
+### `infrastructure`
+
+| Field | Type | Description |
+|---|---|---|
+| `infrastructure.sandboxId` | `string \| null` | Sandbox instance ID |
+| `infrastructure.instanceId` | `string \| null` | Instance ID |
+| `infrastructure.os` | `string \| null` | Operating system of the sandbox (`"linux"` or `"windows"`) |
+| `infrastructure.amiId` | `string \| null` | AWS AMI ID used for provisioning |
+| `infrastructure.e2bTemplateId` | `string \| null` | E2B template ID used for provisioning |
+| `infrastructure.imageVersion` | `string \| null` | Sandbox image version |
+
+### `realtime`
+
+| Field | Type | Description |
+|---|---|---|
+| `realtime.channel` | `string \| null` | Ably channel name used for communication |
+| `realtime.messageCount` | `number` | Number of messages published to the realtime channel |
+
+### `interactions`
+
+| Field | Type | Description |
+|---|---|---|
+| `interactions.total` | `number` | Total number of interactions recorded |
+| `interactions.cached` | `number` | Number of interactions served from cache |
+| `interactions.byType` | `object` | Breakdown of interactions by type (e.g. `find`, `click`, `assert`) |
+
+## Example Output
+
+```json
+{
+  "sdkVersion": "7.8.0",
+  "vitestVersion": "4.0.0",
+  "apiVersion": "1.45.0",
+  "runnerVersionBefore": "2.1.0",
+  "runnerVersionAfter": "2.1.1",
+  "wasUpdated": true,
+  "apiUrl": "https://api.testdriver.ai",
+  "consoleUrl": "https://console.testdriver.ai",
+  "testRunLink": "https://console.testdriver.ai/runs/abc123/def456",
+  "dashcamUrl": "https://app.dashcam.io/replay/abc123",
+  "vncUrl": "wss://sandbox-123.testdriver.ai/vnc",
+  "date": "2025-01-15T14:30:00.000Z",
+  "team": {
+    "id": "team_abc123",
+    "sessionId": "sess_xyz789"
+  },
+  "infrastructure": {
+    "sandboxId": "sandbox-123",
+    "instanceId": "i-abc123",
+    "os": "linux",
+    "amiId": "ami-0abc123",
+    "e2bTemplateId": null,
+    "imageVersion": "v2.1.0"
+  },
+  "realtime": {
+    "channel": "sandbox:sandbox-123",
+    "messageCount": 42
+  },
+  "interactions": {
+    "total": 15,
+    "cached": 3,
+    "byType": {
+      "find": 8,
+      "click": 5,
+      "assert": 2
+    }
+  }
+}
+```
+
+## Using Result Files in CI
+
+Result files are useful for extracting test metadata in CI pipelines without parsing log output.
+
+### GitHub Actions Example
+
+Use `fromJSON` to parse a result file into a GitHub Actions expression you can reference in subsequent steps:
+
+```yaml
+- name: Run tests
+  run: npx vitest run tests/login.test.mjs
+
+- name: Parse result
+  id: result
+  run: |
+    # Read the first JSON result file
+    FILE=$(find .testdriver/results -name '*.json' | head -n 1)
+    echo "json=$(cat "$FILE")" >> "$GITHUB_OUTPUT"
+
+- name: Comment on PR
+  if: fromJSON(steps.result.outputs.json).test.passed == false
+  uses: actions/github-script@v7
+  with:
+    script: |
+      const result = ${{ steps.result.outputs.json }};
+      await github.rest.issues.createComment({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        issue_number: context.issue.number,
+        body: [
+          `❌ **${result.test.name}** failed`,
+          ``,
+          `Error: ${result.test.error}`,
+          ``,
+          result.replay.markdown,
+          ``,
+          `[View full recording](${result.urls.testRun})`
+        ].join('\n')
+      });
+```
+
+You can also load all results into a matrix or iterate over them:
+
+```yaml
+- name: Run tests
+  run: npx vitest run tests/*.test.mjs
+
+- name: Collect results
+  id: results
+  run: |
+    # Merge all result files into a JSON array
+    echo "json=$(find .testdriver/results -name '*.json' -exec cat {} + | jq -s '.')" >> "$GITHUB_OUTPUT"
+
+- name: Summary
+  run: |
+    echo '## Test Results' >> $GITHUB_STEP_SUMMARY
+    RESULTS='${{ steps.results.outputs.json }}'
+    echo "$RESULTS" | jq -r '.[] | "| \(.test.name) | \(if .test.passed then "✅" else "❌" end) | \(.urls.testRun) |"' >> $GITHUB_STEP_SUMMARY
+```
+
+### Reading Results Programmatically
+
+```javascript
+import fs from "fs";
+import path from "path";
+
+const resultsDir = ".testdriver/results";
+
+function readResults(dir) {
+  const results = [];
+  for (const testDir of fs.readdirSync(dir, { recursive: true })) {
+    const fullPath = path.join(dir, testDir);
+    if (fullPath.endsWith(".json") && fs.statSync(fullPath).isFile()) {
+      results.push(JSON.parse(fs.readFileSync(fullPath, "utf-8")));
+    }
+  }
+  return results;
+}
+
+const results = readResults(resultsDir);
+const passed = results.filter(r => r.test.passed);
+const failed = results.filter(r => !r.test.passed);
+
+console.log(`${passed.length} passed, ${failed.length} failed`);
+for (const r of failed) {
+  console.log(`  FAIL: ${r.test.name} — ${r.test.error}`);
+  console.log(`  Recording: ${r.urls.testRun}`);
+  console.log(`  Embed: ${r.replay.markdown}`);
+}
+```

package/interfaces/vitest-plugin.mjs

@@ -1218,6 +1218,122 @@ class TestDriverReporter {
       );
       console.log("");
 
+      // Write per-test-case JSON result file
+      {
+        const testResult = meta.testResult || {};
+
+        // Parse replay URL to extract replayId and shareKey for embed links
+        let replayUrl = dashcamUrl || null;
+        let replayGifUrl = null;
+        let replayEmbedUrl = null;
+        let replayMarkdown = null;
+        const replayMatch = dashcamUrl && dashcamUrl.match(/\/replay\/([a-f0-9]+)\?share=([^&\s]+)/);
+        if (replayMatch) {
+          const [, replayId, shareKey] = replayMatch;
+          const apiRoot = pluginState.apiRoot;
+          replayGifUrl = `${apiRoot}/replay/${replayId}/gif?shareKey=${shareKey}`;
+          replayEmbedUrl = `${consoleUrl}/replay/${replayId}?share=${shareKey}&embed=true`;
+          replayMarkdown = `[![Test Recording](${replayGifUrl})](${replayUrl})`;
+        }
+
+        const resultData = {
+          // Versions
+          versions: {
+            sdk: testResult.sdkVersion || null,
+            vitest: resolveVitestVersion() || null,
+            api: testResult.apiVersion || null,
+            runnerBefore: testResult.runnerVersionBefore || null,
+            runnerAfter: testResult.runnerVersionAfter || null,
+            runnerWasUpdated: testResult.wasUpdated || false,
+          },
+
+          // Test info
+          test: {
+            file: testResult.testFile || null,
+            name: testResult.testName || null,
+            suite: testResult.suiteName || null,
+            passed: status === "passed",
+            caseId: testCaseDbId || null,
+            runId: testRunDbId || null,
+            error: errorMessage || testResult.error || null,
+            errorStack: errorStack || testResult.errorStack || null,
+          },
+
+          // URLs
+          urls: {
+            api: testResult.apiUrl || null,
+            console: consoleUrl || null,
+            vnc: testResult.vncUrl || null,
+            testRun: testCaseDbId ? `${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}` : null,
+          },
+
+          // Recording replay
+          replay: {
+            url: replayUrl,
+            gifUrl: replayGifUrl,
+            embedUrl: replayEmbedUrl,
+            markdown: replayMarkdown,
+          },
+
+          // Timing
+          date: testResult.date || new Date().toISOString(),
+
+          // Team & session
+          team: {
+            id: testResult.teamId || null,
+            sessionId: testResult.sessionId || null,
+          },
+
+          // Infrastructure
+          infrastructure: {
+            sandboxId: testResult.sandboxId || null,
+            instanceId: testResult.instanceId || null,
+            os: testResult.os || null,
+            amiId: testResult.amiId || null,
+            e2bTemplateId: testResult.e2bTemplateId || null,
+            imageVersion: testResult.imageVersion || null,
+          },
+
+          // Realtime
+          realtime: {
+            channel: testResult.realtimeChannel || null,
+            messageCount: testResult.realtimeMessageCount || 0,
+          },
+
+          // Interactions
+          interactions: testResult.interactions || { total: 0, cached: 0, byType: {} },
+        };
+
+        // Sanitize testName for filesystem use
+        const safeName = (test.name || "unknown").replace(/[^a-zA-Z0-9_.-]/g, "_").substring(0, 200);
+        const resultDir = path.join(process.cwd(), ".testdriver", "results", testFile);
+        fs.mkdirSync(resultDir, { recursive: true });
+
+        // Include a stable unique suffix in the filename to avoid collisions
+        // when multiple tests in the same file share the same name.
+        const hashSourceParts = [];
+        if (test.id) {
+          hashSourceParts.push(String(test.id));
+        }
+        if (Array.isArray(test.suitePath)) {
+          hashSourceParts.push(test.suitePath.join(" > "));
+        }
+        if (test.file && (test.file.name || test.file.path)) {
+          hashSourceParts.push(test.file.name || test.file.path);
+        }
+        // Fallback to the test name if no other identifiers are available.
+        if (hashSourceParts.length === 0) {
+          hashSourceParts.push(test.name || "unknown");
+        }
+        const hashSource = hashSourceParts.join(" | ");
+        const uniqueHash = crypto.createHash("sha256").update(hashSource).digest("hex").slice(0, 8);
+
+        fs.writeFileSync(
+          path.join(resultDir, `${safeName}-${uniqueHash}.json`),
+          JSON.stringify(resultData, null, 2),
+        );
+      }
+
       // If there were retries, list all per-attempt dashcam URLs for debugging
       if (hasRetries) {
         const validAttempts = dashcamUrls.filter(a => a.url);
@@ -1229,13 +1345,6 @@ class TestDriverReporter {
         }
       }
 
-      // Output parseable format for docs generation (examples only)
-      if (testFile.startsWith("examples/")) {
-        const testFileName = path.basename(testFile);
-        console.log(
-          `TESTDRIVER_EXAMPLE_URL::${testFileName}::${consoleUrl}/runs/${testRunDbId}/${testCaseDbId}`,
-        );
-      }
     } catch (error) {
       logger.error("Failed to report test case:", error.message);
     }

package/lib/vitest/hooks.mjs

@@ -649,6 +649,61 @@ export function TestDriver(context, options = {}) {
       // Clean up console spies
       cleanupConsoleSpy(currentInstance);
 
+      // Build test result metadata for JSON report output
+      {
+        const sdkPkg = require("../../package.json");
+        const inst = currentInstance.getInstance?.() || {};
+        const sbx = currentInstance.sandbox || {};
+        const apiRoot = currentInstance.config?.TD_API_ROOT || null;
+
+        context.task.meta.testResult = {
+          // Versions
+          sdkVersion: sdkPkg.version || null,
+          apiVersion: currentInstance._apiVersion || null,
+          runnerVersionBefore: inst.runnerVersionBefore || null,
+          runnerVersionAfter: inst.runnerVersionAfter || null,
+          wasUpdated: inst.wasUpdated || false,
+
+          // URLs
+          apiUrl: apiRoot,
+          vncUrl: inst.vncUrl || inst.url || null,
+
+          // Dates
+          date: new Date().toISOString(),
+
+          // Team / session
+          teamId: sbx._teamId || null,
+          sessionId: currentInstance.getSessionId?.() || null,
+
+          // Test info
+          testFile: context.task.meta.testFile || null,
+          testName: context.task.name || null,
+          suiteName: context.task.suite?.name || null,
+
+          // Test result
+          testPassed: context.task.result?.state === "pass",
+          error: context.task.result?.errors?.[0]?.message || null,
+          errorStack: context.task.result?.errors?.[0]?.stack || null,
+
+          // Infrastructure
+          sandboxId: inst.sandboxId || inst.instanceId || null,
+          instanceId: inst.instanceId || null,
+          os: currentInstance.os || inst.os || null,
+          amiId: inst.amiId || null,
+          e2bTemplateId: inst.e2bTemplateId || null,
+          imageVersion: inst.imageVersion || null,
+
+          // Realtime
+          realtimeChannel: inst.channelName || sbx._channelName || null,
+          realtimeMessageCount: typeof sbx.getPublishCount === "function" ? sbx.getPublishCount() : 0,
+
+          // Interactions
+          interactions: currentInstance._interactionStats
+            ? { ...currentInstance._interactionStats, byType: { ...currentInstance._interactionStats.byType } }
+            : { total: 0, cached: 0, byType: {} },
+        };
+      }
+
       // Wait for connection to finish if it was initiated
       if (currentInstance.__connectionPromise) {
         await currentInstance.__connectionPromise.catch(() => { }); // Ignore connection errors during cleanup

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.8.0-test.60",
+  "version": "7.8.0-test.62",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",

package/sdk.js

@@ -623,6 +623,11 @@ class Element {
 
     // Track find interaction once at the end (fire-and-forget, don't block)
     const sessionId = this.sdk.getSessionId();
+    const findCacheHit = response?.cacheHit || response?.cache_hit || response?.cached || false;
+    // Increment local interaction counters
+    this.sdk._interactionStats.total++;
+    this.sdk._interactionStats.byType.find = (this.sdk._interactionStats.byType.find || 0) + 1;
+    if (findCacheHit) this.sdk._interactionStats.cached++;
     if (sessionId && this.sdk.apiClient) {
       this.sdk.apiClient
         .req("interaction/track", {
@@ -632,11 +637,7 @@ class Element {
           timestamp: absoluteTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
           success: this._found,
           error: findError,
-          cacheHit:
-            response?.cacheHit ||
-            response?.cache_hit ||
-            response?.cached ||
-            false,
+          cacheHit: findCacheHit,
           selector: response?.selector,
           selectorUsed: !!response?.selector,
           confidence: response?.confidence ?? null,
@@ -1617,6 +1618,12 @@ class TestDriverSDK {
     // Uploaded to S3 at cleanup so they can be displayed alongside dashcam replays.
     this._logBuffer = [];
 
+    // API version discovered by _logEnvironmentInfo()
+    this._apiVersion = null;
+
+    // Local interaction counters — incremented at each interaction/track call site
+    this._interactionStats = { total: 0, cached: 0, byType: {} };
+
     // Set up event listeners once (they live for the lifetime of the SDK instance)
     this._setupLogging();
 
@@ -3201,6 +3208,11 @@ CAPTCHA_SOLVER_EOF`,
 
         // Track successful findAll interaction (fire-and-forget, don't block)
         const sessionId = this.getSessionId();
+        const findAllCacheHit = response.cached || false;
+        // Increment local interaction counters
+        this._interactionStats.total++;
+        this._interactionStats.byType.findAll = (this._interactionStats.byType.findAll || 0) + 1;
+        if (findAllCacheHit) this._interactionStats.cached++;
         if (sessionId && this.apiClient) {
           this.apiClient
             .req("interaction/track", {
@@ -3210,7 +3222,7 @@ CAPTCHA_SOLVER_EOF`,
               timestamp: absoluteTimestamp, // Absolute epoch timestamp - frontend calculates relative using clientStartDate
               success: true,
               input: { count: elements.length },
-              cacheHit: response.cached || false,
+              cacheHit: findAllCacheHit,
               selector: response.selector,
               selectorUsed: !!response.selector,
               screenshotUrl: response.screenshotKey ?? null,
@@ -3256,6 +3268,11 @@ CAPTCHA_SOLVER_EOF`,
 
         // No elements found - track interaction (fire-and-forget, don't block)
         const sessionId = this.getSessionId();
+        const noResultCacheHit = response?.cached || false;
+        // Increment local interaction counters
+        this._interactionStats.total++;
+        this._interactionStats.byType.findAll = (this._interactionStats.byType.findAll || 0) + 1;
+        if (noResultCacheHit) this._interactionStats.cached++;
         if (sessionId && this.apiClient) {
           this.apiClient
             .req("interaction/track", {
@@ -3266,7 +3283,7 @@ CAPTCHA_SOLVER_EOF`,
               success: false,
               error: "No elements found",
               input: { count: 0 },
-              cacheHit: response?.cached || false,
+              cacheHit: noResultCacheHit,
               selector: response?.selector,
               selectorUsed: !!response?.selector,
               screenshotUrl: response?.screenshotKey ?? null,
@@ -3300,6 +3317,9 @@ CAPTCHA_SOLVER_EOF`,
 
       // Track findAll error interaction (fire-and-forget, don't block)
       const sessionId = this.getSessionId();
+      // Increment local interaction counters
+      this._interactionStats.total++;
+      this._interactionStats.byType.findAll = (this._interactionStats.byType.findAll || 0) + 1;
       if (sessionId && this.apiClient) {
         this.apiClient
           .req("interaction/track", {
@@ -3851,6 +3871,8 @@ CAPTCHA_SOLVER_EOF`,
       res.on('end', () => {
         try {
           const info = JSON.parse(data);
+          // Persist API version for test result metadata
+          this._apiVersion = info.version || null;
           const commit = info.commit || 'unknown';
           const shortCommit = commit.substring(0, 7);
           const commitUrl = commit !== 'unknown'