testdriverai 7.8.0-test.32 → 7.8.0-test.39

package/lib/github-comment.mjs

@@ -49,7 +49,7 @@ function generateTestResultsTable(testCases, testRunUrl) {
 
   // Filter out skipped tests
   const nonSkippedTests = testCases.filter(test => test.status !== 'skipped');
-  
+
   if (nonSkippedTests.length === 0) {
     return '_No test cases to display (all tests were skipped)_';
   }
@@ -62,12 +62,12 @@ function generateTestResultsTable(testCases, testRunUrl) {
     const name = test.testName || 'Unknown';
     const file = test.testFile || 'unknown';
     const duration = formatDuration(test.duration || 0);
-    
+
     // Use test run context URL instead of direct replay URL
     let replay = '-';
     if (test.replayUrl) {
       const linkUrl = (test.id && testRunUrl) ? `${testRunUrl}/${test.id}` : test.replayUrl;
-      
+
       // Extract replay ID and generate GIF URL
       const replayId = extractReplayId(test.replayUrl);
       if (replayId) {
@@ -97,28 +97,28 @@ function generateTestResultsTable(testCases, testRunUrl) {
  */
 function generateExceptionsSection(testCases, testRunUrl) {
   const failedTests = testCases.filter(t => t.status === 'failed' && t.errorMessage);
-  
+
   if (failedTests.length === 0) {
     return '';
   }
 
   let section = '\n## 🔴 Failures\n\n';
-  
+
   for (const test of failedTests) {
     section += `### ${test.testName}\n\n`;
     section += `**File:** \`${test.testFile}\`\n\n`;
-    
+
     // Use test run context URL instead of direct replay URL
     if (test.id && testRunUrl) {
       section += `**📹 [Watch Replay](${testRunUrl}/${test.id})**\n\n`;
     } else if (test.replayUrl) {
       section += `**📹 [Watch Replay](${test.replayUrl})**\n\n`;
     }
-    
+
     section += '```\n';
     section += test.errorMessage || 'Unknown error';
     section += '\n```\n\n';
-    
+
     if (test.errorStack) {
       section += '<details>\n';
       section += '<summary>Stack Trace</summary>\n\n';
@@ -140,22 +140,22 @@ function generateExceptionsSection(testCases, testRunUrl) {
  */
 function generateReplaySection(testCases, testRunUrl) {
   const testsWithReplays = testCases.filter(t => t.replayUrl);
-  
+
   if (testsWithReplays.length === 0) {
     return '';
   }
 
   let section = '\n## 🎥 Dashcam Replays\n\n';
-  
+
   for (const test of testsWithReplays) {
     section += `### ${test.testName}\n\n`;
-    
+
     // Determine the link URL - prefer test run context
     let linkUrl = test.replayUrl;
     if (test.id && testRunUrl) {
       linkUrl = `${testRunUrl}/${test.id}`;
     }
-    
+
     // Extract replay ID from URL for GIF embed
     const replayId = extractReplayId(test.replayUrl);
     if (replayId) {
@@ -177,7 +177,7 @@ function generateReplaySection(testCases, testRunUrl) {
  */
 function extractReplayId(url) {
   if (!url) return null;
-  
+
   // Match pattern: /replay/{id} or /replay/{id}?params
   const match = url.match(/\/replay\/([^?/#]+)/);
   return match ? match[1] : null;
@@ -190,7 +190,7 @@ function extractReplayId(url) {
  */
 function extractShareKey(url) {
   if (!url) return null;
-  
+
   // Match pattern: ?share=KEY or &share=KEY
   const match = url.match(/[?&]share=([^&#]+)/);
   return match ? match[1] : null;
@@ -204,32 +204,50 @@ function extractShareKey(url) {
  */
 function getReplayGifUrl(replayUrl, replayId) {
   // Determine the API base URL based on the replay URL
+  // Replay URLs use console domains; GIF endpoints live on the corresponding API domain
   let apiBaseUrl;
-  
+
   if (replayUrl.includes('app.dashcam.io')) {
     // Production dashcam uses Heroku API
     apiBaseUrl = 'https://testdriverai-v6-c96fc597be11.herokuapp.com';
-  } else if (replayUrl.includes('console.testdriver.ai')) {
-    // TestDriver console
-    apiBaseUrl = 'https://api.testdriver.ai';
   } else if (replayUrl.includes('localhost')) {
     // Local development
     apiBaseUrl = 'http://localhost:1337';
   } else {
-    // Default: try to extract base URL from replay URL
-    const urlObj = new URL(replayUrl);
-    apiBaseUrl = `${urlObj.protocol}//${urlObj.host}`;
+    // Map console URLs → API URLs for all environments
+    // console-test.testdriver.ai → api-test.testdriver.ai
+    // console-canary.testdriver.ai → api-canary.testdriver.ai
+    // console.testdriver.ai → api.testdriver.ai
+    const consoleEnvMatch = replayUrl.match(/https:\/\/console-(test|canary)\.testdriver\.ai/);
+    if (consoleEnvMatch) {
+      apiBaseUrl = `https://api-${consoleEnvMatch[1]}.testdriver.ai`;
+    } else if (replayUrl.includes('console.testdriver.ai')) {
+      apiBaseUrl = 'https://api.testdriver.ai';
+    }
+    // Fly.io: map web app → API app
+    // pr-123-web.fly.dev → pr-123-api.fly.dev
+    // td-test-web.fly.dev → td-test-api.fly.dev
+    else {
+      const flyWebMatch = replayUrl.match(/https:\/\/([\w-]+)-web\.fly\.dev/);
+      if (flyWebMatch) {
+        apiBaseUrl = `https://${flyWebMatch[1]}-api.fly.dev`;
+      } else {
+        // Fallback: extract base URL from replay URL as-is
+        const urlObj = new URL(replayUrl);
+        apiBaseUrl = `${urlObj.protocol}//${urlObj.host}`;
+      }
+    }
   }
-  
+
   // Extract share key if present
   const shareKey = extractShareKey(replayUrl);
-  
+
   // Build GIF URL with shareKey parameter
   let gifUrl = `${apiBaseUrl}/replay/${replayId}/gif`;
   if (shareKey) {
     gifUrl += `?shareKey=${shareKey}`;
   }
-  
+
   return gifUrl;
 }
 
@@ -257,9 +275,9 @@ export function generateGitHubComment(testRunData, testCases = []) {
   // Header with overall status
   const statusEmoji = getStatusEmoji(status);
   const statusColor = status === 'passed' ? '🟢' : status === 'failed' ? '🔴' : '🟡';
-  
+
   let comment = `# ${statusColor} TestDriver Test Results\n\n`;
-  
+
   // Compact summary line
   comment += `**Status:** ${statusEmoji} ${status.toUpperCase()}`;
   comment += ` • **Duration:** ${formatDuration(duration)}`;
@@ -270,23 +288,23 @@ export function generateGitHubComment(testRunData, testCases = []) {
     comment += `, ${skippedTests} skipped`;
   }
   comment += `\n\n`;
-  
+
   // Exceptions section (only if there are failures) - show first
   comment += generateExceptionsSection(testCases, testRunUrl);
-  
+
   // Test results table (now includes embedded GIFs)
   comment += '## 📝 Test Results\n\n';
   comment += generateTestResultsTable(testCases, testRunUrl);
-  
+
   // Link to full test run (below table)
   if (testRunUrl) {
     comment += `\n[📋 View Full Test Run](${testRunUrl})\n`;
   }
-  
+
   // Footer
   comment += '\n---\n';
   comment += `<sub>Generated by [TestDriver](https://testdriver.ai) • Run ID: \`${runId}\`</sub>\n`;
-  
+
   return comment;
 }
 
@@ -303,15 +321,15 @@ export function generateGitHubComment(testRunData, testCases = []) {
  */
 export async function postGitHubComment(options) {
   const { token, owner, repo, prNumber, commitSha, body } = options;
-  
+
   if (!token) {
     throw new Error('GitHub token is required');
   }
-  
+
   if (!owner || !repo) {
     throw new Error('Repository owner and name are required');
   }
-  
+
   if (!prNumber && !commitSha) {
     throw new Error('Either prNumber or commitSha must be provided');
   }
@@ -351,7 +369,7 @@ export async function postGitHubComment(options) {
  */
 export async function updateGitHubComment(options) {
   const { token, owner, repo, commentId, body } = options;
-  
+
   if (!token || !owner || !repo || !commentId) {
     throw new Error('Token, owner, repo, and commentId are required');
   }
@@ -364,7 +382,7 @@ export async function updateGitHubComment(options) {
     comment_id: commentId,
     body,
   });
-  
+
   return response.data;
 }
 
@@ -379,7 +397,7 @@ export async function updateGitHubComment(options) {
  */
 export async function findExistingComment(options) {
   const { token, owner, repo, prNumber } = options;
-  
+
   if (!token || !owner || !repo || !prNumber) {
     return null;
   }
@@ -410,11 +428,11 @@ export async function findExistingComment(options) {
  */
 export async function postOrUpdateTestResults(testRunData, testCases, githubOptions) {
   const commentBody = generateGitHubComment(testRunData, testCases);
-  
+
   // Try to find and delete existing comment to keep it at the end
   if (githubOptions.prNumber) {
     const existingComment = await findExistingComment(githubOptions);
-    
+
     if (existingComment) {
       // Delete the old comment
       const octokit = new Octokit({ auth: githubOptions.token });
@@ -425,7 +443,7 @@ export async function postOrUpdateTestResults(testRunData, testCases, githubOpti
       });
     }
   }
-  
+
   // Always create a new comment (will be at the end of the thread)
   return await postGitHubComment({
     ...githubOptions,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "testdriverai",
-  "version": "7.8.0-test.32",
+  "version": "7.8.0-test.39",
   "description": "Next generation autonomous AI agent for end-to-end testing of web & desktop",
   "main": "sdk.js",
   "types": "sdk.d.ts",

package/setup/aws/install-dev-runner.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Usage: ./install-dev-runner.sh <instance-id>
+INSTANCE_ID="${1:?Usage: $0 <instance-id>}"
+AWS_REGION="${AWS_REGION:-us-east-2}"
+
+RUNNER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../runner" && pwd)"
+
+echo "Packing local runner..."
+TMPDIR=$(mktemp -d)
+pushd "$RUNNER_DIR" > /dev/null
+npm pack --pack-destination "$TMPDIR" > /dev/null 2>&1
+TARBALL=$(ls "$TMPDIR"/*.tgz)
+popd > /dev/null
+echo "Tarball: $TARBALL"
+
+echo "Uploading to S3..."
+S3_KEY="runner-dev/$(date +%s)-$(openssl rand -hex 4)/runner.tgz"
+aws s3 cp "$TARBALL" "s3://v7-transfer/${S3_KEY}" --region "$AWS_REGION" > /dev/null
+DOWNLOAD_URL=$(aws s3 presign "s3://v7-transfer/${S3_KEY}" --expires-in 900 --region "$AWS_REGION")
+rm -rf "$TMPDIR"
+
+echo "Creating SSM params file..."
+
+# Write Python script to temp file to generate valid JSON
+PYTHON_SCRIPT=$(mktemp --suffix=.py)
+cat > "$PYTHON_SCRIPT" << 'PYEOF'
+import json
+import sys
+
+url = sys.argv[1]
+
+commands = [
+    "Write-Host '=== Stopping runner ==='",
+    "Stop-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+    "Stop-Process -Name node -Force -ErrorAction SilentlyContinue", 
+    "Start-Sleep -Seconds 2",
+    "Set-Location 'C:\\testdriver\\sandbox-agent'",
+    "$tarball = 'C:\\Windows\\Temp\\runner-dev.tgz'",
+    f"Invoke-WebRequest -Uri '{url}' -OutFile $tarball",
+    "Write-Host 'Tarball size:'; (Get-Item $tarball).Length",
+    "Remove-Item -Path lib -Recurse -Force -ErrorAction SilentlyContinue", 
+    "tar -xzf $tarball --strip-components=1 -C .",
+    "Get-Content 'package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Write-Host '=== Starting runner ==='",
+    "Start-ScheduledTask -TaskName RunTestDriverAgent",
+    "Start-Sleep -Seconds 3",
+    "Get-Content 'C:\\testdriver\\log.txt' -Tail 20"
+]
+
+params = {"commands": commands}
+print(json.dumps(params))
+PYEOF
+
+python3 "$PYTHON_SCRIPT" "$DOWNLOAD_URL" > /tmp/ssm-install-params.json
+rm "$PYTHON_SCRIPT"
+
+echo "Sending SSM command..."
+CMD_JSON=$(aws ssm send-command \
+  --region "$AWS_REGION" \
+  --instance-ids "$INSTANCE_ID" \
+  --document-name "AWS-RunPowerShellScript" \
+  --parameters "file:///tmp/ssm-install-params.json" \
+  --output json)
+
+COMMAND_ID=$(echo "$CMD_JSON" | jq -r '.Command.CommandId')
+echo "Command ID: $COMMAND_ID"
+
+echo "Waiting for completion..."
+aws ssm wait command-executed --region "$AWS_REGION" --command-id "$COMMAND_ID" --instance-id "$INSTANCE_ID" || true
+
+echo "Getting output..."
+aws ssm get-command-invocation \
+  --region "$AWS_REGION" \
+  --command-id "$COMMAND_ID" \
+  --instance-id "$INSTANCE_ID" \
+  --query 'StandardOutputContent' \
+  --output text

package/setup/aws/spawn-runner.sh

@@ -141,6 +141,140 @@ while :; do
   sleep 20
 done
 
+# --- 4) Install/update runner ---
+echo "Installing runner..."
+
+# Determine environment and version
+TD_ENV="${TD_ENV:-stable}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SDK_PKG_JSON="${SCRIPT_DIR}/../../../sdk/package.json"
+RUNNER_DIR="${SCRIPT_DIR}/../../../runner"
+
+if [ -f "$SDK_PKG_JSON" ]; then
+  RUNNER_VERSION=$(jq -r '.version' "$SDK_PKG_JSON")
+  echo "Runner version from SDK: $RUNNER_VERSION"
+else
+  RUNNER_VERSION="$TD_ENV"
+  echo "SDK package.json not found, using env tag: $RUNNER_VERSION"
+fi
+
+if [ "$TD_ENV" = "dev" ]; then
+  echo "Dev mode: packing and uploading local runner to S3..."
+  
+  # Pack local runner
+  TMPDIR=$(mktemp -d)
+  pushd "$RUNNER_DIR" > /dev/null
+  npm pack --pack-destination "$TMPDIR" > /dev/null 2>&1
+  TARBALL=$(ls "$TMPDIR"/*.tgz | head -1)
+  popd > /dev/null
+  
+  # Upload to S3
+  S3_BUCKET="${AWS_BUCKET_IMAGE_TRANSFER:-v7-transfer}"
+  S3_KEY="runner-dev/$(date +%s)-$(openssl rand -hex 4)/runner.tgz"
+  aws s3 cp "$TARBALL" "s3://${S3_BUCKET}/${S3_KEY}" --region "$AWS_REGION"
+  
+  # Generate presigned URL (15 min)
+  DOWNLOAD_URL=$(aws s3 presign "s3://${S3_BUCKET}/${S3_KEY}" --expires-in 900 --region "$AWS_REGION")
+  rm -rf "$TMPDIR"
+  
+  # Build SSM parameters JSON in a temp file to avoid shell escaping issues with URL
+  PARAMS_FILE=$(mktemp)
+  cat > "$PARAMS_FILE" << 'PARAMS_EOF'
+{
+  "commands": [
+    "Write-Host '=== Starting runner dev install ==='",
+    "Write-Host 'Stopping existing runner processes...'",
+    "Stop-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+    "Stop-Process -Name node -Force -ErrorAction SilentlyContinue",
+    "Start-Sleep -Seconds 2",
+    "Write-Host 'Current runner version:'",
+    "Get-Content 'C:\\testdriver\\sandbox-agent\\package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Set-Location 'C:\\testdriver\\sandbox-agent'",
+    "Write-Host 'Dev mode: downloading runner from S3...'",
+    "$tarball = 'C:\\Windows\\Temp\\runner-dev.tgz'",
+PARAMS_EOF
+  
+  # Add the URL line with proper JSON escaping
+  echo "    \"Invoke-WebRequest -Uri '$(echo "$DOWNLOAD_URL" | sed 's/"/\\"/g')' -OutFile \$tarball\"," >> "$PARAMS_FILE"
+  
+  cat >> "$PARAMS_FILE" << 'PARAMS_EOF'
+    "Write-Host 'Downloaded tarball size:'",
+    "(Get-Item $tarball).Length",
+    "Write-Host 'Extracting runner...'",
+    "tar -xzf $tarball -C 'C:\\Windows\\Temp'",
+    "Write-Host 'Extracted package contents:'",
+    "Get-ChildItem 'C:\\Windows\\Temp\\package' -Recurse | Select-Object FullName",
+    "Write-Host 'New runner version in package:'",
+    "Get-Content 'C:\\Windows\\Temp\\package\\package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Write-Host 'Clearing old lib folder...'",
+    "Remove-Item 'C:\\testdriver\\sandbox-agent\\lib' -Recurse -Force -ErrorAction SilentlyContinue",
+    "Write-Host 'Copying files to sandbox-agent...'",
+    "xcopy 'C:\\Windows\\Temp\\package\\*' 'C:\\testdriver\\sandbox-agent\\' /E /Y /I",
+    "Write-Host 'Files after copy:'",
+    "Get-ChildItem 'C:\\testdriver\\sandbox-agent' | Select-Object Name",
+    "Remove-Item 'C:\\Windows\\Temp\\package' -Recurse -Force -ErrorAction SilentlyContinue",
+    "Remove-Item $tarball -Force -ErrorAction SilentlyContinue",
+    "Write-Host 'Runner version after copy:'",
+    "Get-Content 'C:\\testdriver\\sandbox-agent\\package.json' | ConvertFrom-Json | Select-Object -ExpandProperty version",
+    "Write-Host 'Installing npm dependencies...'",
+    "npm install --omit=dev 2>&1 | Write-Host",
+    "Write-Host 'Final verification - ably-service.js exists:'",
+    "Test-Path 'C:\\testdriver\\sandbox-agent\\lib\\ably-service.js'",
+    "Write-Host 'Restarting RunTestDriverAgent scheduled task...'",
+    "Start-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue",
+    "Write-Host '=== Runner install complete (dev) ==='"
+  ]
+}
+PARAMS_EOF
+
+  echo "Sending SSM command to download and install runner from S3..."
+  INSTALL_CMD=$(aws ssm send-command \
+    --region "$AWS_REGION" \
+    --instance-ids "$INSTANCE_ID" \
+    --document-name "AWS-RunPowerShellScript" \
+    --parameters "file://$PARAMS_FILE" \
+    --timeout-seconds 180 \
+    --output json)
+  rm -f "$PARAMS_FILE"
+else
+  echo "Installing @testdriverai/runner@${RUNNER_VERSION} via npm..."
+  INSTALL_CMD=$(aws ssm send-command \
+    --region "$AWS_REGION" \
+    --instance-ids "$INSTANCE_ID" \
+    --document-name "AWS-RunPowerShellScript" \
+    --parameters "commands=[
+      \"Set-Location 'C:\\\\testdriver\\\\sandbox-agent'\",
+      \"Write-Host 'Installing @testdriverai/runner@${RUNNER_VERSION}...'\",
+      \"npm install @testdriverai/runner@${RUNNER_VERSION} --omit=dev 2>&1 | Write-Host\",
+      \"Write-Host 'Stopping runner (config not yet provisioned)...'\",
+      \"Stop-Process -Name node -Force -ErrorAction SilentlyContinue\",
+      \"Stop-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue\",
+      \"Start-Sleep -Seconds 2\",
+      \"Start-ScheduledTask -TaskName RunTestDriverAgent -ErrorAction SilentlyContinue\",
+      \"Write-Host 'Runner install complete'\"
+    ]" \
+    --timeout-seconds 120 \
+    --output json)
+fi
+
+INSTALL_CMD_ID=$(jq -r '.Command.CommandId' <<<"$INSTALL_CMD")
+echo "Runner install command sent (Command ID: $INSTALL_CMD_ID)"
+
+# Wait for install to complete
+echo "Waiting for runner install to complete..."
+if aws ssm wait command-executed --region "$AWS_REGION" --command-id "$INSTALL_CMD_ID" --instance-id "$INSTANCE_ID" 2>/dev/null; then
+  echo "✓ Runner install succeeded"
+else
+  INSTALL_STATUS=$(aws ssm get-command-invocation \
+    --region "$AWS_REGION" \
+    --command-id "$INSTALL_CMD_ID" \
+    --instance-id "$INSTANCE_ID" \
+    --output json 2>/dev/null || echo '{}')
+  echo "⚠ Runner install status: $(jq -r '.Status // "Unknown"' <<<"$INSTALL_STATUS")"
+  echo "Output: $(jq -r '.StandardOutputContent // "No output"' <<<"$INSTALL_STATUS" | head -20)"
+  echo "Errors: $(jq -r '.StandardErrorContent // "No errors"' <<<"$INSTALL_STATUS" | head -10)"
+fi
+
 echo "Getting Public IP..."
 
 # --- 5) Get instance Public IP ---

package/vitest.runner.config.mjs

@@ -0,0 +1,33 @@
+/**
+ * Vitest config for runner/packer tests.
+ * Lives under sdk/ so vitest resolves from sdk/node_modules,
+ * but uses shared/resolve-env.js for environment variable loading.
+ */
+import { defineConfig } from "vitest/config";
+import { createRequire } from "module";
+import { dirname, resolve } from "path";
+import { fileURLToPath } from "url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const monoRoot = resolve(__dirname, "..");
+
+const require = createRequire(import.meta.url);
+const { resolveEnv } = require("../shared/resolve-env");
+
+const plan = process.env.TD_PLAN || "enterprise";
+const env = process.env.TD_ENV || "dev";
+const resolved = resolveEnv(env, plan);
+
+// Apply to the main process so test code sees the vars immediately
+Object.assign(process.env, resolved);
+
+export default defineConfig({
+  test: {
+    root: monoRoot,
+    testTimeout: 900_000,   // 15 min per test
+    hookTimeout: 2_400_000, // 40 min for beforeAll (AMI build + spawn)
+    reporters: ["default"],
+    include: ["runner/packer/test/**/*.test.mjs"],
+    env: resolved,
+  },
+});