testdriverai 7.2.25 → 7.2.27

package/lib/github-comment-formatter.js

@@ -0,0 +1,263 @@
+/**
+ * GitHub Comment Formatter for TestDriver Test Results
+ * 
+ * This module creates beautiful, formatted GitHub comments with test results,
+ * dashcam replays, screenshots, exceptions, and more.
+ */
+
+const { Octokit } = require('@octokit/rest');
+
+/**
+ * Format test results into a beautiful GitHub comment markdown
+ * @param {Object} testRunData - Test run data from Vitest plugin
+ * @returns {string} Formatted markdown comment
+ */
+function formatGitHubComment(testRunData) {
+  const { testCases, startTime, endTime, totalDuration } = testRunData;
+  
+  const passed = testCases.filter(tc => tc.result?.state === 'pass').length;
+  const failed = testCases.filter(tc => tc.result?.state === 'fail').length;
+  const skipped = testCases.filter(tc => tc.result?.state === 'skip').length;
+  
+  const statusEmoji = failed > 0 ? '❌' : '✅';
+  const statusText = failed > 0 ? 'Failed' : 'Passed';
+  
+  let comment = `## 🎯 TestDriver Test Results\n\n`;
+  comment += `**Status:** ${statusEmoji} ${statusText} • `;
+  comment += `${passed} passed`;
+  if (failed > 0) comment += `, ${failed} failed`;
+  if (skipped > 0) comment += `, ${skipped} skipped`;
+  comment += `\n`;
+  comment += `**Duration:** ${(totalDuration / 1000).toFixed(1)}s\n\n`;
+  
+  // Group test cases by file
+  const testsByFile = {};
+  for (const testCase of testCases) {
+    const file = testCase.file || 'unknown';
+    if (!testsByFile[file]) {
+      testsByFile[file] = [];
+    }
+    testsByFile[file].push(testCase);
+  }
+  
+  comment += `### Test Cases\n\n`;
+  
+  for (const [file, tests] of Object.entries(testsByFile)) {
+    comment += `#### 📁 ${file}\n\n`;
+    
+    for (const testCase of tests) {
+      const emoji = testCase.result?.state === 'pass' ? '✅' : 
+                    testCase.result?.state === 'fail' ? '❌' : '⏭️';
+      const testName = testCase.name || 'Unnamed test';
+      
+      comment += `${emoji} **${testName}**\n\n`;
+      
+      // Duration
+      if (testCase.result?.duration) {
+        comment += `- ⏱️ **Duration:** ${(testCase.result.duration / 1000).toFixed(2)}s\n`;
+      }
+      
+      // Dashcam replay
+      if (testCase.recordingData?.dashcamUrl) {
+        comment += `- 🎬 **Dashcam Replay:** [View recording](${testCase.recordingData.dashcamUrl})\n`;
+      }
+      
+      // Sandbox info
+      if (testCase.sandbox?.sandboxId) {
+        comment += `- 🖥️ **Sandbox:** \`${testCase.sandbox.sandboxId}\`\n`;
+      }
+      
+      // Exception/error details
+      if (testCase.result?.errors && testCase.result.errors.length > 0) {
+        comment += `\n<details>\n<summary>❌ Error Details</summary>\n\n`;
+        comment += `\`\`\`\n`;
+        for (const error of testCase.result.errors) {
+          comment += `${error.message || error}\n`;
+          if (error.stack) {
+            comment += `${error.stack}\n`;
+          }
+        }
+        comment += `\`\`\`\n\n`;
+        comment += `</details>\n\n`;
+      }
+      
+      // Screenshots
+      if (testCase.recordingData?.screenshots && testCase.recordingData.screenshots.length > 0) {
+        comment += `\n<details>\n<summary>📸 Screenshots (${testCase.recordingData.screenshots.length})</summary>\n\n`;
+        for (const screenshot of testCase.recordingData.screenshots) {
+          if (screenshot.url) {
+            comment += `![Screenshot](${screenshot.url})\n\n`;
+          }
+        }
+        comment += `</details>\n\n`;
+      }
+      
+      comment += `\n`;
+    }
+  }
+  
+  comment += `---\n\n`;
+  comment += `*Powered by [TestDriver.ai](https://testdriver.ai) • `;
+  comment += `Generated at ${new Date(endTime).toISOString()}*\n`;
+  
+  return comment;
+}
+
+/**
+ * Get GitHub context from environment variables
+ * Supports multiple CI/CD environments:
+ * - GitHub Actions: GITHUB_TOKEN, GITHUB_REPOSITORY, github.event.pull_request.number
+ * - Manual: TD_GITHUB_TOKEN, TD_GITHUB_REPO, TD_GITHUB_PR
+ * @returns {Object|null} GitHub context or null if not available
+ */
+function getGitHubContext() {
+  // Try TestDriver-specific env vars first (for manual control)
+  let token = process.env.TD_GITHUB_TOKEN || process.env.GITHUB_TOKEN;
+  let repo = process.env.TD_GITHUB_REPO || process.env.GITHUB_REPOSITORY;
+  let prNumber = process.env.TD_GITHUB_PR || 
+                 process.env.GITHUB_PR_NUMBER ||
+                 process.env.PR_NUMBER;
+
+  // GitHub Actions: try to extract PR number from event path
+  if (!prNumber && process.env.GITHUB_EVENT_PATH) {
+    try {
+      const fs = require('fs');
+      const eventData = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8'));
+      prNumber = eventData.pull_request?.number;
+    } catch (err) {
+      // Ignore errors reading event file
+    }
+  }
+
+  // GitHub Actions: try to extract from GITHUB_REF (refs/pull/123/merge)
+  if (!prNumber && process.env.GITHUB_REF) {
+    const match = process.env.GITHUB_REF.match(/refs\/pull\/(\d+)\/(merge|head)/);
+    if (match) {
+      prNumber = match[1];
+    }
+  }
+
+  if (!token || !repo || !prNumber) {
+    return null;
+  }
+
+  const [owner, repoName] = repo.split('/');
+  if (!owner || !repoName) {
+    return null;
+  }
+
+  return {
+    token,
+    owner,
+    repo: repoName,
+    prNumber: parseInt(prNumber, 10),
+  };
+}
+
+/**
+ * Check if GitHub integration is properly configured
+ * @returns {Object} Status object with configured flag and message
+ */
+function checkGitHubIntegration() {
+  const context = getGitHubContext();
+  
+  if (!context) {
+    const missing = [];
+    if (!process.env.TD_GITHUB_TOKEN && !process.env.GITHUB_TOKEN) {
+      missing.push('GITHUB_TOKEN or TD_GITHUB_TOKEN');
+    }
+    if (!process.env.TD_GITHUB_REPO && !process.env.GITHUB_REPOSITORY) {
+      missing.push('GITHUB_REPOSITORY or TD_GITHUB_REPO');
+    }
+    if (!process.env.TD_GITHUB_PR && !process.env.GITHUB_PR_NUMBER && !process.env.GITHUB_REF) {
+      missing.push('TD_GITHUB_PR or PR number detection');
+    }
+    
+    return {
+      configured: false,
+      message: `GitHub integration not configured. Missing: ${missing.join(', ')}`,
+      context: null,
+    };
+  }
+  
+  return {
+    configured: true,
+    message: `GitHub integration configured for ${context.owner}/${context.repo}#${context.prNumber}`,
+    context,
+  };
+}
+
+/**
+ * Post a comment to a GitHub PR
+ * @param {Object} testRunData - Test run data from Vitest plugin
+ * @returns {Promise<Object>} GitHub API response
+ */
+async function postGitHubComment(testRunData) {
+  const context = getGitHubContext();
+  
+  if (!context) {
+    console.log('ℹ️  Skipping GitHub comment: GitHub context not configured');
+    return null;
+  }
+  
+  // Check if comments are explicitly disabled
+  if (process.env.TD_GITHUB_COMMENTS === 'false') {
+    console.log('ℹ️  Skipping GitHub comment: TD_GITHUB_COMMENTS=false');
+    return null;
+  }
+  
+  try {
+    const octokit = new Octokit({ auth: context.token });
+    const commentBody = formatGitHubComment(testRunData);
+    
+    // Check if we already posted a comment
+    const { data: comments } = await octokit.issues.listComments({
+      owner: context.owner,
+      repo: context.repo,
+      issue_number: context.prNumber,
+    });
+    
+    const botComment = comments.find(c => 
+      c.body?.includes('🎯 TestDriver Test Results') && 
+      c.user?.login === (process.env.GITHUB_ACTOR || 'github-actions[bot]')
+    );
+    
+    let response;
+    if (botComment) {
+      // Update existing comment
+      response = await octokit.issues.updateComment({
+        owner: context.owner,
+        repo: context.repo,
+        comment_id: botComment.id,
+        body: commentBody,
+      });
+      console.log(`✅ Updated GitHub comment: ${response.data.html_url}`);
+    } else {
+      // Create new comment
+      response = await octokit.issues.createComment({
+        owner: context.owner,
+        repo: context.repo,
+        issue_number: context.prNumber,
+        body: commentBody,
+      });
+      console.log(`✅ Posted GitHub comment: ${response.data.html_url}`);
+    }
+    
+    return response.data;
+  } catch (error) {
+    console.error('❌ Failed to post GitHub comment:', error.message);
+    if (error.status === 401) {
+      console.error('   Check that your GITHUB_TOKEN has the correct permissions');
+    } else if (error.status === 404) {
+      console.error('   Check that the repository and PR number are correct');
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  formatGitHubComment,
+  postGitHubComment,
+  getGitHubContext,
+  checkGitHubIntegration,
+};

package/lib/github-comment.mjs

@@ -0,0 +1,424 @@
+/**
+ * GitHub Comment Generator for TestDriver Test Results
+ * 
+ * Creates beautifully formatted GitHub comments with:
+ * - Test results summary
+ * - Dashcam GIF replays
+ * - Exception details
+ * - Links to test runs
+ */
+
+import { Octokit } from '@octokit/rest';
+
+/**
+ * Format test duration in human-readable format
+ * @param {number} ms - Duration in milliseconds
+ * @returns {string} Formatted duration
+ */
+function formatDuration(ms) {
+  if (ms < 1000) return `${ms}ms`;
+  if (ms < 60000) return `${(ms / 1000).toFixed(2)}s`;
+  return `${Math.floor(ms / 60000)}m ${Math.floor((ms % 60000) / 1000)}s`;
+}
+
+/**
+ * Get status emoji for test result
+ * @param {string} status - Test status (passed, failed, skipped)
+ * @returns {string} Emoji
+ */
+function getStatusEmoji(status) {
+  switch (status) {
+    case 'passed': return '✅';
+    case 'failed': return '❌';
+    case 'skipped': return '⏭️';
+    case 'cancelled': return '🚫';
+    default: return '❓';
+  }
+}
+
+/**
+ * Generate markdown for test results table
+ * @param {Array} testCases - Array of test case objects
+ * @param {string} testRunUrl - Base URL for test run
+ * @returns {string} Markdown table
+ */
+function generateTestResultsTable(testCases, testRunUrl) {
+  if (!testCases || testCases.length === 0) {
+    return '_No test cases recorded_';
+  }
+
+  let table = '| Status | Test | File | Duration | Replay |\n';
+  table += '|--------|------|------|----------|--------|\n';
+
+  for (const test of testCases) {
+    const status = getStatusEmoji(test.status);
+    const name = test.testName || 'Unknown';
+    const file = test.testFile || 'unknown';
+    const duration = formatDuration(test.duration || 0);
+    
+    // Use test run context URL instead of direct replay URL
+    let replay = '-';
+    if (test.replayUrl) {
+      const linkUrl = (test.id && testRunUrl) ? `${testRunUrl}/${test.id}` : test.replayUrl;
+      
+      // Extract replay ID and generate GIF URL
+      const replayId = extractReplayId(test.replayUrl);
+      if (replayId) {
+        const gifUrl = getReplayGifUrl(test.replayUrl, replayId);
+        // Embed GIF with link using HTML for width control
+        replay = `<a href="${linkUrl}"><img src="${gifUrl}" width="250" alt="${name}" /></a>`;
+      } else {
+        // Fallback to text link if no GIF available
+        replay = `[🎥 View](${linkUrl})`;
+      }
+    } else if (test.id && testRunUrl) {
+      // Link without replay URL
+      replay = `[🎥 View](${testRunUrl}/${test.id})`;
+    }
+
+    table += `| ${status} | ${name} | \`${file}\` | ${duration} | ${replay} |\n`;
+  }
+
+  return table;
+}
+
+/**
+ * Generate markdown for exceptions/errors
+ * @param {Array} testCases - Array of test case objects with errors
+ * @param {string} testRunUrl - Base URL for test run
+ * @returns {string} Markdown with error details
+ */
+function generateExceptionsSection(testCases, testRunUrl) {
+  const failedTests = testCases.filter(t => t.status === 'failed' && t.errorMessage);
+  
+  if (failedTests.length === 0) {
+    return '';
+  }
+
+  let section = '\n## 🔴 Failures\n\n';
+  
+  for (const test of failedTests) {
+    section += `### ${test.testName}\n\n`;
+    section += `**File:** \`${test.testFile}\`\n\n`;
+    
+    // Use test run context URL instead of direct replay URL
+    if (test.id && testRunUrl) {
+      section += `**📹 [Watch Replay](${testRunUrl}/${test.id})**\n\n`;
+    } else if (test.replayUrl) {
+      section += `**📹 [Watch Replay](${test.replayUrl})**\n\n`;
+    }
+    
+    section += '```\n';
+    section += test.errorMessage || 'Unknown error';
+    section += '\n```\n\n';
+    
+    if (test.errorStack) {
+      section += '<details>\n';
+      section += '<summary>Stack Trace</summary>\n\n';
+      section += '```\n';
+      section += test.errorStack;
+      section += '\n```\n';
+      section += '</details>\n\n';
+    }
+  }
+
+  return section;
+}
+
+/**
+ * Generate markdown for dashcam replays section
+ * @param {Array} testCases - Array of test case objects
+ * @param {string} testRunUrl - Base URL for test run
+ * @returns {string} Markdown with replay embeds
+ */
+function generateReplaySection(testCases, testRunUrl) {
+  const testsWithReplays = testCases.filter(t => t.replayUrl);
+  
+  if (testsWithReplays.length === 0) {
+    return '';
+  }
+
+  let section = '\n## 🎥 Dashcam Replays\n\n';
+  
+  for (const test of testsWithReplays) {
+    section += `### ${test.testName}\n\n`;
+    
+    // Determine the link URL - prefer test run context
+    let linkUrl = test.replayUrl;
+    if (test.id && testRunUrl) {
+      linkUrl = `${testRunUrl}/${test.id}`;
+    }
+    
+    // Extract replay ID from URL for GIF embed
+    const replayId = extractReplayId(test.replayUrl);
+    if (replayId) {
+      const gifUrl = getReplayGifUrl(test.replayUrl, replayId);
+      section += `[![${test.testName}](${gifUrl})](${linkUrl})\n\n`;
+      section += `[🎬 View Full Replay](${linkUrl})\n\n`;
+    } else {
+      section += `[🎬 View Replay](${linkUrl})\n\n`;
+    }
+  }
+
+  return section;
+}
+
+/**
+ * Extract replay ID from dashcam URL
+ * @param {string} url - Dashcam replay URL
+ * @returns {string|null} Replay ID or null
+ */
+function extractReplayId(url) {
+  if (!url) return null;
+  
+  // Match pattern: /replay/{id} or /replay/{id}?params
+  const match = url.match(/\/replay\/([^?/#]+)/);
+  return match ? match[1] : null;
+}
+
+/**
+ * Extract share key from dashcam URL
+ * @param {string} url - Dashcam replay URL
+ * @returns {string|null} Share key or null
+ */
+function extractShareKey(url) {
+  if (!url) return null;
+  
+  // Match pattern: ?share=KEY or &share=KEY
+  const match = url.match(/[?&]share=([^&#]+)/);
+  return match ? match[1] : null;
+}
+
+/**
+ * Get GIF URL for replay
+ * @param {string} replayUrl - Full replay URL
+ * @param {string} replayId - Replay ID
+ * @returns {string} GIF URL
+ */
+function getReplayGifUrl(replayUrl, replayId) {
+  // Determine the API base URL based on the replay URL
+  let apiBaseUrl;
+  
+  if (replayUrl.includes('app.dashcam.io')) {
+    // Production dashcam uses Heroku API
+    apiBaseUrl = 'https://testdriverai-v6-c96fc597be11.herokuapp.com';
+  } else if (replayUrl.includes('console.testdriver.ai')) {
+    // TestDriver console
+    apiBaseUrl = 'https://testdriver-api.onrender.com';
+  } else if (replayUrl.includes('localhost')) {
+    // Local development
+    apiBaseUrl = 'http://localhost:1337';
+  } else {
+    // Default: try to extract base URL from replay URL
+    const urlObj = new URL(replayUrl);
+    apiBaseUrl = `${urlObj.protocol}//${urlObj.host}`;
+  }
+  
+  // Extract share key if present
+  const shareKey = extractShareKey(replayUrl);
+  
+  // Build GIF URL with shareKey parameter
+  let gifUrl = `${apiBaseUrl}/replay/${replayId}/gif`;
+  if (shareKey) {
+    gifUrl += `?shareKey=${shareKey}`;
+  }
+  
+  return gifUrl;
+}
+
+/**
+ * Generate complete GitHub comment markdown
+ * @param {Object} testRunData - Test run data
+ * @param {Array} testCases - Array of test case objects
+ * @returns {string} Complete markdown comment
+ */
+export function generateGitHubComment(testRunData, testCases = []) {
+  const {
+    runId,
+    status,
+    totalTests = 0,
+    passedTests = 0,
+    failedTests = 0,
+    skippedTests = 0,
+    duration = 0,
+    testRunUrl,
+    platform = 'unknown',
+    branch = 'unknown',
+    commit = 'unknown',
+  } = testRunData;
+
+  // Header with overall status
+  const statusEmoji = getStatusEmoji(status);
+  const statusColor = status === 'passed' ? '🟢' : status === 'failed' ? '🔴' : '🟡';
+  
+  let comment = `# ${statusColor} TestDriver Test Results\n\n`;
+  
+  // Compact summary line
+  comment += `**Status:** ${statusEmoji} ${status.toUpperCase()}`;
+  comment += ` • **Duration:** ${formatDuration(duration)}`;
+  comment += ` • ${passedTests} passed`;
+  if (failedTests > 0) comment += `, ${failedTests} failed`;
+  if (skippedTests > 0) comment += `, ${skippedTests} skipped`;
+  comment += `\n\n`;
+  
+  // Test results table (now includes embedded GIFs)
+  comment += '## 📝 Test Results\n\n';
+  comment += generateTestResultsTable(testCases, testRunUrl);
+  
+  // Link to full test run (below table)
+  if (testRunUrl) {
+    comment += `\n[📋 View Full Test Run](${testRunUrl})\n`;
+  }
+  
+  // Exceptions section (only if there are failures)
+  comment += generateExceptionsSection(testCases, testRunUrl);
+  
+  // Footer
+  comment += '\n---\n';
+  comment += `<sub>Generated by [TestDriver](https://testdriver.ai) • Run ID: \`${runId}\`</sub>\n`;
+  
+  return comment;
+}
+
+/**
+ * Post comment to GitHub PR or commit
+ * @param {Object} options - Options
+ * @param {string} options.token - GitHub token
+ * @param {string} options.owner - Repository owner
+ * @param {string} options.repo - Repository name
+ * @param {number} [options.prNumber] - Pull request number (if commenting on PR)
+ * @param {string} [options.commitSha] - Commit SHA (if commenting on commit)
+ * @param {string} options.body - Comment body (markdown)
+ * @returns {Promise<Object>} GitHub API response
+ */
+export async function postGitHubComment(options) {
+  const { token, owner, repo, prNumber, commitSha, body } = options;
+  
+  if (!token) {
+    throw new Error('GitHub token is required');
+  }
+  
+  if (!owner || !repo) {
+    throw new Error('Repository owner and name are required');
+  }
+  
+  if (!prNumber && !commitSha) {
+    throw new Error('Either prNumber or commitSha must be provided');
+  }
+
+  const octokit = new Octokit({ auth: token });
+
+  if (prNumber) {
+    // Comment on PR
+    const response = await octokit.rest.issues.createComment({
+      owner,
+      repo,
+      issue_number: prNumber,
+      body,
+    });
+    return response.data;
+  } else if (commitSha) {
+    // Comment on commit
+    const response = await octokit.rest.repos.createCommitComment({
+      owner,
+      repo,
+      commit_sha: commitSha,
+      body,
+    });
+    return response.data;
+  }
+}
+
+/**
+ * Update existing GitHub comment
+ * @param {Object} options - Options
+ * @param {string} options.token - GitHub token
+ * @param {string} options.owner - Repository owner
+ * @param {string} options.repo - Repository name
+ * @param {number} options.commentId - Comment ID to update
+ * @param {string} options.body - Updated comment body (markdown)
+ * @returns {Promise<Object>} GitHub API response
+ */
+export async function updateGitHubComment(options) {
+  const { token, owner, repo, commentId, body } = options;
+  
+  if (!token || !owner || !repo || !commentId) {
+    throw new Error('Token, owner, repo, and commentId are required');
+  }
+
+  const octokit = new Octokit({ auth: token });
+
+  const response = await octokit.rest.issues.updateComment({
+    owner,
+    repo,
+    comment_id: commentId,
+    body,
+  });
+  
+  return response.data;
+}
+
+/**
+ * Find existing TestDriver comment on PR
+ * @param {Object} options - Options
+ * @param {string} options.token - GitHub token
+ * @param {string} options.owner - Repository owner
+ * @param {string} options.repo - Repository name
+ * @param {number} options.prNumber - Pull request number
+ * @returns {Promise<Object|null>} Existing comment or null
+ */
+export async function findExistingComment(options) {
+  const { token, owner, repo, prNumber } = options;
+  
+  if (!token || !owner || !repo || !prNumber) {
+    return null;
+  }
+
+  const octokit = new Octokit({ auth: token });
+
+  const comments = await octokit.rest.issues.listComments({
+    owner,
+    repo,
+    issue_number: prNumber,
+  });
+
+  // Find comment with TestDriver signature
+  const existingComment = comments.data.find(comment =>
+    comment.body && comment.body.includes('Generated by [TestDriver]')
+  );
+
+  return existingComment || null;
+}
+
+/**
+ * Post or update GitHub comment with test results
+ * Always keeps the comment at the end of the thread by deleting and recreating
+ * @param {Object} testRunData - Test run data
+ * @param {Array} testCases - Array of test case objects
+ * @param {Object} githubOptions - GitHub API options
+ * @returns {Promise<Object>} GitHub API response
+ */
+export async function postOrUpdateTestResults(testRunData, testCases, githubOptions) {
+  const commentBody = generateGitHubComment(testRunData, testCases);
+  
+  // Try to find and delete existing comment to keep it at the end
+  if (githubOptions.prNumber) {
+    const existingComment = await findExistingComment(githubOptions);
+    
+    if (existingComment) {
+      // Delete the old comment
+      const octokit = new Octokit({ auth: githubOptions.token });
+      await octokit.rest.issues.deleteComment({
+        owner: githubOptions.owner,
+        repo: githubOptions.repo,
+        comment_id: existingComment.id,
+      });
+    }
+  }
+  
+  // Always create a new comment (will be at the end of the thread)
+  return await postGitHubComment({
+    ...githubOptions,
+    body: commentBody,
+  });
+}

package/lib/vitest/hooks.mjs

@@ -234,6 +234,21 @@ export function TestDriver(context, options = {}) {
   testdriver.__vitestContext = context.task;
   testDriverInstances.set(context.task, testdriver);
   
+  // Set platform metadata early so the reporter can show the correct OS from the start
+  if (!context.task.meta) {
+    context.task.meta = {};
+  }
+  const platform = mergedOptions.os || 'linux';
+  const absolutePath = context.task.file?.filepath || context.task.file?.name || 'unknown';
+  const projectRoot = process.cwd();
+  const testFile = absolutePath !== 'unknown'
+    ? path.relative(projectRoot, absolutePath)
+    : absolutePath;
+  
+  context.task.meta.platform = platform;
+  context.task.meta.testFile = testFile;
+  context.task.meta.testOrder = 0;
+  
   // Auto-connect if enabled (default: true)
   const autoConnect = config.autoConnect !== undefined ? config.autoConnect : true;
   const debugConsoleSpy = process.env.TD_DEBUG_CONSOLE_SPY === 'true';