tlc-claude-code 2.4.10 → 2.6.0

package/server/lib/e2e/verify-runner.js

@@ -0,0 +1,191 @@
+'use strict';
+
+const { parseAcceptanceCriteria } = require('./acceptance-parser.js');
+const { detectE2eFramework } = require('./framework-detector.js');
+
+function normalizeLogErrors(logErrors) {
+  if (Array.isArray(logErrors)) {
+    return logErrors
+      .map((entry) => String(entry || '').trim())
+      .filter(Boolean);
+  }
+
+  if (typeof logErrors === 'string') {
+    const trimmed = logErrors.trim();
+    return trimmed ? [trimmed] : [];
+  }
+
+  return [];
+}
+
+function buildExecFailureLogs(execResult) {
+  const explicitLogs = normalizeLogErrors(execResult && execResult.logErrors);
+  if (explicitLogs.length > 0) {
+    return explicitLogs;
+  }
+
+  const output = [execResult && execResult.stderr, execResult && execResult.stdout]
+    .filter((value) => typeof value === 'string')
+    .join('\n')
+    .trim();
+
+  if (output) {
+    return [output];
+  }
+
+  const exitCode = execResult && Number.isInteger(execResult.code) ? execResult.code : 1;
+  return [`E2E command exited with code ${exitCode}`];
+}
+
+function normalizeCriterionResult(entry, fallback) {
+  const status = entry && typeof entry.status === 'string'
+    ? entry.status.trim().toLowerCase()
+    : fallback.status;
+
+  return {
+    criterion: fallback.criterion,
+    status: status === 'verified' || status === 'manual' || status === 'failed'
+      ? status
+      : fallback.status,
+    screenshot: entry && typeof entry.screenshot === 'string' && entry.screenshot.trim()
+      ? entry.screenshot
+      : fallback.screenshot,
+    logErrors: entry && Object.prototype.hasOwnProperty.call(entry, 'logErrors')
+      ? normalizeLogErrors(entry.logErrors)
+      : fallback.logErrors,
+  };
+}
+
+function buildSummary(results) {
+  return results.reduce((summary, item) => {
+    if (item.status === 'verified') summary.verified += 1;
+    if (item.status === 'manual') summary.manual += 1;
+    if (item.status === 'failed') summary.failed += 1;
+    return summary;
+  }, { verified: 0, manual: 0, failed: 0 });
+}
+
+function buildCommand(framework) {
+  if (framework === 'playwright') {
+    return {
+      command: 'npx',
+      args: ['playwright', 'test', '--reporter=list'],
+    };
+  }
+
+  if (framework === 'supertest') {
+    return {
+      command: 'npx',
+      args: ['vitest', 'run'],
+    };
+  }
+
+  return null;
+}
+
+async function runVerification(options) {
+  const {
+    planPath,
+    projectDir,
+    exec,
+    fs,
+  } = options || {};
+
+  if (typeof planPath !== 'string' || !planPath.trim()) {
+    throw new TypeError('runVerification requires planPath');
+  }
+
+  if (typeof projectDir !== 'string' || !projectDir.trim()) {
+    throw new TypeError('runVerification requires projectDir');
+  }
+
+  if (typeof exec !== 'function') {
+    throw new TypeError('runVerification requires exec');
+  }
+
+  if (!fs || typeof fs.readFileSync !== 'function' || typeof fs.existsSync !== 'function') {
+    throw new TypeError('runVerification requires fs with readFileSync and existsSync');
+  }
+
+  const planContent = fs.readFileSync(planPath, 'utf8');
+  const criteria = parseAcceptanceCriteria({ planContent });
+  const manualResults = [];
+  const actionableCriteria = [];
+
+  for (const item of criteria) {
+    if (item.type === 'manual') {
+      manualResults.push({
+        criterion: item.criterion,
+        status: 'manual',
+        screenshot: null,
+        logErrors: [],
+      });
+      continue;
+    }
+
+    actionableCriteria.push(item);
+  }
+
+  if (actionableCriteria.length === 0) {
+    const results = manualResults;
+    return {
+      ...buildSummary(results),
+      results,
+    };
+  }
+
+  const { framework } = detectE2eFramework({ projectDir, fs });
+  const command = buildCommand(framework);
+
+  if (!command) {
+    const results = actionableCriteria.map(({ criterion }) => ({
+      criterion,
+      status: 'failed',
+      screenshot: null,
+      logErrors: ['No supported E2E framework detected'],
+    })).concat(manualResults);
+
+    return {
+      ...buildSummary(results),
+      results,
+    };
+  }
+
+  const execResult = await exec({
+    ...command,
+    cwd: projectDir,
+  });
+
+  const passed = execResult && execResult.code === 0;
+  const fallback = actionableCriteria.map(({ criterion }) => ({
+    criterion,
+    status: passed ? 'verified' : 'failed',
+    screenshot: execResult && typeof execResult.screenshot === 'string' ? execResult.screenshot : null,
+    logErrors: passed ? normalizeLogErrors(execResult && execResult.logErrors) : buildExecFailureLogs(execResult),
+  }));
+
+  const explicitResults = new Map();
+  for (const entry of Array.isArray(execResult && execResult.results) ? execResult.results : []) {
+    if (!entry || typeof entry.criterion !== 'string') {
+      continue;
+    }
+
+    explicitResults.set(entry.criterion.trim(), entry);
+  }
+
+  const actionableResults = fallback.map((item) => normalizeCriterionResult(
+    explicitResults.get(item.criterion),
+    item
+  ));
+
+  const results = actionableResults.concat(manualResults);
+
+  return {
+    ...buildSummary(results),
+    results,
+  };
+}
+
+module.exports = {
+  runVerification,
+};

package/server/lib/e2e/verify-runner.test.js

@@ -0,0 +1,167 @@
+'use strict';
+
+import { describe, expect, it, vi } from 'vitest';
+
+const { runVerification } = require('./verify-runner.js');
+
+function createFs(planContent, extraFiles = {}) {
+  return {
+    readFileSync(path) {
+      if (path === '/tmp/phase-plan.md') {
+        return planContent;
+      }
+
+      if (Object.prototype.hasOwnProperty.call(extraFiles, path)) {
+        return extraFiles[path];
+      }
+
+      throw new Error(`Unexpected read: ${path}`);
+    },
+
+    existsSync(path) {
+      return Object.prototype.hasOwnProperty.call(extraFiles, path);
+    },
+  };
+}
+
+describe('runVerification', () => {
+  it('runs the detected E2E command and marks automatable criteria verified', async () => {
+    const fs = createFs(`
+- [ ] User can click the export button
+Data is retained for 30 days.
+`, {
+      '/repo/package.json': JSON.stringify({
+        devDependencies: {
+          supertest: '^7.2.2',
+        },
+      }),
+    });
+
+    const exec = vi.fn().mockResolvedValue({
+      code: 0,
+      results: [
+        {
+          criterion: 'User can click the export button',
+          screenshot: 'artifacts/export.png',
+          logErrors: [],
+        },
+      ],
+    });
+
+    const result = await runVerification({
+      planPath: '/tmp/phase-plan.md',
+      projectDir: '/repo',
+      exec,
+      fs,
+    });
+
+    expect(exec).toHaveBeenCalledWith({
+      command: 'npx',
+      args: ['vitest', 'run'],
+      cwd: '/repo',
+    });
+    expect(result).toEqual({
+      verified: 1,
+      manual: 1,
+      failed: 0,
+      results: [
+        {
+          criterion: 'User can click the export button',
+          status: 'verified',
+          screenshot: 'artifacts/export.png',
+          logErrors: [],
+        },
+        {
+          criterion: 'Data is retained for 30 days.',
+          status: 'manual',
+          screenshot: null,
+          logErrors: [],
+        },
+      ],
+    });
+  });
+
+  it('uses playwright when the project has a playwright config', async () => {
+    const fs = createFs('- [ ] User can click the sign in button', {
+      '/repo/playwright.config.ts': 'export default {};',
+    });
+
+    const exec = vi.fn().mockResolvedValue({ code: 0 });
+
+    await runVerification({
+      planPath: '/tmp/phase-plan.md',
+      projectDir: '/repo',
+      exec,
+      fs,
+    });
+
+    expect(exec).toHaveBeenCalledWith({
+      command: 'npx',
+      args: ['playwright', 'test', '--reporter=list'],
+      cwd: '/repo',
+    });
+  });
+
+  it('fails automatable criteria when no supported E2E framework is detected', async () => {
+    const fs = createFs('- [ ] User can click the sign in button');
+    const exec = vi.fn();
+
+    const result = await runVerification({
+      planPath: '/tmp/phase-plan.md',
+      projectDir: '/repo',
+      exec,
+      fs,
+    });
+
+    expect(exec).not.toHaveBeenCalled();
+    expect(result).toEqual({
+      verified: 0,
+      manual: 0,
+      failed: 1,
+      results: [
+        {
+          criterion: 'User can click the sign in button',
+          status: 'failed',
+          screenshot: null,
+          logErrors: ['No supported E2E framework detected'],
+        },
+      ],
+    });
+  });
+
+  it('propagates failed executions into each automatable criterion', async () => {
+    const fs = createFs(`
+Given the user is on the login page
+When they click the sign in button
+Then the dashboard page should load
+`, {
+      '/repo/playwright.config.ts': 'export default {};',
+    });
+
+    const exec = vi.fn().mockResolvedValue({
+      code: 1,
+      stderr: 'Timeout while waiting for dashboard',
+    });
+
+    const result = await runVerification({
+      planPath: '/tmp/phase-plan.md',
+      projectDir: '/repo',
+      exec,
+      fs,
+    });
+
+    expect(result).toEqual({
+      verified: 0,
+      manual: 0,
+      failed: 1,
+      results: [
+        {
+          criterion: 'the dashboard page should load',
+          status: 'failed',
+          screenshot: null,
+          logErrors: ['Timeout while waiting for dashboard'],
+        },
+      ],
+    });
+  });
+});