sofia-cli 0.1.1

package/tests/e2e/developFailureE2e.spec.ts

@@ -0,0 +1,289 @@
+/**
+ * T050: E2E failure/recovery test.
+ *
+ * Verifies graceful termination; verifies `finalStatus` is "failed" or "partial"
+ * in session state; verifies `terminationReason: "max-iterations"`;
+ * verifies user-facing output includes recovery guidance (Constitution VI compliance).
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { createRequire } from 'node:module';
+
+import { RalphLoop } from '../../src/develop/ralphLoop.js';
+import { PocScaffolder } from '../../src/develop/pocScaffolder.js';
+import { TestRunner } from '../../src/develop/testRunner.js';
+import type { WorkshopSession } from '../../src/shared/schemas/session.js';
+import type { LoopIO } from '../../src/loop/conversationLoop.js';
+import type { CopilotClient } from '../../src/shared/copilotClient.js';
+import type { TestResults } from '../../src/shared/schemas/session.js';
+
+vi.mock('node:child_process', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('node:child_process')>();
+  return {
+    ...actual,
+    spawn: vi.fn((cmd: string, args: string[]) => {
+      if (cmd === 'npm' && args.includes('install')) {
+        return {
+          stdout: { on: vi.fn() },
+          stderr: { on: vi.fn() },
+          on: vi.fn((event: string, cb: (code: number) => void) => {
+            if (event === 'close') cb(0);
+          }),
+          kill: vi.fn(),
+          killed: false,
+        };
+      }
+      return actual.spawn(cmd, args);
+    }),
+  };
+});
+
+const require = createRequire(import.meta.url);
+const fixtureSession: WorkshopSession = require('../fixtures/completedSession.json') as WorkshopSession;
+
+describe('E2E: failure/recovery (T050)', () => {
+  let tmpDir: string;
+  let originalExitCode: number | undefined;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(join(tmpdir(), 'sofia-e2e-failure-'));
+    originalExitCode = process.exitCode as number | undefined;
+    process.exitCode = undefined;
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+    process.exitCode = originalExitCode;
+    vi.restoreAllMocks();
+  });
+
+  function makeIo(): LoopIO & { writtenLines: string[]; activityLines: string[] } {
+    const writtenLines: string[] = [];
+    const activityLines: string[] = [];
+    return {
+      writtenLines,
+      activityLines,
+      write: vi.fn((text: string) => { writtenLines.push(text); }),
+      writeActivity: vi.fn((text: string) => { activityLines.push(text); }),
+      writeToolSummary: vi.fn(),
+      readInput: vi.fn().mockResolvedValue(null),
+      showDecisionGate: vi.fn(),
+      isJsonMode: false,
+      isTTY: false,
+    };
+  }
+
+  function makeFakeScaffolder(outputDir: string): PocScaffolder {
+    return {
+      scaffold: vi.fn().mockImplementation(async () => {
+        const { writeFile, mkdir } = await import('node:fs/promises');
+        await mkdir(join(outputDir, 'src'), { recursive: true });
+        await writeFile(join(outputDir, 'package.json'), JSON.stringify({
+          name: 'test-poc',
+          scripts: { test: 'vitest run' },
+          dependencies: {},
+          devDependencies: {},
+        }), 'utf-8');
+        await writeFile(join(outputDir, 'src', 'index.ts'), 'export function main() {}', 'utf-8');
+        return {
+          createdFiles: ['package.json', 'src/index.ts'],
+          skippedFiles: [],
+          context: {
+            projectName: 'test-poc',
+            ideaTitle: 'Test',
+            ideaDescription: 'Test',
+            techStack: { language: 'TypeScript', runtime: 'Node.js 20', testRunner: 'npm test' },
+            planSummary: 'Test',
+            sessionId: fixtureSession.sessionId,
+            outputDir,
+          },
+        };
+      }),
+      getTemplateFiles: () => [],
+    } as unknown as PocScaffolder;
+  }
+
+  function makeAlwaysFailingClient(): CopilotClient {
+    return {
+      createSession: vi.fn().mockResolvedValue({
+        send: vi.fn().mockReturnValue({
+          async *[Symbol.asyncIterator]() {
+            yield { type: 'TextDelta', text: '', timestamp: '' };
+          },
+        }),
+        getHistory: () => [],
+      }),
+    };
+  }
+
+  function makeAlwaysFailingTestRunner(): TestRunner {
+    return {
+      run: vi.fn().mockResolvedValue({
+        passed: 0,
+        failed: 1,
+        skipped: 0,
+        total: 1,
+        durationMs: 400,
+        failures: [{ testName: 'test', message: 'always fails' }],
+        rawOutput: '',
+      } satisfies TestResults),
+    } as unknown as TestRunner;
+  }
+
+  it('terminates with max-iterations when all tests keep failing', async () => {
+    const io = makeIo();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+    const client = makeAlwaysFailingClient();
+    const testRunner = makeAlwaysFailingTestRunner();
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 2,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+
+    expect(result.terminationReason).toBe('max-iterations');
+    expect(['failed', 'partial']).toContain(result.finalStatus);
+  });
+
+  it('verifies terminationReason=max-iterations in session state', async () => {
+    const io = makeIo();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+    const client = makeAlwaysFailingClient();
+    const testRunner = makeAlwaysFailingTestRunner();
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 2,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+    expect(result.session.poc?.terminationReason).toBe('max-iterations');
+    expect(result.session.poc?.finalStatus).toBeDefined();
+  });
+
+  it('session has iteration history after failed loop', async () => {
+    const io = makeIo();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+    const client = makeAlwaysFailingClient();
+    const testRunner = makeAlwaysFailingTestRunner();
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 2,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+
+    // Should have at least scaffold iteration
+    expect(result.session.poc?.iterations.length).toBeGreaterThan(0);
+    expect(result.session.poc?.iterations[0].outcome).toBe('scaffold');
+  });
+
+  it('shows recovery guidance in non-JSON output for failed status (Constitution VI)', async () => {
+    const { developCommand } = await import('../../src/cli/developCommand.js');
+
+    const devIo = makeIo();
+    const client = makeAlwaysFailingClient();
+
+    const store = {
+      load: vi.fn().mockResolvedValue(fixtureSession),
+      save: vi.fn().mockResolvedValue(undefined),
+      list: vi.fn().mockResolvedValue([fixtureSession.sessionId]),
+    };
+
+    // Mock RalphLoop.prototype.run to return failed immediately
+    const originalRun = RalphLoop.prototype.run;
+    const sessionWithFailedPoc: WorkshopSession = {
+      ...fixtureSession,
+      poc: {
+        repoSource: 'local' as const,
+        repoPath: tmpDir,
+        iterations: [],
+        finalStatus: 'failed' as const,
+        terminationReason: 'max-iterations' as const,
+      },
+    };
+    RalphLoop.prototype.run = vi.fn().mockResolvedValue({
+      session: sessionWithFailedPoc,
+      finalStatus: 'failed' as const,
+      terminationReason: 'max-iterations' as const,
+      iterationsCompleted: 2,
+      outputDir: tmpDir,
+    });
+
+    try {
+      await developCommand(
+        { session: fixtureSession.sessionId, maxIterations: 1, output: tmpDir },
+        { store, io: devIo, client },
+      );
+    } finally {
+      RalphLoop.prototype.run = originalRun;
+    }
+
+    const allOutput = devIo.writtenLines.join('\n');
+    // developCommand should show recovery guidance for non-success status
+    expect(allOutput).toMatch(/resume|retry|force|more.*iter/i);
+  });
+
+  it('sets process.exitCode=1 when loop terminates with failed status', async () => {
+    const { developCommand } = await import('../../src/cli/developCommand.js');
+    const devIo = makeIo();
+    const client = makeAlwaysFailingClient();
+
+    const store = {
+      load: vi.fn().mockResolvedValue(fixtureSession),
+      save: vi.fn().mockResolvedValue(undefined),
+      list: vi.fn().mockResolvedValue([fixtureSession.sessionId]),
+    };
+
+    const sessionWithFailedPoc: WorkshopSession = {
+      ...fixtureSession,
+      poc: {
+        repoSource: 'local' as const,
+        repoPath: tmpDir,
+        iterations: [],
+        finalStatus: 'failed' as const,
+        terminationReason: 'max-iterations' as const,
+      },
+    };
+
+    const originalRun = RalphLoop.prototype.run;
+    RalphLoop.prototype.run = vi.fn().mockResolvedValue({
+      session: sessionWithFailedPoc,
+      finalStatus: 'failed' as const,
+      terminationReason: 'max-iterations' as const,
+      iterationsCompleted: 2,
+      outputDir: tmpDir,
+    });
+
+    try {
+      await developCommand(
+        { session: fixtureSession.sessionId },
+        { store, io: devIo, client },
+      );
+    } finally {
+      RalphLoop.prototype.run = originalRun;
+    }
+
+    expect(process.exitCode).toBe(1);
+  });
+});

package/tests/e2e/developPty.spec.ts

@@ -0,0 +1,86 @@
+/**
+ * T049-T051: PTY-based interactive E2E tests for `sofia dev`.
+ *
+ * Validates Ctrl+C handling, progress output, and clean exit behavior.
+ * Gracefully skips if node-pty allocation fails (e.g., CI without TTY).
+ */
+import { describe, it, expect } from 'vitest';
+
+// ── PTY availability guard (T051) ────────────────────────────────────────────
+
+let pty: typeof import('node-pty') | undefined;
+let ptyAvailable = false;
+
+try {
+  pty = await import('node-pty');
+  // Attempt a minimal allocation to verify PTY works
+  const testProc = pty.spawn('echo', ['test'], { cols: 80, rows: 24 });
+  testProc.kill();
+  ptyAvailable = true;
+} catch {
+  ptyAvailable = false;
+}
+
+describe('PTY-based E2E: sofia dev', () => {
+  // T051: Skip gracefully if node-pty allocation fails
+  const itPty = ptyAvailable ? it : it.skip;
+
+  itPty('help output appears in PTY buffer (T050)', async () => {
+    if (!pty) return;
+
+    const proc = pty.spawn('npx', ['tsx', 'src/cli/index.ts', 'dev', '--help'], {
+      cols: 120,
+      rows: 40,
+      cwd: process.cwd(),
+      env: { ...process.env },
+    });
+
+    let output = '';
+    proc.onData((data: string) => {
+      output += data;
+    });
+
+    const exitCode = await new Promise<number>((resolve) => {
+      proc.onExit(({ exitCode: code }) => {
+        resolve(code);
+      });
+      setTimeout(() => {
+        proc.kill();
+        resolve(-1);
+      }, 15_000);
+    });
+
+    // --help should produce usage output containing 'dev'
+    expect(output).toContain('dev');
+    expect(exitCode).toBe(0);
+  }, 20_000);
+
+  itPty('Ctrl+C sends signal to running process (T049)', async () => {
+    if (!pty) return;
+
+    // Use a simple process that sleeps, then send Ctrl+C
+    const proc = pty.spawn('sleep', ['30'], {
+      cols: 80,
+      rows: 24,
+      cwd: process.cwd(),
+      env: { ...process.env },
+    });
+
+    // Wait briefly then send Ctrl+C
+    await new Promise((resolve) => setTimeout(resolve, 500));
+    proc.write('\x03'); // Ctrl+C
+
+    const exitCode = await new Promise<number>((resolve) => {
+      proc.onExit(({ exitCode: code }) => {
+        resolve(code);
+      });
+      setTimeout(() => {
+        proc.kill();
+        resolve(-999);
+      }, 5_000);
+    });
+
+    // Process should have been interrupted (not timed out)
+    expect(exitCode).not.toBe(-999);
+  }, 10_000);
+});

package/tests/e2e/discoveryWebSearchRelevance.spec.ts

@@ -0,0 +1,103 @@
+/**
+ * T041: Discovery web search enrichment relevance validation (SC-003-005).
+ *
+ * Validates that discovery web search enrichment retrieves keyword-relevant
+ * context for at least 3 out of 5 test company descriptions.
+ * Gated behind SOFIA_LIVE_MCP_TESTS=true because it requires real web search.
+ *
+ * Acceptance criteria:
+ * - Run enrichFromWebSearch() for 5 different company descriptions
+ * - At least 3/5 must return results with keyword-relevant content
+ * - "Keyword-relevant" = at least one result snippet contains a word from the company/industry
+ */
+import { describe, it, expect } from 'vitest';
+
+const LIVE = process.env.SOFIA_LIVE_MCP_TESTS === 'true';
+
+const TEST_COMPANIES = [
+  {
+    summary: '"Nestlé" is a global food and beverage company headquartered in Switzerland.',
+    keywords: ['food', 'beverage', 'switzerland', 'global', 'company'],
+  },
+  {
+    summary: '"Zara" is a global retail company headquartered in Spain.',
+    keywords: ['retail', 'fashion', 'spain', 'global', 'company'],
+  },
+  {
+    summary:
+      '"Microsoft Corporation" is a global technology company headquartered in Redmond, Washington.',
+    keywords: ['technology', 'software', 'hardware', 'cloud', 'global'],
+  },
+  {
+    summary:
+      '"Maersk" is a global shipping and logistics company headquartered in Copenhagen, Denmark.',
+    keywords: ['shipping', 'logistics', 'denmark', 'global', 'company'],
+  },
+  {
+    summary:
+      '"Hasbro" is a global toy and entertainment company headquartered in Pawtucket, Rhode Island.',
+    keywords: ['toy', 'entertainment', 'rhode island', 'global', 'company'],
+  },
+];
+
+describe.skipIf(!LIVE)('Discovery web search relevance validation (T041 / SC-003-005)', () => {
+  it('at least 3/5 company descriptions return keyword-relevant results', async () => {
+    const { DiscoveryEnricher } = await import('../../src/phases/discoveryEnricher.js');
+    const { createWebSearchTool } = await import('../../src/mcp/webSearch.js');
+
+    const webSearchFn = createWebSearchTool({
+      projectEndpoint: process.env.FOUNDRY_PROJECT_ENDPOINT!,
+      modelDeploymentName: process.env.FOUNDRY_MODEL_DEPLOYMENT_NAME!,
+    });
+
+    const webSearchClient = {
+      search: async (query: string) => webSearchFn(query),
+    };
+
+    const enricher = new DiscoveryEnricher();
+    let relevantCount = 0;
+
+    const results: Array<{ company: string; relevant: boolean; snippetCount: number }> = [];
+
+    for (const company of TEST_COMPANIES) {
+      const enrichment = await enricher.enrichFromWebSearch(company.summary, webSearchClient);
+
+      // Collect all result strings from enrichment
+      const allText = [
+        ...(enrichment.companyNews ?? []),
+        ...(enrichment.competitorInfo ?? []),
+        ...(enrichment.industryTrends ?? []),
+      ]
+        .join(' ')
+        .toLowerCase();
+
+      const snippetCount =
+        (enrichment.companyNews?.length ?? 0) +
+        (enrichment.competitorInfo?.length ?? 0) +
+        (enrichment.industryTrends?.length ?? 0);
+
+      // Check if any keyword appears in the results
+      const hasRelevantKeyword = company.keywords.some((kw) => allText.includes(kw.toLowerCase()));
+
+      if (hasRelevantKeyword && snippetCount > 0) {
+        relevantCount++;
+      }
+
+      results.push({
+        company: company.summary.split('"')[1] || company.summary.slice(0, 30),
+        relevant: hasRelevantKeyword && snippetCount > 0,
+        snippetCount,
+      });
+    }
+
+    // Log outcomes for manual review
+    console.log('=== T041 Web Search Relevance Validation ===');
+    for (const r of results) {
+      console.log(`  ${r.relevant ? '✓' : '✗'} ${r.company}: ${r.snippetCount} snippets`);
+    }
+    console.log(`Result: ${relevantCount}/5 companies have relevant results`);
+
+    // Acceptance: at least 3 out of 5
+    expect(relevantCount).toBeGreaterThanOrEqual(3);
+  }, 120_000); // 2 minute timeout for multiple web searches
+});

package/tests/e2e/harness.spec.ts

@@ -0,0 +1,104 @@
+/**
+ * E2E test harness skeleton.
+ *
+ * Uses node-pty to drive the sofIA CLI interactively, simulating
+ * user input and verifying streaming output. This is a skeleton —
+ * actual E2E test scenarios will be added in US1 (T021).
+ *
+ * Requirements:
+ * - node-pty must be installed (`npm install node-pty`)
+ * - Tests run under the `test:e2e` npm script
+ */
+import { describe, it, expect } from 'vitest';
+import { spawn } from 'node:child_process';
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROJECT_ROOT = join(__dirname, '..', '..');
+const CLI_ENTRY = join(PROJECT_ROOT, 'src', 'cli', 'index.ts');
+
+// ── Helpers ─────────────────────────────────────────────────────────────────
+
+interface CliResult {
+  stdout: string;
+  stderr: string;
+  exitCode: number | null;
+}
+
+/**
+ * Run the sofIA CLI with given arguments and return output.
+ * Uses tsx to run TypeScript directly.
+ */
+function runCli(args: string[], timeoutMs = 10000): Promise<CliResult> {
+  return new Promise((resolve, reject) => {
+    const child = spawn('npx', ['tsx', CLI_ENTRY, ...args], {
+      cwd: PROJECT_ROOT,
+      env: { ...process.env, NODE_ENV: 'test' },
+      stdio: ['pipe', 'pipe', 'pipe'],
+    });
+
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+
+    child.stdout.on('data', (chunk) => stdout.push(chunk));
+    child.stderr.on('data', (chunk) => stderr.push(chunk));
+
+    const timer = setTimeout(() => {
+      child.kill('SIGTERM');
+      reject(new Error(`CLI timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      resolve({
+        stdout: Buffer.concat(stdout).toString('utf-8'),
+        stderr: Buffer.concat(stderr).toString('utf-8'),
+        exitCode: code,
+      });
+    });
+
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      reject(err);
+    });
+  });
+}
+
+// ── Tests ────────────────────────────────────────────────────────────────────
+
+describe('E2E Harness', () => {
+  it('displays help when invoked with --help', async () => {
+    const result = await runCli(['--help']);
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain('sofIA');
+    expect(result.stdout).toContain('workshop');
+    expect(result.stdout).toContain('status');
+    expect(result.stdout).toContain('export');
+  }, 15_000);
+
+  it('displays version when invoked with --version', async () => {
+    const result = await runCli(['--version']);
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout.trim()).toMatch(/^\d+\.\d+\.\d+$/);
+  }, 15_000);
+
+  it('shows workshop help', async () => {
+    const result = await runCli(['workshop', '--help']);
+    expect(result.exitCode).toBe(0);
+    expect(result.stdout).toContain('workshop');
+  }, 15_000);
+
+  it('lists sessions or reports none when status invoked without session', async () => {
+    const result = await runCli(['status', '--json']);
+    // Either lists sessions or reports no sessions found — both valid
+    const parsed = JSON.parse(result.stdout);
+    expect(parsed).toBeDefined();
+    expect('sessions' in parsed || 'error' in parsed).toBe(true);
+  }, 15_000);
+
+  it('returns error for export without session', async () => {
+    const result = await runCli(['export', '--json']);
+    expect(result.stdout).toContain('No session specified');
+  }, 15_000);
+});