sofia-cli 0.1.1

package/tests/integration/ralphLoopPartial.spec.ts

@@ -0,0 +1,416 @@
+/**
+ * T024: Integration test for partial/failed outcomes.
+ *
+ * Tests:
+ * - max-iterations with some tests passing (partial status)
+ * - max-iterations with no tests passing (failed status)
+ * - LLM error mid-loop (error outcome on iteration, loop continues)
+ */
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { createRequire } from 'node:module';
+
+import { RalphLoop } from '../../src/develop/ralphLoop.js';
+import { PocScaffolder } from '../../src/develop/pocScaffolder.js';
+import { TestRunner } from '../../src/develop/testRunner.js';
+import type { WorkshopSession } from '../../src/shared/schemas/session.js';
+import type { LoopIO } from '../../src/loop/conversationLoop.js';
+import type { CopilotClient } from '../../src/shared/copilotClient.js';
+import type { TestResults } from '../../src/shared/schemas/session.js';
+
+// Mock npm install
+vi.mock('node:child_process', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('node:child_process')>();
+  return {
+    ...actual,
+    spawn: vi.fn((cmd: string, args: string[]) => {
+      if (cmd === 'npm' && args.includes('install')) {
+        return {
+          stdout: { on: vi.fn() },
+          stderr: { on: vi.fn() },
+          on: vi.fn((event: string, cb: (code: number) => void) => {
+            if (event === 'close') cb(0);
+          }),
+          kill: vi.fn(),
+          killed: false,
+        };
+      }
+      return actual.spawn(cmd, args);
+    }),
+  };
+});
+
+const require = createRequire(import.meta.url);
+const fixtureSession: WorkshopSession = require('../fixtures/completedSession.json') as WorkshopSession;
+
+function makeIo(): LoopIO {
+  return {
+    write: vi.fn(),
+    writeActivity: vi.fn(),
+    writeToolSummary: vi.fn(),
+    readInput: vi.fn().mockResolvedValue(null),
+    showDecisionGate: vi.fn(),
+    isJsonMode: false,
+    isTTY: false,
+  };
+}
+
+function makeFakeScaffolder(outputDir: string): PocScaffolder {
+  return {
+    scaffold: vi.fn().mockImplementation(async () => {
+      const { writeFile, mkdir } = await import('node:fs/promises');
+      await mkdir(join(outputDir, 'src'), { recursive: true });
+      await writeFile(join(outputDir, 'package.json'), JSON.stringify({
+        name: 'test-poc',
+        scripts: { test: 'vitest run' },
+        dependencies: {},
+        devDependencies: {},
+      }), 'utf-8');
+      await writeFile(join(outputDir, 'src', 'index.ts'), 'export function main() {}', 'utf-8');
+      return {
+        createdFiles: ['package.json', 'src/index.ts'],
+        skippedFiles: [],
+        context: {
+          projectName: 'test-poc',
+          ideaTitle: 'Test',
+          ideaDescription: 'Test',
+          techStack: { language: 'TypeScript', runtime: 'Node.js 20', testRunner: 'npm test' },
+          planSummary: 'Test',
+          sessionId: fixtureSession.sessionId,
+          outputDir,
+        },
+      };
+    }),
+    getTemplateFiles: () => [],
+  } as unknown as PocScaffolder;
+}
+
+function makeClient(): CopilotClient {
+  return {
+    createSession: vi.fn().mockResolvedValue({
+      send: vi.fn().mockReturnValue({
+        async *[Symbol.asyncIterator]() {
+          yield {
+            type: 'TextDelta',
+            text: '```typescript file=src/index.ts\nexport function main() { return 1; }\n```\n',
+            timestamp: '',
+          };
+        },
+      }),
+      getHistory: () => [],
+    }),
+  };
+}
+
+describe('RalphLoop integration — partial/failed outcomes', () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(join(tmpdir(), 'sofia-ralph-partial-'));
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+    vi.clearAllMocks();
+  });
+
+  it('sets finalStatus=partial when some tests pass at max-iterations', async () => {
+    const io = makeIo();
+    const client = makeClient();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+
+    const testRunner: TestRunner = {
+      run: vi.fn().mockResolvedValue({
+        passed: 1,
+        failed: 1,
+        skipped: 0,
+        total: 2,
+        durationMs: 400,
+        failures: [{ testName: 'suite > test B', message: 'fails always' }],
+        rawOutput: '',
+      } satisfies TestResults),
+    } as unknown as TestRunner;
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 2,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+
+    expect(result.terminationReason).toBe('max-iterations');
+    expect(result.finalStatus).toBe('partial');
+    expect(result.session.poc?.terminationReason).toBe('max-iterations');
+    expect(result.session.poc?.finalStatus).toBe('partial');
+  });
+
+  it('sets finalStatus=failed when no tests pass at max-iterations', async () => {
+    const io = makeIo();
+    const client = makeClient();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+
+    const testRunner: TestRunner = {
+      run: vi.fn().mockResolvedValue({
+        passed: 0,
+        failed: 2,
+        skipped: 0,
+        total: 2,
+        durationMs: 400,
+        failures: [
+          { testName: 'test A', message: 'always fails' },
+          { testName: 'test B', message: 'always fails too' },
+        ],
+        rawOutput: '',
+      } satisfies TestResults),
+    } as unknown as TestRunner;
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 2,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+
+    expect(result.terminationReason).toBe('max-iterations');
+    expect(result.finalStatus).toBe('failed');
+  });
+
+  it('records error iteration when LLM returns empty response, continues loop', async () => {
+    const io = makeIo();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+
+    let testCallCount = 0;
+    const testRunner: TestRunner = {
+      run: vi.fn().mockImplementation(async (): Promise<TestResults> => {
+        testCallCount++;
+        if (testCallCount >= 2) {
+          // After error iteration, tests pass
+          return {
+            passed: 1,
+            failed: 0,
+            skipped: 0,
+            total: 1,
+            durationMs: 300,
+            failures: [],
+            rawOutput: '',
+          };
+        }
+        return {
+          passed: 0,
+          failed: 1,
+          skipped: 0,
+          total: 1,
+          durationMs: 400,
+          failures: [{ testName: 'test A', message: 'fails' }],
+          rawOutput: '',
+        };
+      }),
+    } as unknown as TestRunner;
+
+    let llmCallCount = 0;
+    const client: CopilotClient = {
+      createSession: vi.fn().mockResolvedValue({
+        send: vi.fn().mockImplementation(() => {
+          llmCallCount++;
+          if (llmCallCount === 1) {
+            // First LLM call: returns empty response (simulating error)
+            return {
+              async *[Symbol.asyncIterator]() {
+                // Empty - no TextDelta events
+              },
+            };
+          }
+          // Subsequent calls: return a fix
+          return {
+            async *[Symbol.asyncIterator]() {
+              yield {
+                type: 'TextDelta',
+                text: '```typescript file=src/index.ts\nexport function main() { return 1; }\n```\n',
+                timestamp: '',
+              };
+            },
+          };
+        }),
+        getHistory: () => [],
+      }),
+    };
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 5,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+
+    // Loop should continue after LLM error and eventually succeed or hit max
+    expect(['success', 'failed', 'partial', 'max-iterations']).toContain(result.finalStatus);
+
+    // Check that error iterations are recorded
+    const poc = result.session.poc!;
+    const hasErrorIter = poc.iterations.some((i) => i.outcome === 'error');
+    // With empty LLM response, we should have an error iteration
+    expect(hasErrorIter).toBe(true);
+  });
+
+  it('records terminationReason in session poc state', async () => {
+    const io = makeIo();
+    const client = makeClient();
+    const scaffolder = makeFakeScaffolder(tmpDir);
+
+    const testRunner: TestRunner = {
+      run: vi.fn().mockResolvedValue({
+        passed: 0,
+        failed: 1,
+        skipped: 0,
+        total: 1,
+        durationMs: 400,
+        failures: [{ testName: 'test', message: 'fails' }],
+        rawOutput: '',
+      } satisfies TestResults),
+    } as unknown as TestRunner;
+
+    const ralph = new RalphLoop({
+      client,
+      io,
+      session: fixtureSession,
+      outputDir: tmpDir,
+      maxIterations: 2,
+      testRunner,
+      scaffolder,
+    });
+
+    const result = await ralph.run();
+
+    expect(result.session.poc?.terminationReason).toBeDefined();
+    expect(result.session.poc?.finalStatus).toBeDefined();
+    expect(result.session.poc?.iterations.length).toBeGreaterThan(0);
+  });
+});
+
+// ── Resume from interrupted session (T019) ────────────────────────────────
+
+describe('resume from interrupted session', () => {
+  let tmpDir: string;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(join(tmpdir(), 'ralph-resume-'));
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  it('resumes from session with 2 completed iterations and starts at iteration 3 (T019)', async () => {
+    const io = makeIo();
+
+    // Create a session with 2 prior iterations
+    const session: WorkshopSession = {
+      ...fixtureSession,
+      poc: {
+        repoSource: 'local',
+        repoPath: tmpDir,
+        iterations: [
+          {
+            iteration: 1,
+            startedAt: new Date().toISOString(),
+            endedAt: new Date().toISOString(),
+            outcome: 'scaffold',
+            filesChanged: ['package.json', 'src/index.ts'],
+          },
+          {
+            iteration: 2,
+            startedAt: new Date().toISOString(),
+            endedAt: new Date().toISOString(),
+            outcome: 'tests-failing',
+            filesChanged: ['src/index.ts'],
+            testResults: {
+              passed: 1,
+              failed: 1,
+              skipped: 0,
+              total: 2,
+              durationMs: 200,
+              failures: [{ testName: 'test1', message: 'Expected x' }],
+            },
+          },
+        ],
+      },
+    };
+
+    // Mock a client that returns passing code on first call
+    const passingClient: CopilotClient = {
+      createSession: vi.fn().mockResolvedValue({
+        send: vi.fn().mockReturnValue({
+          async *[Symbol.asyncIterator]() {
+            yield {
+              type: 'TextDelta',
+              text: '```typescript file=src/index.ts\nexport function main() { return "ok"; }\n```',
+              timestamp: '',
+            };
+          },
+        }),
+        getHistory: () => [],
+      }),
+    };
+
+    // TestRunner that passes on first call
+    const testRunner = {
+      run: vi.fn().mockResolvedValue({
+        passed: 2,
+        failed: 0,
+        skipped: 0,
+        total: 2,
+        durationMs: 100,
+        failures: [],
+      }),
+    } as unknown as TestRunner;
+
+    const scaffolder = makeFakeScaffolder(tmpDir);
+
+    const ralph = new RalphLoop({
+      client: passingClient,
+      io,
+      session,
+      outputDir: tmpDir,
+      maxIterations: 10,
+      testRunner,
+      scaffolder,
+      checkpoint: {
+        hasPriorRun: true,
+        completedIterations: 2,
+        lastIterationIncomplete: false,
+        resumeFromIteration: 3,
+        canSkipScaffold: false,
+        priorFinalStatus: undefined,
+        priorIterations: session.poc!.iterations,
+      },
+    });
+
+    const result = await ralph.run();
+
+    // Should have seeded from prior iterations and continued
+    // Note: finalStatus may be 'partial' since makeFakeScaffolder doesn't create all
+    // files required by validatePocOutput (README.md, tsconfig.json, etc.)
+    expect(['success', 'partial']).toContain(result.finalStatus);
+    // Total iterations should include the 2 prior + scaffold + tests-passing
+    expect(result.iterationsCompleted).toBeGreaterThanOrEqual(3);
+    // Session should have iterations from resume
+    expect(result.session.poc?.iterations.length).toBeGreaterThanOrEqual(3);
+  });
+});

package/tests/integration/resumeAndBacktrack.spec.ts

@@ -0,0 +1,278 @@
+/**
+ * Integration test: Resume and Backtrack flow (T036)
+ *
+ * Tests the resume, backtrack, and artifact invalidation flows:
+ * - Resume an existing session and continue from the current phase
+ * - Backtrack to an earlier phase with deterministic invalidation
+ * - Verify downstream data is cleared after backtrack
+ * - Verify handler detects cleaned state as incomplete
+ * - Re-run a phase and produce fresh results after backtrack
+ */
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+import { ConversationLoop } from '../../src/loop/conversationLoop.js';
+import type { LoopIO, DecisionGateResult } from '../../src/loop/conversationLoop.js';
+import { createFakeCopilotClient } from '../../src/shared/copilotClient.js';
+import type { WorkshopSession, PhaseValue } from '../../src/shared/schemas/session.js';
+import { SessionStore } from '../../src/sessions/sessionStore.js';
+import { createPhaseHandler } from '../../src/phases/phaseHandlers.js';
+import { backtrackSession } from '../../src/sessions/sessionManager.js';
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function createTestSession(overrides?: Partial<WorkshopSession>): WorkshopSession {
+  const now = new Date().toISOString();
+  return {
+    sessionId: 'test-resume-session',
+    schemaVersion: '1.0.0',
+    createdAt: now,
+    updatedAt: now,
+    phase: 'Discover',
+    status: 'Active',
+    participants: [],
+    artifacts: { generatedFiles: [] },
+    turns: [],
+    ...overrides,
+  };
+}
+
+function createScriptedIO(
+  inputs: (string | null)[],
+  decisionGateChoice: DecisionGateResult = { choice: 'continue' },
+): LoopIO & { output: string[]; activityLog: string[] } {
+  let inputIdx = 0;
+  const output: string[] = [];
+  const activityLog: string[] = [];
+
+  return {
+    write(text: string) {
+      output.push(text);
+    },
+    writeActivity(text: string) {
+      activityLog.push(text);
+    },
+    writeToolSummary(_toolName: string, _summary: string) {
+      // no-op
+    },
+    async readInput(_prompt?: string): Promise<string | null> {
+      if (inputIdx >= inputs.length) return null;
+      return inputs[inputIdx++];
+    },
+    async showDecisionGate(_phase: PhaseValue): Promise<DecisionGateResult> {
+      return decisionGateChoice;
+    },
+    isJsonMode: false,
+    isTTY: false,
+    output,
+    activityLog,
+  };
+}
+
+// ── Tests ────────────────────────────────────────────────────────────────────
+
+describe('Resume and Backtrack Flow', () => {
+  let tmpDir: string;
+  let store: SessionStore;
+
+  beforeEach(async () => {
+    tmpDir = await mkdtemp(join(tmpdir(), 'sofia-resume-'));
+    store = new SessionStore(tmpDir);
+  });
+
+  afterEach(async () => {
+    await rm(tmpDir, { recursive: true, force: true });
+  });
+
+  it('resumes an existing session from disk', async () => {
+    // Save a session mid-Ideate
+    const session = createTestSession({
+      phase: 'Ideate',
+      businessContext: {
+        businessDescription: 'Logistics company',
+        challenges: ['Slow deliveries'],
+      },
+      workflow: {
+        activities: [{ id: 'a1', name: 'Route Planning' }],
+        edges: [],
+      },
+    });
+    await store.save(session);
+
+    // Load it back (resume)
+    const loaded = await store.load(session.sessionId);
+    expect(loaded.phase).toBe('Ideate');
+    expect(loaded.businessContext!.businessDescription).toBe('Logistics company');
+  });
+
+  it('resumes and continues Ideate phase with ConversationLoop', async () => {
+    const session = createTestSession({
+      phase: 'Ideate',
+      businessContext: {
+        businessDescription: 'Logistics company',
+        challenges: ['Slow deliveries'],
+      },
+      workflow: {
+        activities: [{ id: 'a1', name: 'Route Planning' }],
+        edges: [],
+      },
+    });
+    await store.save(session);
+
+    const loaded = await store.load(session.sessionId);
+    const client = createFakeCopilotClient([
+      { role: 'assistant', content: 'Let me help you brainstorm ideas for optimizing routes.' },
+    ]);
+    const io = createScriptedIO(['Can we use AI for routing?', null]);
+    const handler = createPhaseHandler('Ideate');
+    await handler._preload();
+
+    const loop = new ConversationLoop({
+      client,
+      io,
+      session: loaded,
+      phaseHandler: handler,
+      onEvent: () => {},
+      onSessionUpdate: async (s) => { await store.save(s); },
+    });
+
+    const result = await loop.run();
+    expect(result.turns!.length).toBeGreaterThan(0);
+  });
+
+  it('backtracks from Design to Ideate and clears downstream data', async () => {
+    const session = createTestSession({
+      phase: 'Design',
+      businessContext: {
+        businessDescription: 'ACME Corp',
+        challenges: ['Cost reduction'],
+      },
+      workflow: {
+        activities: [{ id: 'a1', name: 'Procurement' }],
+        edges: [],
+      },
+      ideas: [
+        { id: 'i1', title: 'AI Procurement', description: 'Automated purchasing', workflowStepIds: ['a1'] },
+      ],
+      evaluation: {
+        method: 'feasibility-value-matrix',
+        ideas: [{ ideaId: 'i1', feasibility: 4, value: 5 }],
+      },
+      turns: [
+        { phase: 'Discover', sequence: 1, role: 'user', content: 'hello', timestamp: new Date().toISOString() },
+        { phase: 'Ideate', sequence: 2, role: 'user', content: 'ideas', timestamp: new Date().toISOString() },
+        { phase: 'Design', sequence: 3, role: 'user', content: 'evaluate', timestamp: new Date().toISOString() },
+      ],
+    });
+    await store.save(session);
+
+    const loaded = await store.load(session.sessionId);
+    const result = backtrackSession(loaded, 'Ideate');
+
+    expect(result.success).toBe(true);
+    expect(result.session.phase).toBe('Ideate');
+    expect(result.session.ideas).toBeUndefined();
+    expect(result.session.evaluation).toBeUndefined();
+    // Discover data preserved
+    expect(result.session.businessContext).toBeDefined();
+    // Only Discover turns remain
+    expect(result.session.turns?.length).toBe(1);
+    expect(result.session.turns![0].phase).toBe('Discover');
+
+    // Save backtracked session
+    await store.save(result.session);
+    const reloaded = await store.load(session.sessionId);
+    expect(reloaded.phase).toBe('Ideate');
+  });
+
+  it('re-runs Ideate after backtrack and produces fresh ideas', async () => {
+    const session = createTestSession({
+      phase: 'Ideate',
+      status: 'Active',
+      businessContext: {
+        businessDescription: 'ACME Corp',
+        challenges: ['Cost reduction'],
+      },
+    });
+
+    // LLM returns fresh ideas in JSON
+    const client = createFakeCopilotClient([
+      {
+        role: 'assistant',
+        content: '```json\n[{"id": "new-1", "title": "Smart Scheduling", "description": "AI scheduling", "workflowStepIds": ["a1"]}]\n```',
+      },
+    ]);
+    const io = createScriptedIO(['Generate new ideas', null]);
+    const handler = createPhaseHandler('Ideate');
+    await handler._preload();
+
+    const loop = new ConversationLoop({
+      client,
+      io,
+      session,
+      phaseHandler: handler,
+      onEvent: () => {},
+      onSessionUpdate: async (s) => { await store.save(s); },
+    });
+
+    const result = await loop.run();
+    // The extractResult should have captured the ideas
+    expect(result.ideas).toBeDefined();
+    expect(result.ideas!.length).toBeGreaterThan(0);
+    expect(result.ideas![0].id).toBe('new-1');
+  });
+
+  it('backtrack to Discover clears all downstream phase data', () => {
+    const session = createTestSession({
+      phase: 'Plan',
+      businessContext: {
+        businessDescription: 'Tech Co',
+        challenges: ['Scaling'],
+      },
+      workflow: {
+        activities: [{ id: 'a1', name: 'Deploy' }],
+        edges: [],
+      },
+      ideas: [
+        { id: 'i1', title: 'Auto-scale', description: 'Auto scaling', workflowStepIds: ['a1'] },
+      ],
+      evaluation: {
+        method: 'feasibility-value-matrix',
+        ideas: [{ ideaId: 'i1', feasibility: 5, value: 5 }],
+      },
+      selection: {
+        ideaId: 'i1',
+        selectionRationale: 'Only option',
+        confirmedByUser: true,
+      },
+      plan: {
+        milestones: [{ id: 'm1', title: 'Phase 1', items: ['Setup'] }],
+      },
+    });
+
+    const result = backtrackSession(session, 'Discover');
+    expect(result.success).toBe(true);
+    expect(result.invalidatedPhases).toContain('Discover');
+    expect(result.invalidatedPhases).toContain('Ideate');
+    expect(result.invalidatedPhases).toContain('Design');
+    expect(result.invalidatedPhases).toContain('Select');
+    expect(result.invalidatedPhases).toContain('Plan');
+
+    const s = result.session;
+    expect(s.businessContext).toBeUndefined();
+    expect(s.workflow).toBeUndefined();
+    expect(s.ideas).toBeUndefined();
+    expect(s.evaluation).toBeUndefined();
+    expect(s.selection).toBeUndefined();
+    expect(s.plan).toBeUndefined();
+  });
+
+  it('forward backtrack is rejected', () => {
+    const session = createTestSession({ phase: 'Ideate' });
+    const result = backtrackSession(session, 'Plan');
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('forward');
+  });
+});