popeye-cli 2.1.0 → 2.7.0

package/tests/pipeline/skills/coverage-gate.test.ts

@@ -0,0 +1,370 @@
+/**
+ * Skill Coverage Gate tests — assertSkillCoverage with conditional roles.
+ */
+
+import { describe, it, expect } from 'vitest';
+import {
+  assertSkillCoverage,
+  ROLE_REQUIRED_USAGE,
+  PHASE_ORDER,
+} from '../../../src/pipeline/skills/coverage-gate.js';
+import type { SkillUsageEvent } from '../../../src/pipeline/skills/usage-registry.js';
+import type { PipelineState, PipelineRole, PipelinePhase } from '../../../src/pipeline/types.js';
+import { createDefaultPipelineState } from '../../../src/pipeline/types.js';
+
+/** Helper to create a usage event */
+function makeEvent(
+  role: PipelineRole,
+  phase: string = 'ROLE_PLANNING',
+): SkillUsageEvent {
+  return {
+    role,
+    phase: phase as SkillUsageEvent['phase'],
+    used_as: 'system_prompt',
+    skill_source: 'defaults',
+    timestamp: new Date().toISOString(),
+  };
+}
+
+/** Helper to create pipeline state with overrides */
+function makePipeline(overrides: Partial<PipelineState> = {}): PipelineState {
+  return {
+    ...createDefaultPipelineState(),
+    ...overrides,
+  };
+}
+
+describe('assertSkillCoverage', () => {
+  describe('expected use — all active roles have usage', () => {
+    it('should pass when all active roles have recorded usage', () => {
+      const activeRoles: PipelineRole[] = [
+        'DISPATCHER', 'ARCHITECT', 'DB_EXPERT', 'BACKEND_PROGRAMMER',
+      ];
+      const events: SkillUsageEvent[] = [
+        makeEvent('ARCHITECT', 'ARCHITECTURE'),
+        makeEvent('DB_EXPERT', 'ROLE_PLANNING'),
+        makeEvent('BACKEND_PROGRAMMER', 'ROLE_PLANNING'),
+      ];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, events, pipeline);
+
+      expect(result.pass).toBe(true);
+      expect(result.missing).toHaveLength(0);
+      expect(result.covered).toContain('DISPATCHER');
+      expect(result.covered).toContain('ARCHITECT');
+    });
+  });
+
+  describe('fail case — active role missing usage', () => {
+    it('should fail when active UI_UX_SPECIALIST has no usage events', () => {
+      const activeRoles: PipelineRole[] = [
+        'DISPATCHER', 'ARCHITECT', 'UI_UX_SPECIALIST',
+      ];
+      const events: SkillUsageEvent[] = [
+        makeEvent('ARCHITECT', 'ARCHITECTURE'),
+        // UI_UX_SPECIALIST has no events
+      ];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, events, pipeline);
+
+      expect(result.pass).toBe(false);
+      expect(result.missing).toHaveLength(1);
+      expect(result.missing[0].role).toBe('UI_UX_SPECIALIST');
+      expect(result.missing[0].reason).toContain('No skill usage recorded');
+    });
+  });
+
+  describe('DISPATCHER — meta-only role always passes', () => {
+    it('should mark DISPATCHER as covered without any events', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER'],
+        [],
+        makePipeline({ activeRoles: ['DISPATCHER'] }),
+      );
+
+      expect(result.pass).toBe(true);
+      expect(result.covered).toContain('DISPATCHER');
+    });
+  });
+
+  describe('DEBUGGER — conditional on recovery', () => {
+    it('should pass when recoveryCount=0 (not required)', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'DEBUGGER'],
+        [],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'DEBUGGER'],
+          recoveryCount: 0,
+        }),
+      );
+
+      expect(result.pass).toBe(true);
+      expect(result.covered).toContain('DEBUGGER');
+    });
+
+    it('should fail when recoveryCount>0 and no usage events', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'DEBUGGER'],
+        [],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'DEBUGGER'],
+          recoveryCount: 1,
+        }),
+      );
+
+      expect(result.pass).toBe(false);
+      expect(result.missing).toHaveLength(1);
+      expect(result.missing[0].role).toBe('DEBUGGER');
+    });
+
+    it('should pass when recoveryCount>0 and usage recorded', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'DEBUGGER'],
+        [makeEvent('DEBUGGER', 'RECOVERY_LOOP')],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'DEBUGGER'],
+          recoveryCount: 2,
+        }),
+      );
+
+      expect(result.pass).toBe(true);
+      expect(result.covered).toContain('DEBUGGER');
+    });
+  });
+
+  describe('ARBITRATOR — conditional on arbitration', () => {
+    it('should pass when no arbitration artifacts exist', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'ARBITRATOR'],
+        [],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'ARBITRATOR'],
+          artifacts: [],
+        }),
+      );
+
+      expect(result.pass).toBe(true);
+      expect(result.covered).toContain('ARBITRATOR');
+    });
+
+    it('should fail when arbitration artifacts exist but no usage', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'ARBITRATOR'],
+        [],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'ARBITRATOR'],
+          artifacts: [{
+            id: 'arb-1',
+            type: 'arbitration',
+            phase: 'CONSENSUS_MASTER_PLAN',
+            version: 1,
+            path: 'docs/arbitration.json',
+            sha256: 'abc',
+            timestamp: new Date().toISOString(),
+            immutable: true,
+            content_type: 'json',
+            group_id: 'g1',
+          }],
+        }),
+      );
+
+      expect(result.pass).toBe(false);
+      expect(result.missing[0].role).toBe('ARBITRATOR');
+    });
+  });
+
+  describe('JOURNALIST — conditional on journal triggers', () => {
+    it('should pass when no journal-triggering phases completed', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'JOURNALIST'],
+        [],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'JOURNALIST'],
+          gateResults: {},
+        }),
+      );
+
+      expect(result.pass).toBe(true);
+    });
+
+    it('should fail when journal phase completed but no usage', () => {
+      const result = assertSkillCoverage(
+        ['DISPATCHER', 'JOURNALIST'],
+        [],
+        makePipeline({
+          activeRoles: ['DISPATCHER', 'JOURNALIST'],
+          gateResults: {
+            CONSENSUS_MASTER_PLAN: {
+              phase: 'CONSENSUS_MASTER_PLAN',
+              pass: true,
+              blockers: [],
+              missingArtifacts: [],
+              failedChecks: [],
+              timestamp: new Date().toISOString(),
+            },
+          },
+        }),
+      );
+
+      expect(result.pass).toBe(false);
+      expect(result.missing[0].role).toBe('JOURNALIST');
+    });
+  });
+
+  describe('ROLE_REQUIRED_USAGE configuration', () => {
+    it('should have entries for all 16 pipeline roles', () => {
+      const expectedRoles: PipelineRole[] = [
+        'DISPATCHER', 'ARCHITECT', 'DB_EXPERT', 'BACKEND_PROGRAMMER',
+        'FRONTEND_PROGRAMMER', 'WEBSITE_PROGRAMMER', 'UI_UX_SPECIALIST',
+        'MARKETING_EXPERT', 'SOCIAL_EXPERT', 'QA_TESTER', 'REVIEWER',
+        'ARBITRATOR', 'DEBUGGER', 'AUDITOR', 'JOURNALIST', 'RELEASE_MANAGER',
+      ];
+
+      for (const role of expectedRoles) {
+        expect(ROLE_REQUIRED_USAGE[role]).toBeDefined();
+      }
+    });
+  });
+
+  // ─── v2.4.5: Phase-aware deferral ────────────────────────
+
+  describe('phase-aware deferral (v2.4.5)', () => {
+    it('should defer AUDITOR at CONSENSUS_ROLE_PLANS', () => {
+      // AUDITOR requires AUDIT phase, which comes after CONSENSUS_ROLE_PLANS
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'AUDITOR'];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, [], pipeline, 'CONSENSUS_ROLE_PLANS');
+
+      expect(result.pass).toBe(true);
+      expect(result.deferred).toContain('AUDITOR');
+      expect(result.missing).toHaveLength(0);
+    });
+
+    it('should check AUDITOR at PRODUCTION_GATE (AUDIT <= PRODUCTION_GATE)', () => {
+      // AUDIT comes before PRODUCTION_GATE in PHASE_ORDER, so AUDITOR is checked
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'AUDITOR'];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, [], pipeline, 'PRODUCTION_GATE');
+
+      expect(result.pass).toBe(false);
+      expect(result.missing).toHaveLength(1);
+      expect(result.missing[0].role).toBe('AUDITOR');
+      expect(result.deferred).not.toContain('AUDITOR');
+    });
+
+    it('should pass AUDITOR at PRODUCTION_GATE with events', () => {
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'AUDITOR'];
+      const events = [makeEvent('AUDITOR', 'AUDIT')];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, events, pipeline, 'PRODUCTION_GATE');
+
+      expect(result.pass).toBe(true);
+      expect(result.covered).toContain('AUDITOR');
+      expect(result.deferred).toHaveLength(0);
+    });
+
+    it('should defer RELEASE_MANAGER at PRODUCTION_GATE (DONE > PRODUCTION_GATE)', () => {
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'RELEASE_MANAGER'];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, [], pipeline, 'PRODUCTION_GATE');
+
+      expect(result.pass).toBe(true);
+      expect(result.deferred).toContain('RELEASE_MANAGER');
+    });
+
+    it('should check QA_TESTER at CONSENSUS_ROLE_PLANS (ROLE_PLANNING <= CONSENSUS_ROLE_PLANS)', () => {
+      // QA_TESTER has phases [ROLE_PLANNING, QA_VALIDATION].
+      // ROLE_PLANNING comes before CONSENSUS_ROLE_PLANS, so QA_TESTER is checked.
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'QA_TESTER'];
+      const events = [makeEvent('QA_TESTER', 'ROLE_PLANNING')];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(activeRoles, events, pipeline, 'CONSENSUS_ROLE_PLANS');
+
+      expect(result.pass).toBe(true);
+      expect(result.covered).toContain('QA_TESTER');
+      expect(result.deferred).not.toContain('QA_TESTER');
+    });
+
+    it('should pass full production scenario at CONSENSUS_ROLE_PLANS', () => {
+      // Simulate real pipeline: all SUPPORT_ROLES + website impl roles active.
+      // Provide events only for roles through ROLE_PLANNING phase.
+      const activeRoles: PipelineRole[] = [
+        'DISPATCHER', 'ARCHITECT', 'QA_TESTER', 'REVIEWER', 'ARBITRATOR',
+        'DEBUGGER', 'AUDITOR', 'JOURNALIST', 'RELEASE_MANAGER',
+        'DB_EXPERT', 'BACKEND_PROGRAMMER', 'WEBSITE_PROGRAMMER',
+      ];
+      const events: SkillUsageEvent[] = [
+        makeEvent('ARCHITECT', 'ARCHITECTURE'),
+        makeEvent('DB_EXPERT', 'ROLE_PLANNING'),
+        makeEvent('BACKEND_PROGRAMMER', 'ROLE_PLANNING'),
+        makeEvent('WEBSITE_PROGRAMMER', 'ROLE_PLANNING'),
+        makeEvent('QA_TESTER', 'ROLE_PLANNING'),
+      ];
+      const pipeline = makePipeline({
+        activeRoles,
+        recoveryCount: 0,
+        artifacts: [],
+        gateResults: {},
+      });
+
+      const result = assertSkillCoverage(activeRoles, events, pipeline, 'CONSENSUS_ROLE_PLANS');
+
+      expect(result.pass).toBe(true);
+      // AUDITOR and RELEASE_MANAGER should be deferred
+      expect(result.deferred).toContain('AUDITOR');
+      expect(result.deferred).toContain('RELEASE_MANAGER');
+      // REVIEWER should also be deferred (REVIEW phase > CONSENSUS_ROLE_PLANS)
+      expect(result.deferred).toContain('REVIEWER');
+      // Conditional roles with no trigger are covered (not deferred)
+      expect(result.covered).toContain('ARBITRATOR');
+      expect(result.covered).toContain('DEBUGGER');
+      expect(result.covered).toContain('JOURNALIST');
+    });
+
+    it('should fall back to strict mode with unknown currentPhase', () => {
+      // An unknown phase not in PHASE_ORDER gets indexOf = -1, same as strict mode
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'AUDITOR'];
+      const pipeline = makePipeline({ activeRoles });
+
+      const result = assertSkillCoverage(
+        activeRoles, [], pipeline, 'SOME_NEW_PHASE' as PipelinePhase,
+      );
+
+      // Strict mode: AUDITOR is checked (not deferred) and fails
+      expect(result.pass).toBe(false);
+      expect(result.missing).toHaveLength(1);
+      expect(result.missing[0].role).toBe('AUDITOR');
+      expect(result.deferred).toHaveLength(0);
+    });
+
+    it('should use strict mode when currentPhase is omitted (backward compat)', () => {
+      const activeRoles: PipelineRole[] = ['DISPATCHER', 'AUDITOR'];
+      const pipeline = makePipeline({ activeRoles });
+
+      // No currentPhase param — same as original behavior
+      const result = assertSkillCoverage(activeRoles, [], pipeline);
+
+      expect(result.pass).toBe(false);
+      expect(result.missing).toHaveLength(1);
+      expect(result.missing[0].role).toBe('AUDITOR');
+      expect(result.deferred).toHaveLength(0);
+    });
+
+    it('PHASE_ORDER should include all key phases', () => {
+      const keyPhases: PipelinePhase[] = [
+        'INTAKE', 'CONSENSUS_ROLE_PLANS', 'PRODUCTION_GATE', 'DONE', 'STUCK',
+        'AUDIT', 'REVIEW', 'ARCHITECTURE', 'ROLE_PLANNING',
+      ];
+      for (const phase of keyPhases) {
+        expect(PHASE_ORDER).toContain(phase);
+      }
+    });
+  });
+});

package/tests/pipeline/skills/generator.test.ts

@@ -0,0 +1,213 @@
+/**
+ * Skill generator tests — prompt building, parsing, rendering, skip logic.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdirSync, rmSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import {
+  shouldGenerateSkill,
+  buildSkillGenPrompt,
+  parseSkillPrompts,
+  renderSkillMarkdown,
+  writeGenerationMarker,
+} from '../../../src/pipeline/skills/generator.js';
+import type { SkillGenerationContext } from '../../../src/pipeline/skills/types.js';
+import type { RepoSnapshot } from '../../../src/pipeline/types.js';
+
+const TEST_DIR = join(process.cwd(), '.test-skill-generator');
+const SKILLS_DIR = join(TEST_DIR, 'skills');
+
+function makeSnapshot(): RepoSnapshot {
+  return {
+    snapshot_id: 'test-snap',
+    timestamp: new Date().toISOString(),
+    tree_summary: '',
+    config_files: [],
+    languages_detected: [],
+    scripts: {},
+    env_files: [],
+    migrations_present: false,
+    ports_entrypoints: [],
+    total_files: 0,
+    total_lines: 0,
+  };
+}
+
+function makeContext(overrides: Partial<SkillGenerationContext> = {}): SkillGenerationContext {
+  return {
+    language: 'python',
+    expandedSpec: 'Build a REST API for task management',
+    snapshot: makeSnapshot(),
+    activeRoles: ['DISPATCHER', 'ARCHITECT', 'BACKEND_PROGRAMMER', 'DB_EXPERT'],
+    skillsDir: SKILLS_DIR,
+    projectName: 'TestProject',
+    ...overrides,
+  };
+}
+
+describe('generator', () => {
+  beforeEach(() => {
+    if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true });
+    mkdirSync(SKILLS_DIR, { recursive: true });
+  });
+
+  afterEach(() => {
+    if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true });
+  });
+
+  describe('shouldGenerateSkill', () => {
+    it('should return true when no .md file exists', () => {
+      expect(shouldGenerateSkill(SKILLS_DIR, 'BACKEND_PROGRAMMER')).toBe(true);
+    });
+
+    it('should return false when .md file already exists', () => {
+      writeFileSync(join(SKILLS_DIR, 'BACKEND_PROGRAMMER.md'), 'existing content');
+      expect(shouldGenerateSkill(SKILLS_DIR, 'BACKEND_PROGRAMMER')).toBe(false);
+    });
+
+    it('should check per-role independently', () => {
+      writeFileSync(join(SKILLS_DIR, 'ARCHITECT.md'), 'existing');
+      expect(shouldGenerateSkill(SKILLS_DIR, 'ARCHITECT')).toBe(false);
+      expect(shouldGenerateSkill(SKILLS_DIR, 'BACKEND_PROGRAMMER')).toBe(true);
+    });
+  });
+
+  describe('buildSkillGenPrompt', () => {
+    it('should include project name and tech stack', () => {
+      const prompt = buildSkillGenPrompt(
+        makeContext(),
+        ['BACKEND_PROGRAMMER'],
+        { backend: 'FastAPI', language: 'Python 3.11+' },
+      );
+      expect(prompt).toContain('TestProject');
+      expect(prompt).toContain('FastAPI');
+      expect(prompt).toContain('Python 3.11+');
+    });
+
+    it('should include role descriptions', () => {
+      const prompt = buildSkillGenPrompt(
+        makeContext(),
+        ['BACKEND_PROGRAMMER', 'DB_EXPERT'],
+        { backend: 'FastAPI' },
+      );
+      expect(prompt).toContain('BACKEND_PROGRAMMER');
+      expect(prompt).toContain('DB_EXPERT');
+    });
+
+    it('should include session guidance when present', () => {
+      const prompt = buildSkillGenPrompt(
+        makeContext({ sessionGuidance: 'Focus on security' }),
+        ['BACKEND_PROGRAMMER'],
+        { backend: 'FastAPI' },
+      );
+      expect(prompt).toContain('Focus on security');
+    });
+
+    it('should include expanded spec', () => {
+      const prompt = buildSkillGenPrompt(
+        makeContext({ expandedSpec: 'Build a REST API with auth' }),
+        ['BACKEND_PROGRAMMER'],
+        { backend: 'FastAPI' },
+      );
+      expect(prompt).toContain('Build a REST API with auth');
+    });
+  });
+
+  describe('parseSkillPrompts', () => {
+    it('should parse valid JSON response', () => {
+      const response = JSON.stringify({
+        BACKEND_PROGRAMMER: 'You are the Backend Programmer for MyProject.',
+        DB_EXPERT: 'You are the DB Expert for MyProject.',
+      });
+      const result = parseSkillPrompts(response, ['BACKEND_PROGRAMMER', 'DB_EXPERT']);
+      expect(result.BACKEND_PROGRAMMER).toContain('Backend Programmer');
+      expect(result.DB_EXPERT).toContain('DB Expert');
+    });
+
+    it('should extract JSON from markdown code fences', () => {
+      const response = '```json\n{"BACKEND_PROGRAMMER": "You are the Backend Programmer."}\n```';
+      const result = parseSkillPrompts(response, ['BACKEND_PROGRAMMER']);
+      expect(result.BACKEND_PROGRAMMER).toContain('Backend Programmer');
+    });
+
+    it('should return empty for malformed JSON', () => {
+      const result = parseSkillPrompts('not json at all', ['BACKEND_PROGRAMMER']);
+      expect(Object.keys(result)).toHaveLength(0);
+    });
+
+    it('should skip roles not in expectedRoles', () => {
+      const response = JSON.stringify({
+        BACKEND_PROGRAMMER: 'Valid prompt for backend.',
+        FRONTEND_PROGRAMMER: 'Should be ignored.',
+      });
+      const result = parseSkillPrompts(response, ['BACKEND_PROGRAMMER']);
+      expect(result.BACKEND_PROGRAMMER).toBeDefined();
+      expect(result.FRONTEND_PROGRAMMER).toBeUndefined();
+    });
+
+    it('should skip prompts shorter than 10 chars', () => {
+      const response = JSON.stringify({
+        BACKEND_PROGRAMMER: 'Short',
+      });
+      const result = parseSkillPrompts(response, ['BACKEND_PROGRAMMER']);
+      expect(result.BACKEND_PROGRAMMER).toBeUndefined();
+    });
+  });
+
+  describe('renderSkillMarkdown', () => {
+    it('should produce valid YAML frontmatter format', () => {
+      const md = renderSkillMarkdown(
+        'BACKEND_PROGRAMMER',
+        'You are the Backend Programmer.',
+        ['follow_architecture', 'must_follow_master_plan'],
+        ['endpoints', 'services'],
+        ['ARCHITECT'],
+      );
+
+      expect(md).toMatch(/^---\n/);
+      expect(md).toContain('role: BACKEND_PROGRAMMER');
+      expect(md).toContain('version: 1.0-project');
+      expect(md).toContain('  - endpoints');
+      expect(md).toContain('  - services');
+      expect(md).toContain('  - follow_architecture');
+      expect(md).toContain('  - must_follow_master_plan');
+      expect(md).toContain('depends_on:');
+      expect(md).toContain('  - ARCHITECT');
+      expect(md).toContain('You are the Backend Programmer.');
+    });
+
+    it('should omit depends_on when empty', () => {
+      const md = renderSkillMarkdown(
+        'DISPATCHER',
+        'You are the Dispatcher.',
+        ['governance'],
+        ['phase_transition'],
+        [],
+      );
+      expect(md).not.toContain('depends_on:');
+    });
+  });
+
+  describe('writeGenerationMarker', () => {
+    it('should write valid JSON marker file', () => {
+      const marker = {
+        timestamp: '2026-02-22T14:30:00Z',
+        pipelineVersion: '1.0',
+        activeRoles: ['DISPATCHER', 'BACKEND_PROGRAMMER'],
+        techStack: { backend: 'FastAPI' },
+        aiGenerated: true,
+      };
+      writeGenerationMarker(SKILLS_DIR, marker);
+
+      const markerPath = join(SKILLS_DIR, '.popeye-skills-generated.json');
+      expect(existsSync(markerPath)).toBe(true);
+
+      const content = JSON.parse(readFileSync(markerPath, 'utf-8'));
+      expect(content.pipelineVersion).toBe('1.0');
+      expect(content.aiGenerated).toBe(true);
+      expect(content.activeRoles).toContain('BACKEND_PROGRAMMER');
+      expect(content.techStack.backend).toBe('FastAPI');
+    });
+  });
+});