popeye-cli 1.10.0 → 2.0.0

package/skills/POPEYE_FULL_AUTONOMY_PIPELINE.md

@@ -1,6 +1,6 @@
 # Popeye Full Autonomy Pipeline Spec
-Version: 1.0
-Goal: 1 idea → 1 prompt → Popeye → production-ready system (PASS Production Gate)
+Version: 1.1 (Autonomy Hardening Gap Fixes)
+Goal: 1 idea -> 1 prompt -> Popeye -> production-ready system (PASS Production Gate)
 
 This spec defines the end-to-end autonomous workflow Popeye must execute, including:
 - Phase sequencing (no skipping)
@@ -90,6 +90,27 @@ Artifacts are never overwritten. New versions create new files.
 - Each phase may only transition to the next if all required artifacts exist and the gate passes.
 - Any failure routes to RECOVERY_LOOP, except missing inputs which route back to the phase that needs them.
 
+### v1.1 Gate Behaviors (Autonomy Hardening)
+
+Between phase execution and transition, the orchestrator now applies three additional checks:
+
+1. **Constitution Verification**: Before every `evaluateGate()` call, verifies that `skills/POPEYE_CONSTITUTION.md` has not been modified since pipeline start (SHA-256 hash comparison). If the constitution is invalid, the gate is blocked.
+
+2. **Gate Result Merge**: After gate evaluation, preserves `score`/`consensusScore` values stored by consensus phase handlers while updating `pass`/`blockers` from the gate engine. Prevents overwriting of consensus scores.
+
+3. **Change Request (CR) Routing**: After REVIEW and AUDIT phases pass their gates, checks `pipeline.pendingChangeRequests` for any `proposed` CRs. If found, deterministically routes to the appropriate consensus phase (based on change type) rather than continuing normal progression. CR lifecycle: `proposed` -> `approved` (when routed) or `rejected`.
+
+### v1.1 Pipeline State Additions
+
+```
+pendingChangeRequests?: Array<{
+  cr_id: string;
+  change_type: 'scope' | 'architecture' | 'dependency' | 'config' | 'requirement';
+  target_phase: PipelinePhase;
+  status: 'proposed' | 'approved' | 'rejected';
+}>;
+```
+
 ---
 
 ## 3) Phase-by-Phase Specification
@@ -321,15 +342,24 @@ Artifacts are never overwritten. New versions create new files.
   - evidence references
   - no shortcuts
   - integration wiring
+- Generate fresh repo snapshot and diff against baseline (role-plan-approval snapshot)
+- Detect implementation drift (config changes, significant line deltas)
 
 **Outputs**
-- Review decision doc → `/docs/consensus/review_<id>_<date>.md`
+- Review decision doc -> `/docs/consensus/review_<id>_<date>.md`
 - If rejected: structured blocking list
+- v1.1: Change Request artifacts for detected drift (stored as `change_request` artifacts)
+- v1.1: Pending CRs registered in `pipeline.pendingChangeRequests` for orchestrator routing
 
 **Gate conditions (PASS)**
 - Reviewer approves with evidence
 - No Constitution violations
 
+**v1.1 Post-Gate Behavior**
+- After REVIEW gate passes, orchestrator checks `pendingChangeRequests` for proposed CRs
+- If CRs exist, routes to the appropriate consensus phase (e.g., CONSENSUS_MASTER_PLAN for scope drift)
+- If no CRs, continues to AUDIT normally
+
 ---
 
 ### Phase 10 — AUDIT
@@ -339,7 +369,7 @@ Artifacts are never overwritten. New versions create new files.
 - Everything above + repo snapshot + logs
 
 **Actions (Auditor)**
-- Integration audit (FE↔BE, BE↔DB)
+- Integration audit (FE<->BE, BE<->DB)
 - Config/env audit
 - Tests/coverage audit
 - Migration audit
@@ -347,8 +377,10 @@ Artifacts are never overwritten. New versions create new files.
 - Deployment readiness audit
 
 **Outputs (required)**
-- Audit Report → `/docs/audit/audit_<id>_<date>.md`
-- Audit Report schema (optional json)
+- Audit Report -> `/docs/audit/audit_<id>_<date>.md`
+- Audit Report schema (JSON)
+- v1.1: Change Request artifacts for blocking architectural/security findings
+- v1.1: Pending CRs registered in `pipeline.pendingChangeRequests` for orchestrator routing
 
 **Gate conditions (PASS)**
 - No P0/P1 findings open
@@ -356,6 +388,13 @@ Artifacts are never overwritten. New versions create new files.
 - No hardcoded secrets
 - End-to-end wiring verified
 
+**v1.1 Post-Gate Behavior**
+- After AUDIT gate passes, orchestrator checks `pendingChangeRequests` for proposed CRs
+- Architectural findings (integration/schema blocking issues) create CRs targeting CONSENSUS_ARCHITECTURE
+- Security findings create CRs targeting CONSENSUS_MASTER_PLAN
+- If CRs exist, routes to the appropriate consensus phase before PRODUCTION_GATE
+- If no CRs, continues to PRODUCTION_GATE normally
+
 **If FAIL**
 - Enter RECOVERY_LOOP
 
@@ -389,7 +428,7 @@ Artifacts are never overwritten. New versions create new files.
 
 ---
 
-## 4) Recovery Loop (FAIL → Debugger → Plan → Fix → Retest)
+## 4) Recovery Loop (FAIL -> Debugger -> Plan -> Fix -> Retest)
 
 ### Phase 12 — RECOVERY_LOOP (conditional)
 **Purpose**: Self-heal deterministically using RCA, not guesswork.
@@ -406,6 +445,7 @@ Artifacts are never overwritten. New versions create new files.
    - ownership (which role must fix)
    - recommended corrective actions
    - prevention suggestion
+   - v1.1: `requires_phase_rewind_to` field specifying which phase to rewind to
 
 2) **Dispatcher generates Recovery Plan**
    - targeted tasks mapped to responsible role(s)
@@ -416,19 +456,23 @@ Artifacts are never overwritten. New versions create new files.
 
 4) **Implement fixes** (by responsible role)
 
-5) **Retest only the failed gate first**, then proceed forward:
-   - If failed at Audit → rerun Audit
-   - If failed at Production Gate → rerun Production checks
-   - If failed tests → rerun tests + QA validation
+5) **Post-recovery routing** (v1.1 — RCA-driven rewind):
+   - Orchestrator reads latest RCA JSON artifact from disk
+   - If RCA specifies `requires_phase_rewind_to`, pipeline rewinds to that phase
+   - Otherwise, retests the originally failed phase, then proceeds forward:
+     - If failed at Audit -> rerun Audit
+     - If failed at Production Gate -> rerun Production checks
+     - If failed tests -> rerun tests + QA validation
+   - If no failed phase is tracked, defaults to QA_VALIDATION
 
 **Outputs (required)**
-- RCA report → `/docs/incidents/rca_<id>_<date>.md`
+- RCA report -> `/docs/incidents/rca_<id>_<date>.md` (both markdown and JSON, JSON includes `requires_phase_rewind_to`)
 - Recovery plan packet + consensus packet
 - Updated artifacts if scope/contracts changed
 
 **Stop conditions**
 - Max recovery iterations (default: 5)
-- If exceeded → STUCK state with “Stuck Report”
+- If exceeded -> STUCK state with “Stuck Report”
 
 ---
 
@@ -479,6 +523,15 @@ If recovery iterations exceed max, Popeye must stop and output:
 - Every plan claim must be evidenced against repo snapshot
 - Production readiness is binary, not “looks good”
 
+### v1.1 Additions
+
+- Constitution integrity is verified at every gate (SHA-256 hash comparison). A modified constitution blocks all gate progression until resolved.
+- Consensus scores from phase handlers are never overwritten by gate engine evaluation (gate result merge rule).
+- REVIEW and AUDIT phases must register detected issues as Change Requests in `pipeline.pendingChangeRequests`.
+- CRs are routed deterministically by the orchestrator (not advisory). The routing is a deterministic transition, not a suggestion.
+- RCA packets must include a `requires_phase_rewind_to` field when the root cause originates in a phase earlier than the one that failed. The orchestrator reads this from the JSON artifact on disk.
+- CRs are processed one at a time (first `proposed` CR is routed and marked `approved` before the next is considered).
+
 ---
 
 End of Spec.

package/src/pipeline/artifact-manager.ts

@@ -0,0 +1,339 @@
+/**
+ * Artifact Manager — manages immutable versioned artifacts under /docs/.
+ * Supports both Markdown and JSON content types (P0-C).
+ * Implements version chains via group_id + previous_id (P1-2).
+ */
+
+import { randomUUID } from 'node:crypto';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, writeFileSync, readFileSync, readdirSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+import type {
+  ArtifactType,
+  ArtifactEntry,
+  ArtifactRef,
+  ContentType,
+  PipelinePhase,
+} from './types.js';
+
+// ─── Constants ───────────────────────────────────────────
+
+/** Directory mappings: artifact type -> subdirectory under /docs/ */
+const ARTIFACT_DIRS: Record<string, string> = {
+  master_plan: 'master-plan',
+  architecture: 'architecture',
+  role_plan: 'role-plans',
+  consensus: 'consensus',
+  arbitration: 'arbitration',
+  audit_report: 'audit',
+  rca_report: 'incidents',
+  production_readiness: 'production',
+  release_notes: 'release',
+  deployment: 'release',
+  rollback: 'release',
+  repo_snapshot: 'snapshots',
+  build_check: 'checks',
+  test_check: 'checks',
+  lint_check: 'checks',
+  typecheck_check: 'checks',
+  placeholder_scan: 'checks',
+  qa_validation: 'role-plans',
+  review_decision: 'consensus',
+  stuck_report: 'incidents',
+  journalist_trace: 'journal',
+  resolved_commands: 'checks',
+  constitution: 'governance',
+  change_request: 'governance',
+};
+
+/** All required subdirectories under /docs/ */
+const DOCS_SUBDIRS = [
+  'master-plan',
+  'architecture',
+  'role-plans',
+  'consensus',
+  'arbitration',
+  'audit',
+  'incidents',
+  'production',
+  'release',
+  'snapshots',
+  'checks',
+  'journal',
+  'governance',
+];
+
+// ─── Helper Functions ────────────────────────────────────
+
+function computeSha256(content: string): string {
+  return createHash('sha256').update(content, 'utf-8').digest('hex');
+}
+
+function shortId(): string {
+  return randomUUID().split('-')[0];
+}
+
+function formatDate(): string {
+  return new Date().toISOString().split('T')[0];
+}
+
+function getExtension(contentType: ContentType): string {
+  return contentType === 'json' ? '.json' : '.md';
+}
+
+// ─── Artifact Manager ────────────────────────────────────
+
+export interface ArtifactManagerOptions {
+  projectDir: string;
+}
+
+export class ArtifactManager {
+  private readonly projectDir: string;
+  private readonly docsDir: string;
+
+  constructor(options: ArtifactManagerOptions) {
+    this.projectDir = options.projectDir;
+    this.docsDir = join(options.projectDir, 'docs');
+  }
+
+  /** Ensure all /docs/ subdirectories exist */
+  ensureDocsStructure(): void {
+    if (!existsSync(this.docsDir)) {
+      mkdirSync(this.docsDir, { recursive: true });
+    }
+    for (const subdir of DOCS_SUBDIRS) {
+      const dirPath = join(this.docsDir, subdir);
+      if (!existsSync(dirPath)) {
+        mkdirSync(dirPath, { recursive: true });
+      }
+    }
+  }
+
+  /** Create an artifact from Markdown content */
+  createArtifactText(
+    type: ArtifactType,
+    markdown: string,
+    phase: PipelinePhase,
+    groupId?: string,
+  ): ArtifactEntry {
+    return this.createArtifact(type, markdown, phase, 'markdown', groupId);
+  }
+
+  /** Create an artifact from a JSON-serializable object */
+  createArtifactJson(
+    type: ArtifactType,
+    jsonObject: unknown,
+    phase: PipelinePhase,
+    groupId?: string,
+  ): ArtifactEntry {
+    const content = JSON.stringify(jsonObject, null, 2);
+    return this.createArtifact(type, content, phase, 'json', groupId);
+  }
+
+  /** Core artifact creation logic */
+  private createArtifact(
+    type: ArtifactType,
+    content: string,
+    phase: PipelinePhase,
+    contentType: ContentType,
+    groupId?: string,
+  ): ArtifactEntry {
+    this.ensureDocsStructure();
+
+    const resolvedGroupId = groupId ?? randomUUID();
+    const existingArtifacts = this.listArtifacts(type);
+    const version = this.getNextVersion(resolvedGroupId, existingArtifacts);
+    const previousEntry = this.getLatestInGroup(resolvedGroupId, existingArtifacts);
+
+    const id = randomUUID();
+    const date = formatDate();
+    const sid = shortId();
+    const ext = getExtension(contentType);
+    const filename = `${type}_${sid}_v${version}_${date}${ext}`;
+
+    const subdir = ARTIFACT_DIRS[type] ?? 'misc';
+    const dirPath = join(this.docsDir, subdir);
+    if (!existsSync(dirPath)) {
+      mkdirSync(dirPath, { recursive: true });
+    }
+
+    const filePath = join(dirPath, filename);
+    const sha256 = computeSha256(content);
+
+    writeFileSync(filePath, content, 'utf-8');
+
+    const relativePath = relative(this.projectDir, filePath);
+
+    const entry: ArtifactEntry = {
+      id,
+      type,
+      phase,
+      version,
+      path: relativePath,
+      sha256,
+      timestamp: new Date().toISOString(),
+      immutable: true,
+      content_type: contentType,
+      group_id: resolvedGroupId,
+      previous_id: previousEntry?.id,
+    };
+
+    return entry;
+  }
+
+  /** Get the file path for a given artifact type and naming components */
+  getArtifactPath(
+    type: ArtifactType,
+    sid: string,
+    version: number,
+    date: string,
+    contentType: ContentType,
+  ): string {
+    const ext = getExtension(contentType);
+    const subdir = ARTIFACT_DIRS[type] ?? 'misc';
+    return join(this.docsDir, subdir, `${type}_${sid}_v${version}_${date}${ext}`);
+  }
+
+  /** List all artifacts, optionally filtered by type */
+  listArtifacts(type?: ArtifactType): ArtifactEntry[] {
+    // Scan for artifact JSON metadata files in a .artifacts/ dir
+    const metaDir = join(this.docsDir, '.artifacts');
+    if (!existsSync(metaDir)) {
+      return [];
+    }
+
+    const entries: ArtifactEntry[] = [];
+    const files = readdirSync(metaDir).filter((f) => f.endsWith('.json'));
+
+    for (const file of files) {
+      try {
+        const raw = readFileSync(join(metaDir, file), 'utf-8');
+        const parsed = JSON.parse(raw) as ArtifactEntry;
+        if (!type || parsed.type === type) {
+          entries.push(parsed);
+        }
+      } catch {
+        // Skip malformed metadata files
+      }
+    }
+
+    return entries.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+  }
+
+  /** Verify an artifact's SHA-256 matches its stored content */
+  verifyArtifact(entry: ArtifactEntry): boolean {
+    const fullPath = join(this.projectDir, entry.path);
+    if (!existsSync(fullPath)) {
+      return false;
+    }
+
+    const content = readFileSync(fullPath, 'utf-8');
+    const currentHash = computeSha256(content);
+    return currentHash === entry.sha256;
+  }
+
+  /** Get the latest artifact of a given type */
+  getLatestArtifact(type: ArtifactType): ArtifactEntry | null {
+    const all = this.listArtifacts(type);
+    if (all.length === 0) return null;
+    return all[all.length - 1];
+  }
+
+  /** Get next version number for a group across existing artifacts */
+  getNextVersion(groupId: string, existingArtifacts: ArtifactEntry[]): number {
+    const groupArtifacts = existingArtifacts.filter((a) => a.group_id === groupId);
+    if (groupArtifacts.length === 0) return 1;
+    const maxVersion = Math.max(...groupArtifacts.map((a) => a.version));
+    return maxVersion + 1;
+  }
+
+  /** Convert an ArtifactEntry to an ArtifactRef */
+  toArtifactRef(entry: ArtifactEntry): ArtifactRef {
+    return {
+      artifact_id: entry.id,
+      path: entry.path,
+      sha256: entry.sha256,
+      version: entry.version,
+      type: entry.type,
+    };
+  }
+
+  /** Store artifact metadata for later retrieval */
+  storeArtifactMetadata(entry: ArtifactEntry): void {
+    const metaDir = join(this.docsDir, '.artifacts');
+    if (!existsSync(metaDir)) {
+      mkdirSync(metaDir, { recursive: true });
+    }
+
+    const metaPath = join(metaDir, `${entry.id}.json`);
+    writeFileSync(metaPath, JSON.stringify(entry, null, 2), 'utf-8');
+  }
+
+  /** Create an artifact and store its metadata in one step */
+  createAndStoreText(
+    type: ArtifactType,
+    markdown: string,
+    phase: PipelinePhase,
+    groupId?: string,
+  ): ArtifactEntry {
+    const entry = this.createArtifactText(type, markdown, phase, groupId);
+    this.storeArtifactMetadata(entry);
+    return entry;
+  }
+
+  /** Create a JSON artifact and store its metadata in one step */
+  createAndStoreJson(
+    type: ArtifactType,
+    jsonObject: unknown,
+    phase: PipelinePhase,
+    groupId?: string,
+  ): ArtifactEntry {
+    const entry = this.createArtifactJson(type, jsonObject, phase, groupId);
+    this.storeArtifactMetadata(entry);
+    return entry;
+  }
+
+  /** Update /docs/INDEX.md with current artifact listing */
+  updateIndex(artifacts: ArtifactEntry[]): void {
+    this.ensureDocsStructure();
+
+    const lines: string[] = [
+      '# Documentation Index',
+      '',
+      `> Auto-generated by Popeye Pipeline — ${new Date().toISOString()}`,
+      '',
+      '## Artifacts',
+      '',
+      '| Type | Version | Path | Phase | Timestamp |',
+      '|------|---------|------|-------|-----------|',
+    ];
+
+    const sorted = [...artifacts].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
+
+    for (const a of sorted) {
+      lines.push(`| ${a.type} | v${a.version} | ${a.path} | ${a.phase} | ${a.timestamp} |`);
+    }
+
+    lines.push('');
+    const indexPath = join(this.docsDir, 'INDEX.md');
+    writeFileSync(indexPath, lines.join('\n'), 'utf-8');
+  }
+
+  /** Get the latest artifact entry in a specific group */
+  private getLatestInGroup(
+    groupId: string,
+    existingArtifacts: ArtifactEntry[],
+  ): ArtifactEntry | null {
+    const groupArtifacts = existingArtifacts
+      .filter((a) => a.group_id === groupId)
+      .sort((a, b) => a.version - b.version);
+    if (groupArtifacts.length === 0) return null;
+    return groupArtifacts[groupArtifacts.length - 1];
+  }
+}
+
+/** Factory function */
+export function createArtifactManager(projectDir: string): ArtifactManager {
+  return new ArtifactManager({ projectDir });
+}