popeye-cli 2.2.0 → 2.7.0

package/src/pipeline/auto-recovery.ts

@@ -0,0 +1,283 @@
+/**
+ * Auto-recovery via arbitrator strategic guidance (v2.6.0).
+ *
+ * When the pipeline exhausts its recovery budget, this module consults
+ * the arbitrator for a strategic perspective before entering STUCK.
+ * The arbitrator sees the full failure history and identifies patterns,
+ * not individual failures.
+ */
+
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import logging from 'node:console';
+
+import type { PipelineState, ArtifactEntry } from './types.js';
+import type { ArtifactManager } from './artifact-manager.js';
+import type { ConsensusConfig } from '../types/consensus.js';
+import { queryProvider } from './consensus/arbitrator-query.js';
+import { getModelForProvider } from './consensus/consensus-runner.js';
+
+const logger = logging;
+
+// ─── Types ───────────────────────────────────────────────
+
+interface FailureContext {
+  failedPhase: string;
+  recoveryCount: number;
+  maxIterations: number;
+  blockers: string[];
+  checkFailures: string[];
+  transitionSequence: string[];
+  rcaSummaries: string[];
+  recoveryPlanSummaries: string[];
+  sessionGuidance: string;
+}
+
+export interface AutoRecoveryResult {
+  success: boolean;
+  guidance: string | null;
+  artifact: ArtifactEntry | null;
+}
+
+interface AutoRecoveryOptions {
+  pipeline: PipelineState;
+  projectDir: string;
+  artifactManager: ArtifactManager;
+  consensusConfig?: Partial<ConsensusConfig>;
+}
+
+// ─── AUTO-RECOVERY GUIDANCE marker ───────────────────────
+
+const AUTO_RECOVERY_MARKER = '--- AUTO-RECOVERY GUIDANCE ---';
+
+// ─── Failure Context Builder ─────────────────────────────
+
+/**
+ * Extract structured failure evidence from pipeline state.
+ * All file reads are wrapped in try/catch with placeholder text.
+ */
+export function buildFailureContext(
+  pipeline: PipelineState,
+  projectDir: string,
+): FailureContext {
+  const failedPhase = pipeline.failedPhase ?? 'unknown';
+  const recoveryCount = pipeline.recoveryCount;
+  const maxIterations = pipeline.maxRecoveryIterations;
+
+  // Gate blockers
+  const gateResult = pipeline.failedPhase
+    ? pipeline.gateResults[pipeline.failedPhase]
+    : undefined;
+  const blockers = gateResult?.blockers ?? [];
+
+  // Check failures (command + stderr excerpt)
+  const checkResults = pipeline.failedPhase
+    ? pipeline.gateChecks[pipeline.failedPhase] ?? []
+    : [];
+  const checkFailures = checkResults
+    .filter((c) => c.status === 'fail')
+    .map((c) => `${c.check_type}: cmd="${c.command}" stderr="${(c.stderr_summary ?? '').slice(0, 200)}"`)
+    .slice(0, 5);
+
+  // Phase transition sequence — last 10 entries from gateResults
+  const transitionSequence = Object.entries(pipeline.gateResults)
+    .map(([phase, result]) => {
+      const topBlocker = result.blockers?.[0] ?? 'none';
+      return `${phase}: ${result.pass ? 'PASS' : 'FAIL'} (${topBlocker})`;
+    })
+    .slice(-10);
+
+  // Last 3 RCA report summaries (read text artifacts from disk)
+  const rcaArtifacts = pipeline.artifacts
+    .filter((a) => a.type === 'rca_report' && a.content_type === 'markdown')
+    .sort((a, b) => b.timestamp.localeCompare(a.timestamp))
+    .slice(0, 3);
+  const rcaSummaries = rcaArtifacts.map((a) => readArtifactSummary(projectDir, a.path, 500));
+
+  // Last 2 recovery fix plan summaries
+  const fixPlanArtifacts = pipeline.artifacts
+    .filter((a) => a.type === 'recovery_fix_plan')
+    .sort((a, b) => b.timestamp.localeCompare(a.timestamp))
+    .slice(0, 2);
+  const recoveryPlanSummaries = fixPlanArtifacts.map((a) => readArtifactSummary(projectDir, a.path, 500));
+
+  const sessionGuidance = pipeline.sessionGuidance ?? '';
+
+  return {
+    failedPhase,
+    recoveryCount,
+    maxIterations,
+    blockers,
+    checkFailures,
+    transitionSequence,
+    rcaSummaries,
+    recoveryPlanSummaries,
+    sessionGuidance,
+  };
+}
+
+// ─── Prompt Builder ──────────────────────────────────────
+
+/**
+ * Build the strategic recovery prompt for the arbitrator.
+ * Focuses on pattern analysis, not individual fixes.
+ */
+export function buildAutoRecoveryPrompt(ctx: FailureContext): string {
+  const lines: string[] = [
+    '# Strategic Recovery Analysis',
+    '',
+    `The pipeline failed at ${ctx.failedPhase} after ${ctx.recoveryCount}/${ctx.maxIterations} recovery attempts.`,
+    'All tactical fixes have been tried. You need to identify the PATTERN, not the individual failure.',
+    '',
+    '## Failure Timeline',
+    ...ctx.transitionSequence.map((s) => `- ${s}`),
+    '',
+    '## Current Blockers',
+    ...ctx.blockers.map((b) => `- ${b}`),
+    ...(ctx.checkFailures.length > 0
+      ? ['', '## Check Failures', ...ctx.checkFailures.map((f) => `- ${f}`)]
+      : []),
+    '',
+    '## What Was Already Tried',
+  ];
+
+  if (ctx.rcaSummaries.length > 0) {
+    lines.push('### RCA Reports');
+    ctx.rcaSummaries.forEach((s, i) => {
+      lines.push(`#### RCA ${i + 1}`, s, '');
+    });
+  }
+
+  if (ctx.recoveryPlanSummaries.length > 0) {
+    lines.push('### Recovery Fix Plans');
+    ctx.recoveryPlanSummaries.forEach((s, i) => {
+      lines.push(`#### Fix Plan ${i + 1}`, s, '');
+    });
+  }
+
+  if (ctx.sessionGuidance) {
+    lines.push('## Current Guidance', ctx.sessionGuidance.slice(0, 1500), '');
+  }
+
+  lines.push(
+    '## Your Task',
+    '',
+    `1. **Root Pattern**: What recurring pattern caused all ${ctx.recoveryCount} attempts to fail?`,
+    '   Pick the SINGLE most likely root pattern.',
+    '',
+    '2. **Primary Strategy**: One concrete strategic change that breaks the pattern.',
+    '   (e.g., "reduce scope to X", "replace Y approach with Z", "remove module W entirely")',
+    '',
+    '3. **Fallback Strategy**: One alternative if the primary doesn\'t work.',
+    '',
+    '4. **Stop Doing** (list 3 things): Based on the history, what should the pipeline',
+    '   STOP trying? These are approaches that have been attempted and failed.',
+    '',
+    '5. **Concrete Next Steps**: 3-5 high-level steps (not code-level fixes).',
+    '',
+    'Be decisive. Pick one direction. Do NOT produce another tactical dump.',
+  );
+
+  return lines.join('\n');
+}
+
+// ─── Main Entry Point ────────────────────────────────────
+
+/**
+ * Attempt auto-recovery by consulting the arbitrator for strategic guidance.
+ *
+ * @returns Result with success flag, guidance text, and stored artifact
+ */
+export async function attemptAutoRecovery(
+  opts: AutoRecoveryOptions,
+): Promise<AutoRecoveryResult> {
+  const { pipeline, projectDir, artifactManager, consensusConfig } = opts;
+
+  // Build failure context and strategic prompt
+  const ctx = buildFailureContext(pipeline, projectDir);
+  const prompt = buildAutoRecoveryPrompt(ctx);
+
+  // Resolve arbitrator provider config
+  const arbitratorName = consensusConfig?.arbitrator ?? 'gemini';
+  const model = getModelForProvider(consensusConfig, arbitratorName);
+  const providerConfig = {
+    provider: arbitratorName,
+    model,
+    temperature: 0.3,
+  };
+
+  logger.log(
+    `[auto-recovery] Querying ${providerConfig.provider}/${providerConfig.model} for strategic guidance`,
+  );
+
+  // Query with 90s timeout
+  const raw = await queryProvider(prompt, providerConfig, 90_000);
+
+  if (!raw) {
+    logger.warn('[auto-recovery] Provider returned no response (timeout or error)');
+    return { success: false, guidance: null, artifact: null };
+  }
+
+  // Validate minimum response length
+  if (raw.trim().length < 50) {
+    logger.warn(`[auto-recovery] Response too short (${raw.trim().length} chars), discarding`);
+    return { success: false, guidance: null, artifact: null };
+  }
+
+  // Store as artifact
+  const artifact = artifactManager.createAndStoreText(
+    'auto_recovery_guidance',
+    `# Auto-Recovery Strategic Guidance\n\n${raw}`,
+    'RECOVERY_LOOP',
+  );
+
+  // Inject into sessionGuidance (idempotent — replaces prior auto-recovery block)
+  injectAutoRecoveryGuidance(pipeline, raw);
+
+  logger.log(`[auto-recovery] Guidance received and injected (${raw.length} chars)`);
+
+  return { success: true, guidance: raw, artifact };
+}
+
+// ─── Guidance Injection ──────────────────────────────────
+
+/**
+ * Inject auto-recovery guidance into pipeline.sessionGuidance.
+ * Replaces any prior auto-recovery block while preserving base guidance.
+ */
+export function injectAutoRecoveryGuidance(
+  pipeline: PipelineState,
+  guidance: string,
+): void {
+  const existing = pipeline.sessionGuidance ?? '';
+
+  // Strip prior auto-recovery block if present
+  const base = existing.includes(AUTO_RECOVERY_MARKER)
+    ? existing.slice(0, existing.indexOf(AUTO_RECOVERY_MARKER)).trim()
+    : existing;
+
+  pipeline.sessionGuidance = [
+    base,
+    '',
+    AUTO_RECOVERY_MARKER,
+    guidance.slice(0, 3000),
+  ].join('\n').trim();
+}
+
+// ─── Helpers ─────────────────────────────────────────────
+
+/**
+ * Read artifact content from disk, returning first N chars or placeholder.
+ */
+function readArtifactSummary(projectDir: string, artifactPath: string, maxChars: number): string {
+  try {
+    const fullPath = join(projectDir, artifactPath);
+    if (!existsSync(fullPath)) {
+      return `[Could not read: ${artifactPath} — file not found]`;
+    }
+    const content = readFileSync(fullPath, 'utf-8');
+    return content.slice(0, maxChars);
+  } catch {
+    return `[Could not read: ${artifactPath}]`;
+  }
+}

package/src/pipeline/change-request.ts

@@ -4,7 +4,7 @@
  * Each CR routes to the appropriate consensus phase for approval.
  */
 
-import { randomUUID } from 'node:crypto';
+import { randomUUID, createHash } from 'node:crypto';
 
 import type {
   PipelinePhase,
@@ -24,6 +24,8 @@ export interface BuildChangeRequestArgs {
   affectedArtifacts: ArtifactRef[];
   affectedPhases: PipelinePhase[];
   riskLevel: 'low' | 'medium' | 'high';
+  /** Deterministic drift fingerprint for CR deduplication (v2.4.9) */
+  driftKey?: string;
 }
 
 /**
@@ -50,6 +52,7 @@ export function buildChangeRequest(args: BuildChangeRequestArgs): ChangeRequest
       risk_level: args.riskLevel,
     },
     status: 'proposed',
+    drift_key: args.driftKey,
   };
 }
 
@@ -117,3 +120,62 @@ export function formatChangeRequest(cr: ChangeRequest): string {
 
   return lines.join('\n');
 }
+
+// ─── Drift Key Dedup (v2.4.9) ────────────────────────────
+
+/**
+ * Compute a deterministic drift key for CR deduplication.
+ * Same drift (same change type, baseline, changed configs, content hashes)
+ * always produces the same key, regardless of input order.
+ *
+ * Args:
+ *   changeType: The CR change type (config, scope, etc.).
+ *   baselineSnapshotId: The artifact_id of the baseline snapshot.
+ *   changedConfigs: List of changed config file paths.
+ *   configHashPairs: Array of "path:beforeHash->afterHash" strings.
+ *
+ * Returns:
+ *   A 32-char hex string (SHA-256 prefix).
+ */
+export function computeDriftKey(
+  changeType: string,
+  baselineSnapshotId: string,
+  changedConfigs: string[],
+  configHashPairs: string[],
+): string {
+  const sortedConfigs = [...changedConfigs].sort().join(',');
+  const sortedPairs = [...configHashPairs].sort().join(',');
+  const input = `${changeType}|${baselineSnapshotId}|${sortedConfigs}|${sortedPairs}`;
+  return createHash('sha256').update(input).digest('hex').slice(0, 32);
+}
+
+/** Pending CR shape from PipelineState */
+interface PendingCR {
+  cr_id: string;
+  change_type: string;
+  target_phase: string;
+  status: string;
+  drift_key?: string;
+}
+
+/**
+ * Check whether a pending CR with the same drift_key already exists.
+ * Returns true if any non-rejected CR has the same drift_key (proposed,
+ * approved, or resolved CRs all count as "already tracked").
+ *
+ * Args:
+ *   pendingCRs: The current pending change requests array (may be undefined).
+ *   driftKey: The drift key to check.
+ *
+ * Returns:
+ *   true if a non-rejected duplicate exists.
+ */
+export function isDuplicateCR(
+  pendingCRs: PendingCR[] | undefined,
+  driftKey: string,
+): boolean {
+  if (!pendingCRs) return false;
+  return pendingCRs.some(
+    (cr) => cr.drift_key === driftKey && cr.status !== 'rejected',
+  );
+}

package/src/pipeline/check-runner.ts

@@ -7,7 +7,7 @@
  */
 
 import { exec } from 'node:child_process';
-import { existsSync, readFileSync, readdirSync } from 'node:fs';
+import { existsSync, readFileSync, readdirSync, writeFileSync, mkdirSync, unlinkSync } from 'node:fs';
 import { join, extname } from 'node:path';
 
 import type {
@@ -17,13 +17,16 @@ import type {
   RepoSnapshot,
   ArtifactEntry,
   PipelinePhase,
+  PipelineState,
 } from './types.js';
 import { ArtifactManager } from './artifact-manager.js';
+import { assertSkillCoverage } from './skills/coverage-gate.js';
 
 // ─── Constants ───────────────────────────────────────────
 
 /** Default timeout per check type in milliseconds */
 const DEFAULT_TIMEOUTS: Record<string, number> = {
+  install: 15 * 60 * 1000,  // 15 minutes
   build: 20 * 60 * 1000,    // 20 minutes
   test: 10 * 60 * 1000,     // 10 minutes
   lint: 5 * 60 * 1000,      // 5 minutes
@@ -58,6 +61,7 @@ const PLACEHOLDER_PATTERNS = [
   /\btemp\b(?!late)/i, // 'temp' but not 'template'
   /lorem ipsum/i,
   /example\.com/i,
+  /coming soon/i,
 ];
 
 // ─── Command Sanitization ────────────────────────────────
@@ -198,13 +202,14 @@ export function storeCheckResults(
 
 function mapCheckTypeToArtifactType(
   checkType: GateCheckType,
-): 'build_check' | 'test_check' | 'lint_check' | 'typecheck_check' | 'placeholder_scan' {
+): 'build_check' | 'test_check' | 'lint_check' | 'typecheck_check' | 'placeholder_scan' | 'install_check' {
   switch (checkType) {
     case 'build': return 'build_check';
     case 'test': return 'test_check';
     case 'lint': return 'lint_check';
     case 'typecheck': return 'typecheck_check';
     case 'placeholder_scan': return 'placeholder_scan';
+    case 'install': return 'install_check';
     default: return 'build_check';
   }
 }
@@ -391,6 +396,41 @@ export async function runStartCheck(
   });
 }
 
+// ─── Skill Coverage Check (v2.2.1) ──────────────────────
+
+/**
+ * Run deterministic skill coverage check against pipeline state.
+ * No subprocess needed — pure in-memory assertion.
+ *
+ * Args:
+ *   pipeline: Current pipeline state with activeRoles and skillUsageEvents.
+ *   currentPhase: Current pipeline phase for phase-aware deferral (v2.4.5).
+ *     Omit for strict mode (checks all roles).
+ *
+ * Returns:
+ *   GateCheckResult with pass/fail status.
+ */
+export function runSkillCoverageCheck(
+  pipeline: PipelineState,
+  currentPhase?: PipelinePhase,
+): GateCheckResult {
+  const events = pipeline.skillUsageEvents ?? [];
+  const result = assertSkillCoverage(pipeline.activeRoles, events, pipeline, currentPhase);
+
+  return {
+    check_type: 'skill_coverage',
+    status: result.pass ? 'pass' : 'fail',
+    command: 'skill-coverage-check',
+    exit_code: result.pass ? 0 : 1,
+    stderr_summary: result.pass
+      ? `All ${result.covered.length} active roles have skill usage` +
+        (result.deferred.length > 0 ? ` (${result.deferred.length} deferred)` : '')
+      : `Missing skill usage: ${result.missing.map((m) => `${m.role} (${m.reason})`).join(', ')}`,
+    duration_ms: 0,
+    timestamp: new Date().toISOString(),
+  };
+}
+
 // ─── Env Check (v1.1 Gap #5) ────────────────────────────
 
 /**
@@ -502,3 +542,102 @@ function parseEnvVarValues(content: string): Record<string, string> {
   }
   return vars;
 }
+
+// ─── Install Skip Heuristic ─────────────────────────────
+
+/** Marker file for install skip heuristic */
+interface InstallMarker {
+  lockfileHash: string;
+  timestamp: string;
+}
+
+/** Canonical lockfile names by ecosystem */
+const LOCKFILE_MAP: Record<string, string> = {
+  npm: 'package-lock.json',
+  pnpm: 'pnpm-lock.yaml',
+  yarn: 'yarn.lock',
+  bun: 'bun.lockb',
+  poetry: 'poetry.lock',
+  pip: 'requirements.txt',
+};
+
+/**
+ * Determine whether install can be skipped based on lockfile hash.
+ * Returns true only if the marker matches the current lockfile hash
+ * AND the install target directory exists (node_modules or .venv).
+ */
+export function shouldSkipInstall(projectDir: string, snapshot: RepoSnapshot): boolean {
+  const markerPath = join(projectDir, '.popeye', 'install-marker.json');
+  if (!existsSync(markerPath)) return false;
+
+  try {
+    const marker: InstallMarker = JSON.parse(readFileSync(markerPath, 'utf-8'));
+    const currentHash = getCanonicalLockfileHash(snapshot);
+    if (!currentHash || marker.lockfileHash !== currentHash) return false;
+
+    // Verify install target directory exists
+    const pm = snapshot.package_manager ?? 'npm';
+    const isPython = pm === 'pip' || pm === 'poetry';
+    const targetDir = isPython
+      ? join(projectDir, '.venv')
+      : join(projectDir, 'node_modules');
+    return existsSync(targetDir);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Write install marker after successful install.
+ * Stores the current lockfile hash for future skip checks.
+ */
+export function writeInstallMarker(projectDir: string, snapshot: RepoSnapshot): void {
+  const markerDir = join(projectDir, '.popeye');
+  const markerPath = join(markerDir, 'install-marker.json');
+
+  const lockfileHash = getCanonicalLockfileHash(snapshot);
+  if (!lockfileHash) return;
+
+  try {
+    if (!existsSync(markerDir)) {
+      mkdirSync(markerDir, { recursive: true });
+    }
+    const marker: InstallMarker = {
+      lockfileHash,
+      timestamp: new Date().toISOString(),
+    };
+    writeFileSync(markerPath, JSON.stringify(marker, null, 2));
+  } catch {
+    // Non-fatal — worst case we re-run install next time
+  }
+}
+
+/**
+ * Invalidate install marker so the next check re-runs install.
+ * Called on missing-module errors or dependency changes.
+ */
+export function invalidateInstallMarker(projectDir: string): void {
+  const markerPath = join(projectDir, '.popeye', 'install-marker.json');
+  try {
+    if (existsSync(markerPath)) unlinkSync(markerPath);
+  } catch {
+    // Non-fatal
+  }
+}
+
+/** Get content_hash of the canonical lockfile from the snapshot */
+function getCanonicalLockfileHash(snapshot: RepoSnapshot): string | undefined {
+  const pm = snapshot.package_manager ?? 'npm';
+  const lockfileName = LOCKFILE_MAP[pm];
+
+  if (lockfileName) {
+    const entry = snapshot.config_files.find((c) => c.type === lockfileName);
+    if (entry) return entry.content_hash;
+  }
+
+  // Fallback: use package.json hash if no lockfile found
+  const pkgEntry = snapshot.config_files.find((c) => c.type === 'package.json');
+  if (pkgEntry) return pkgEntry.content_hash;
+
+  return undefined;
+}

package/src/pipeline/command-resolver.ts

@@ -40,14 +40,21 @@ export function resolveCommands(
     case 'python':
       resolved = resolvePythonCommands(snapshot);
       break;
-    case 'mixed':
+    case 'mixed': {
       // Prefer Node commands, augment with Python where Node is missing
       resolved = resolveNodeCommands(pm, scripts, snapshot);
+      const pyResolved = resolvePythonCommands(snapshot);
       if (!resolved.test) {
-        const pyResolved = resolvePythonCommands(snapshot);
         resolved.test = pyResolved.test;
       }
+      // Chain both install commands for mixed projects
+      if (resolved.install && pyResolved.install) {
+        resolved.install = `${resolved.install} && ${pyResolved.install}`;
+      } else if (!resolved.install) {
+        resolved.install = pyResolved.install;
+      }
       break;
+    }
     default:
       resolved = { resolved_from: 'none' };
   }
@@ -60,6 +67,8 @@ export function resolveCommands(
     if (overrides.typecheck) resolved.typecheck = overrides.typecheck;
     if (overrides.migrations) resolved.migrations = overrides.migrations;
     if (overrides.start) resolved.start = overrides.start;
+    if (overrides.install) resolved.install = overrides.install;
+    if (overrides.install_cwd) resolved.install_cwd = overrides.install_cwd;
   }
 
   return resolved;
@@ -120,6 +129,19 @@ function resolveNodeCommands(
     resolved.start = `${run} dev`;
   }
 
+  // Install — always resolve based on package manager
+  resolved.install = pm === 'yarn' ? 'yarn install' : `${pm} install`;
+
+  // Workspace detection: install must run at the workspace root
+  const rootPkg = snapshot.config_files.find((c) => c.type === 'package.json');
+  const hasWorkspaces = rootPkg?.key_fields?.workspaces !== undefined;
+  const hasPnpmWorkspace = snapshot.config_files.some(
+    (c) => c.type === 'pnpm-workspace.yaml',
+  );
+  if (hasWorkspaces || hasPnpmWorkspace) {
+    resolved.install_cwd = '.';
+  }
+
   return resolved;
 }
 
@@ -164,5 +186,15 @@ function resolvePythonCommands(snapshot: RepoSnapshot): ResolvedCommands {
   // Start
   resolved.start = 'uvicorn main:app --host 0.0.0.0 --port 8000';
 
+  // Install — conservative: only well-known safe patterns
+  const hasPoetryLock = snapshot.config_files.some((c) => c.type === 'poetry.lock');
+  const hasReqs = snapshot.config_files.some((c) => c.type === 'requirements.txt');
+  if (hasPoetryLock) {
+    resolved.install = 'poetry install';
+  } else if (hasReqs) {
+    resolved.install = 'pip install -r requirements.txt';
+  }
+  // No install for pyproject.toml-only (may need build backends, system deps)
+
   return resolved;
 }