swarm-engine 1.1.1 → 1.38.0

package/dist/runtime/engine.js

@@ -19,14 +19,30 @@ import { KnowledgeCompounder } from './compounder.js';
 import { ModelRouter } from './model-router.js';
 import { AutonomyTracker } from './autonomy.js';
 import { ContinuousVerifier } from './verifier.js';
-import { optimizeForCache, estimateCacheSavings } from './cache-optimizer.js';
-import { classifyTaskComplexity, selectStartingTier } from './cascade.js';
-import { createLivingSpec, updateSpecFromPhase, formatLivingSpec } from './living-spec.js';
+import { estimateCacheSavings } from './cache-optimizer.js';
+import { createLivingSpec, updateSpecFromPhase } from './living-spec.js';
 import { estimateTokens } from '../utils/tokens.js';
-import { redactSecrets } from '../utils/redact.js';
-import { join } from 'path';
-import { existsSync, mkdirSync, writeFileSync, unlinkSync } from 'fs';
-import { randomUUID } from 'crypto';
+import { unlinkSync } from 'fs';
+import { AgentRunner } from './agent-runner.js';
+import { LearningEngine } from './learning-engine.js';
+import { ExecutionGraph } from './execution-graph.js';
+import { GraphLearner } from './graph-learner.js';
+import { GraphAnalyzer } from './graph-analyzer.js';
+import { GraphContextRouter } from './graph-context-router.js';
+import { ReviewFeedbackRecorder } from './graph-feedback.js';
+import { parseReviewOutput } from './review-schema.js';
+import { closeDatabase } from './database.js';
+import { CausalGraphEngine } from './graph-causal.js';
+import { FailurePropagationPredictor } from './graph-gnn.js';
+import { AdversarialEvolver } from './graph-adversarial.js';
+import { MetaPatternSelector } from './graph-meta.js';
+import { PredictiveDropout } from './graph-dropout.js';
+import { PatternSynthesizer } from './graph-synthesis.js';
+import { TrajectoryPredictor } from './graph-trajectory.js';
+import { MetaAdversarialTester } from './graph-meta-adversarial.js';
+import { RuleEvolver } from './graph-self-evolve.js';
+import { TaskDiscovery } from './graph-discovery.js';
+import { OrchestrationEmbedder } from './graph-embeddings.js';
 /**
  * The Swarm Orchestration Engine.
  *
@@ -60,6 +76,24 @@ export class SwarmEngine {
     signalHandlers = [];
     sharedContextFiles = [];
     livingSpec = null;
+    agentRunner;
+    learningEngine;
+    executionGraph = null;
+    graphAnalyzer = null;
+    graphLearner = null;
+    reviewFeedback = null;
+    causalEngine = null;
+    gnnPredictor = null;
+    adversarialEvolver = null;
+    metaSelector = null;
+    predictiveDropout = null;
+    gnnWeights = null;
+    patternSynthesizer = null;
+    trajectoryPredictor = null;
+    metaAdversarial = null;
+    ruleEvolver = null;
+    taskDiscovery = null;
+    orchestrationEmbedder = null;
     constructor(options) {
         this.options = options;
         this.registry = options.registry;
@@ -85,43 +119,98 @@ export class SwarmEngine {
         this.stats.attachToEventBus(this.bus, undefined);
         this.costModel = new CostModel(this.stats);
         this.reflectionEngine = new ReflectionEngine();
-        try {
-            this.traceStore = new TraceStore();
-        }
-        catch (e) {
-            this.log.warn('TraceStore init failed', { error: e instanceof Error ? e.message : e });
-        }
+        this.traceStore = this.tryInit(() => new TraceStore(), 'TraceStore');
         this.replanner = new AdaptiveReplanner(options.logger, this.traceStore ?? undefined);
-        try {
-            this.heuristicStore = new HeuristicStore();
-        }
-        catch (e) {
-            this.log.warn('HeuristicStore init failed', { error: e instanceof Error ? e.message : e });
-        }
-        try {
-            this.modelRouter = new ModelRouter();
-        }
-        catch (e) {
-            this.log.warn('ModelRouter init failed', { error: e instanceof Error ? e.message : e });
-        }
-        try {
-            this.autonomyTracker = new AutonomyTracker();
-        }
-        catch (e) {
-            this.log.warn('AutonomyTracker init failed', { error: e instanceof Error ? e.message : e });
-        }
-        try {
-            this.compounder = new KnowledgeCompounder();
+        this.heuristicStore = this.tryInit(() => new HeuristicStore(), 'HeuristicStore');
+        this.modelRouter = this.tryInit(() => new ModelRouter(), 'ModelRouter');
+        this.autonomyTracker = this.tryInit(() => new AutonomyTracker(), 'AutonomyTracker');
+        this.compounder = this.tryInit(() => new KnowledgeCompounder(), 'KnowledgeCompounder');
+        this.executionGraph = this.tryInit(() => new ExecutionGraph(), 'ExecutionGraph');
+        this.executionGraph?.attachToEventBus(this.bus);
+        // Create graph analysis deps (reused by AgentRunner + LearningEngine)
+        this.graphLearner = this.executionGraph ? this.tryInit(() => new GraphLearner(this.executionGraph), 'GraphLearner') : null;
+        this.graphAnalyzer = this.executionGraph ? this.tryInit(() => new GraphAnalyzer(this.executionGraph), 'GraphAnalyzer') : null;
+        this.reviewFeedback = this.executionGraph ? this.tryInit(() => new ReviewFeedbackRecorder(this.executionGraph), 'ReviewFeedbackRecorder') : null;
+        // Cutting-edge ML features
+        this.causalEngine = this.executionGraph
+            ? this.tryInit(() => new CausalGraphEngine(this.executionGraph), 'CausalGraphEngine')
+            : null;
+        this.gnnPredictor = this.executionGraph
+            ? this.tryInit(() => new FailurePropagationPredictor(this.executionGraph), 'FailurePropagationPredictor')
+            : null;
+        this.adversarialEvolver = this.executionGraph
+            ? this.tryInit(() => new AdversarialEvolver(this.executionGraph), 'AdversarialEvolver')
+            : null;
+        this.metaSelector = this.executionGraph
+            ? this.tryInit(() => new MetaPatternSelector(this.executionGraph), 'MetaPatternSelector')
+            : null;
+        this.predictiveDropout = this.executionGraph
+            ? this.tryInit(() => new PredictiveDropout(this.executionGraph), 'PredictiveDropout')
+            : null;
+        // Initialize GNN weights (training happens lazily via background scheduling)
+        if (this.gnnPredictor) {
+            try {
+                this.gnnWeights = this.gnnPredictor.initWeights();
+            }
+            catch { /* silent */ }
+        }
+        // Self-aware engine features
+        this.patternSynthesizer = this.executionGraph ? this.tryInit(() => new PatternSynthesizer(this.executionGraph), 'PatternSynthesizer') : null;
+        this.trajectoryPredictor = this.executionGraph ? this.tryInit(() => new TrajectoryPredictor(this.executionGraph), 'TrajectoryPredictor') : null;
+        this.metaAdversarial = this.executionGraph ? this.tryInit(() => new MetaAdversarialTester(this.executionGraph), 'MetaAdversarialTester') : null;
+        this.ruleEvolver = this.executionGraph ? this.tryInit(() => new RuleEvolver(this.executionGraph), 'RuleEvolver') : null;
+        this.taskDiscovery = this.executionGraph ? this.tryInit(() => new TaskDiscovery(this.executionGraph), 'TaskDiscovery') : null;
+        this.orchestrationEmbedder = this.executionGraph ? this.tryInit(() => new OrchestrationEmbedder(this.executionGraph), 'OrchestrationEmbedder') : null;
+        // Prune stale graph data on init (Feature 6: graph decay)
+        if (this.executionGraph) {
+            try {
+                this.executionGraph.pruneOlderThan(90);
+            }
+            catch { /* silent */ }
         }
-        catch (e) {
-            this.log.warn('KnowledgeCompounder init failed', { error: e instanceof Error ? e.message : e });
+        // Rule evolution proposal on startup (skip in mock mode and when no historical data)
+        if (this.ruleEvolver && !this.options.mock) {
+            try {
+                const proposals = this.ruleEvolver.evolveRules(50);
+                if (proposals.length > 0) {
+                    this.log.info(`Rule evolver proposes ${proposals.length} rule updates (top: ${proposals[0].rule.name}, +${Math.round(proposals[0].backtestedImprovement * 100)}%)`);
+                }
+            }
+            catch (e) {
+                this.log.debug(`Rule evolution failed: ${e}`);
+            }
         }
         if (options.worktreeEnabled) {
             try {
                 this.worktreeManager = new WorktreeManager(options.cwd);
             }
-            catch { /* not a git repo */ }
+            catch {
+                /* not a git repo */
+            }
         }
+        this.agentRunner = new AgentRunner({
+            registry: this.registry,
+            executor: this.executor,
+            bus: this.bus,
+            logger: options.logger,
+            compaction: this.compaction,
+            worktreeManager: this.worktreeManager,
+            contextRouter: this.executionGraph ? new GraphContextRouter(this.executionGraph) : undefined,
+            graphLearner: this.graphLearner,
+            graphAnalyzer: this.graphAnalyzer,
+        });
+        this.learningEngine = new LearningEngine({
+            traceStore: this.traceStore,
+            heuristicStore: this.heuristicStore,
+            modelRouter: this.modelRouter,
+            autonomyTracker: this.autonomyTracker,
+            compounder: this.compounder,
+            graphLearner: this.graphLearner,
+            metaSelector: this.metaSelector,
+            predictiveDropout: this.predictiveDropout,
+            orchestrationEmbedder: this.orchestrationEmbedder,
+            log: options.logger,
+        });
         if (options.pluginLoader) {
             options.pluginLoader.wireHooks(this.bus);
         }
@@ -147,16 +236,14 @@ export class SwarmEngine {
         orchestration.status = 'running';
         orchestration.startedAt = new Date();
         this.phaseOutputs = new Map();
+        // TODO: Checkpoint doesn't store phase outputs — resumed phases lose inter-phase context
         this.reflections = [];
         this.sharedContextFiles = [];
         this.abortController = new AbortController();
         this.livingSpec = createLivingSpec(config.description);
         // Start continuous verification
-        let continuousVerifier = null;
-        if (this.verifier) {
-            continuousVerifier = new ContinuousVerifier(this.verifier, this.bus);
-            continuousVerifier.start();
-        }
+        const continuousVerifier = new ContinuousVerifier(this.verifier, this.bus);
+        continuousVerifier.start();
         try {
             // Execute phases using work-queue approach (handles DAGs correctly)
             // If resuming, seed completed phases so the DAG executor skips them
@@ -166,6 +253,20 @@ export class SwarmEngine {
                     completed.add(phase.config.name);
                 }
             }
+            // GNN pre-execution risk prediction (Feature 2: GNN Failure Propagation)
+            if (this.gnnPredictor && this.gnnWeights) {
+                try {
+                    const risks = this.gnnPredictor.predict(orchestration.config.id, this.gnnWeights);
+                    const highRisk = risks.filter(r => r.riskScore > 0.7);
+                    if (highRisk.length > 0) {
+                        this.log.warn(`GNN predicts high failure risk for ${highRisk.length} nodes`);
+                        this.bus.emit('system:warning', { type: 'gnn-risk', risks: highRisk }, 'engine');
+                    }
+                }
+                catch (e) {
+                    this.log.debug(`GNN prediction failed: ${e}`);
+                }
+            }
             while (completed.size < orchestration.phases.length) {
                 // Check if abort was requested
                 if (this.abortController.signal.aborted) {
@@ -173,17 +274,19 @@ export class SwarmEngine {
                     break;
                 }
                 // Find all runnable phases (dependencies met, not yet completed)
-                const runnable = orchestration.phases.filter(p => {
+                const runnable = orchestration.phases.filter((p) => {
                     if (completed.has(p.config.name))
                         return false;
                     if (p.status === 'failed')
                         return false;
+                    if (p.status === 'skipped')
+                        return false;
                     const deps = p.config.dependsOn ?? [];
-                    return deps.every(d => completed.has(d));
+                    return deps.every((d) => completed.has(d));
                 });
                 if (runnable.length === 0) {
                     // No runnable phases — either all done or deadlocked
-                    const incomplete = orchestration.phases.filter(p => !completed.has(p.config.name) && p.status !== 'failed');
+                    const incomplete = orchestration.phases.filter((p) => !completed.has(p.config.name) && p.status !== 'failed');
                     if (incomplete.length > 0) {
                         // Deadlock — blocked phases with unmet dependencies
                         for (const p of incomplete) {
@@ -191,15 +294,14 @@ export class SwarmEngine {
                         }
                         orchestration.status = 'failed';
                         this.log.error('Orchestration deadlocked: blocked phases with unmet dependencies', {
-                            blocked: incomplete.map(p => p.config.name),
+                            blocked: incomplete.map((p) => p.config.name),
                         });
                     }
                     break;
                 }
-                // Execute runnable phases (could be parallel if multiple are ready)
+                // Handle approval gates sequentially (can't prompt in parallel)
+                const approved = [];
                 for (const phase of runnable) {
-                    orchestration.currentPhase = orchestration.phases.indexOf(phase);
-                    // Check for approval gate
                     if (phase.config.requiresApproval) {
                         this.log.info(`Phase requires approval: ${phase.config.name}`);
                         this.bus.emit('orchestration:paused', {
@@ -207,10 +309,9 @@ export class SwarmEngine {
                             phase: phase.config.name,
                             agentCount: phase.config.agents.length,
                         }, 'engine');
-                        // In interactive mode, wait for approval
                         if (!this.options.nonInteractive) {
-                            const approved = await this.requestApproval(phase.config.name, phase.config.agents);
-                            if (!approved) {
+                            const ok = await this.requestApproval(phase.config.name, phase.config.agents);
+                            if (!ok) {
                                 phase.status = 'skipped';
                                 this.log.info(`Phase skipped (not approved): ${phase.config.name}`);
                                 completed.add(phase.config.name);
@@ -218,14 +319,76 @@ export class SwarmEngine {
                             }
                         }
                     }
+                    approved.push(phase);
+                }
+                // Execute approved phases in parallel
+                orchestration.currentPhase = orchestration.phases.indexOf(approved[approved.length - 1] ?? runnable[0]);
+                const results = await Promise.allSettled(approved.map(async (phase) => {
                     const outputs = await this.executePhase(orchestration, phase);
                     this.phaseOutputs.set(phase.config.name, outputs);
+                }));
+                // Process results — mark completed or failed
+                for (let i = 0; i < results.length; i++) {
+                    const phase = approved[i];
+                    if (results[i].status === 'rejected') {
+                        phase.status = 'failed';
+                    }
                     if (phase.status === 'completed') {
                         completed.add(phase.config.name);
+                        // Confidence-gated phase transition (Feature 3)
+                        if (this.graphAnalyzer) {
+                            try {
+                                const reflection = this.reflectionEngine.reflect(phase);
+                                const patternStats = this.graphLearner?.getPatternStats(orchestration.config.pattern)?.[0];
+                                const gateOutputs = this.phaseOutputs.get(phase.config.name) || [];
+                                const gate = this.graphAnalyzer.evaluatePhaseConfidence({ kind: phase.config.kind ?? 'implement', agentCount: phase.agents.length, outputs: gateOutputs, usage: phase.usage }, { confidence: reflection.confidence }, patternStats ? { successRate: patternStats.successRate, avgDurationMs: patternStats.avgDurationMs } : undefined);
+                                if (gate.result === 'halt') {
+                                    this.log.warn(`Phase gate HALT: ${gate.reason}`);
+                                    orchestration.status = 'failed';
+                                }
+                                if (gate.result === 'warn') {
+                                    this.log.warn(`Phase gate WARNING: ${gate.reason}`);
+                                    this.bus.emit('system:warning', { type: 'gate-warning', orchestrationId: orchestration.config.id, phase: phase.config.name, reason: gate.reason }, 'engine');
+                                }
+                            }
+                            catch { /* Graph gate evaluation failed — continue */ }
+                        }
+                        // Trajectory prediction after each completed phase
+                        if (this.trajectoryPredictor) {
+                            try {
+                                const features = this.trajectoryPredictor.extractFeatures(orchestration, phase);
+                                const pred = this.trajectoryPredictor.predict(features);
+                                if (pred.successProbability < 0.3 && pred.confidence !== 'low') {
+                                    this.log.warn(`Trajectory predictor: ${(pred.successProbability * 100).toFixed(0)}% success probability (${pred.trajectory})`);
+                                    this.bus.emit('system:warning', { type: 'trajectory-risk', ...pred }, 'engine');
+                                }
+                            }
+                            catch (e) {
+                                this.log.debug(`Trajectory prediction failed: ${e}`);
+                            }
+                        }
                     }
                     else if (phase.status === 'failed') {
                         orchestration.status = 'failed';
-                        break;
+                    }
+                }
+                // Trajectory forecast over remaining phases
+                if (this.trajectoryPredictor) {
+                    try {
+                        const forecastCompleted = orchestration.phases
+                            .filter(p => p.status === 'completed')
+                            .map(p => ({ name: p.config.name, status: p.status, tokens: p.usage?.totalTokens || 0, durationMs: p.usage?.durationMs || 0, confidence: 'medium' }));
+                        const forecastRemaining = orchestration.phases.filter(p => p.status === 'pending').map(p => p.config.name);
+                        if (forecastRemaining.length > 0) {
+                            const forecast = this.trajectoryPredictor.predictTrajectory(orchestration.config.id, forecastCompleted, forecastRemaining, orchestration.config.pattern);
+                            if (forecast.overallSuccessProb < 0.3) {
+                                this.log.warn(`Trajectory forecast: ${Math.round(forecast.overallSuccessProb * 100)}% success probability`);
+                                this.bus.emit('system:warning', { type: 'trajectory-forecast', forecast }, 'engine');
+                            }
+                        }
+                    }
+                    catch (e) {
+                        this.log.debug(`Trajectory forecast failed: ${e}`);
                     }
                 }
                 if (orchestration.status === 'failed')
@@ -233,12 +396,13 @@ export class SwarmEngine {
                 // Check cost budget after each batch of phases
                 if (orchestration.config.costBudget) {
                     const spent = orchestration.phases
-                        .filter(p => p.status === 'completed')
+                        .filter((p) => p.status === 'completed')
                         .reduce((sum, p) => sum + p.usage.costUsd, 0);
                     if (spent >= orchestration.config.costBudget) {
                         orchestration.status = 'failed';
                         this.log.warn(`Cost budget exceeded: $${spent.toFixed(2)} >= $${orchestration.config.costBudget}`, {
-                            spent, budget: orchestration.config.costBudget,
+                            spent,
+                            budget: orchestration.config.costBudget,
                         });
                         this.bus.emit('system:warning', {
                             reason: 'budget-exceeded',
@@ -250,7 +414,7 @@ export class SwarmEngine {
                     }
                 }
             }
-            if (orchestration.status !== 'failed') {
+            if (orchestration.status !== 'failed' && orchestration.status !== 'cancelled') {
                 orchestration.status = 'completed';
             }
         }
@@ -260,8 +424,50 @@ export class SwarmEngine {
                 error: error instanceof Error ? error.message : String(error),
             });
         }
+        // Post-failure causal analysis (Feature 1: Causal Inference)
+        // Note: suggestIntervention scans all historical orchestrations — skip when graph is large
+        if (orchestration.status === 'failed' && this.causalEngine && this.executionGraph) {
+            try {
+                const orchCount = this.executionGraph.getNodesByType('orchestration').length;
+                if (orchCount <= 200) {
+                    const suggestions = this.causalEngine.suggestIntervention(orchestration.config.id);
+                    if (suggestions.length > 0) {
+                        this.log.info(`Causal analysis suggests: ${suggestions[0].reasoning}`);
+                        this.bus.emit('system:warning', { type: 'causal-suggestion', suggestions }, 'engine');
+                    }
+                }
+            }
+            catch (e) {
+                this.log.debug(`Causal suggestion failed: ${e}`);
+            }
+        }
+        // Self-audit on failure: red-team the engine's own ML subsystems
+        // Skipped in mock mode and when insufficient historical data to avoid test overhead
+        if (orchestration.status === 'failed' && this.metaAdversarial && !this.options.mock) {
+            try {
+                const orchCount = this.executionGraph?.getNodesByType('orchestration').length ?? 0;
+                if (orchCount >= 5) {
+                    const report = this.metaAdversarial.runFullAudit({
+                        patternSelector: this.metaSelector ?? undefined,
+                        dropout: this.predictiveDropout ?? undefined,
+                        causal: this.causalEngine ? {
+                            estimateCausalEffect: (t, tv, cv) => 
+                            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                            this.causalEngine.estimateCausalEffect(t, tv, cv),
+                        } : undefined,
+                    });
+                    if (report.overallRisk !== 'low') {
+                        this.log.warn(`Self-audit found ${report.vulnerabilities.length} ML vulnerabilities (risk: ${report.overallRisk})`);
+                        this.bus.emit('system:warning', { type: 'self-audit', overallRisk: report.overallRisk, recommendations: report.recommendations }, 'engine');
+                    }
+                }
+            }
+            catch (e) {
+                this.log.debug(`Meta-adversarial audit failed: ${e}`);
+            }
+        }
         orchestration.completedAt = new Date();
-        continuousVerifier?.stop();
+        continuousVerifier.stop();
         // Clean up shared context files
         for (const f of this.sharedContextFiles) {
             try {
@@ -270,91 +476,17 @@ export class SwarmEngine {
             catch { }
         }
         this.sharedContextFiles = [];
-        // Build phase data once for all consumers (avoids 4x redundant mapping)
-        const phaseData = orchestration.phases.map(p => ({
-            name: p.config.name,
-            status: p.status,
-            agentCount: p.agents.length,
-            tokens: p.usage.totalTokens,
-            durationMs: p.usage.durationMs,
-            confidence: p.agents[0]?.result?.confidence ?? 'unknown',
-        }));
-        const traceData = {
-            orchestrationId: config.id,
-            task: config.description,
-            pattern: config.pattern,
-            status: orchestration.status,
-            totalTokens: orchestration.usage.totalTokens,
-            costUsd: orchestration.usage.costUsd,
-            durationMs: orchestration.usage.durationMs,
-            phaseCount: orchestration.phases.length,
-            agentCount: orchestration.phases.reduce((n, p) => n + p.agents.length, 0),
-            phases: phaseData,
-            reflections: this.reflections.map(r => redactSecrets(r)),
-        };
-        // Store execution trace for learning
-        if (this.traceStore) {
-            try {
-                this.traceStore.store(traceData);
-            }
-            catch { /* Don't fail orchestration if trace storage fails */ }
-            // Extract heuristics from the trace for future learning
-            if (this.heuristicStore) {
-                try {
-                    this.heuristicStore.extractFromTrace(traceData);
-                }
-                catch { /* Don't fail orchestration if heuristic extraction fails */ }
-            }
-            // Auto-compound knowledge from completed orchestrations
-            if (orchestration.status === 'completed' && this.compounder) {
-                try {
-                    const doc = this.compounder.extractFromTrace({ ...traceData, status: 'completed' }, this.reflections);
-                    if (doc)
-                        this.compounder.store(doc);
-                }
-                catch { /* Don't fail orchestration if compounding fails */ }
-            }
-            // Auto-capture outcomes to memory
+        // Record learning outcomes (traces, heuristics, model routing, autonomy)
+        await this.learningEngine.recordOutcome(orchestration, config, this.reflections);
+        // Train trajectory predictor on final outcome
+        if (this.trajectoryPredictor) {
             try {
-                const { traceToMemoryEntries } = await import('./traces.js');
-                const memEntries = traceToMemoryEntries(traceData);
-                // Store in memory if SwarmMemory is available
-                let memory;
-                try {
-                    const { SwarmMemory } = await import('../memory/index.js');
-                    memory = new SwarmMemory();
-                    for (const entry of memEntries) {
-                        memory.store({ type: entry.type, title: entry.title, content: entry.content });
-                    }
-                }
-                catch { /* Memory not available */ }
-                finally {
-                    try {
-                        memory?.close();
-                    }
-                    catch { }
-                }
+                this.trajectoryPredictor.recordOutcome(orchestration, orchestration.status === 'completed');
             }
-            catch { /* Don't fail orchestration */ }
-        }
-        // Record model outcomes for routing
-        if (this.modelRouter) {
-            for (const phase of orchestration.phases) {
-                for (const agent of phase.agents) {
-                    const model = agent.config.model;
-                    if (!model)
-                        continue;
-                    const success = agent.status === 'shutdown' || agent.status === 'idle';
-                    this.modelRouter.record(model, agent.config.name, success, success ? 1.0 : 0.0);
-                }
+            catch (e) {
+                this.log.debug(`Trajectory training failed: ${e}`);
             }
         }
-        // Record autonomy outcome
-        if (this.autonomyTracker) {
-            const success = orchestration.status === 'completed';
-            const intervention = orchestration.phases.some(p => p.config.requiresApproval);
-            this.autonomyTracker.record(config.pattern, success, intervention);
-        }
         // Compute total usage
         orchestration.usage = orchestration.phases.reduce((total, phase) => mergeUsageStats(total, phase.usage), emptyUsageStats());
         this.bus.emit(orchestration.status === 'completed' ? 'orchestration:complete' : 'orchestration:failed', {
@@ -377,7 +509,10 @@ export class SwarmEngine {
     async runPattern(patternName, task, options) {
         const pattern = this.patterns.get(patternName);
         if (!pattern) {
-            throw new Error(`Unknown pattern: ${patternName}. Available: ${this.patterns.list().map(p => p.name).join(', ')}`);
+            throw new Error(`Unknown pattern: ${patternName}. Available: ${this.patterns
+                .list()
+                .map((p) => p.name)
+                .join(', ')}`);
         }
         // Inject task into all agent prompts
         const config = {
@@ -385,9 +520,9 @@ export class SwarmEngine {
             name: `${patternName}: ${task.slice(0, 50)}`,
             description: task,
             pattern: patternName,
-            phases: pattern.phases.map(phase => ({
+            phases: pattern.phases.map((phase) => ({
                 ...phase,
-                agents: phase.agents.map(a => ({
+                agents: phase.agents.map((a) => ({
                     ...a,
                     prompt: `${task}\n\n${a.prompt ?? ''}`.trim(),
                 })),
@@ -396,17 +531,7 @@ export class SwarmEngine {
         };
         // Substitute pattern parameters into agent prompts
         if (options?.params && pattern.parameters) {
-            for (const phase of config.phases) {
-                for (const agent of phase.agents) {
-                    if (agent.prompt) {
-                        for (const [key, value] of Object.entries(options.params)) {
-                            // Use string replacement instead of regex to avoid ReDoS from user-controlled keys
-                            const placeholder = '${' + key + '}';
-                            agent.prompt = agent.prompt.split(placeholder).join(String(value));
-                        }
-                    }
-                }
-            }
+            this.substituteParams(config.phases, options.params);
         }
         return this.execute(config);
     }
@@ -417,37 +542,24 @@ export class SwarmEngine {
     plan(patternName, task, options) {
         const pattern = this.patterns.get(patternName);
         if (!pattern) {
-            throw new Error(`Unknown pattern: ${patternName}. Available: ${this.patterns.list().map(p => p.name).join(', ')}`);
+            throw new Error(`Unknown pattern: ${patternName}. Available: ${this.patterns
+                .list()
+                .map((p) => p.name)
+                .join(', ')}`);
         }
         // Check for similar past traces to inform planning
-        if (this.traceStore) {
-            try {
-                const similar = this.traceStore.findSimilar(task, 3);
-                if (similar.length > 0) {
-                    let traceContext = '';
-                    const best = similar.find(t => t.status === 'completed');
-                    if (best) {
-                        traceContext = `Past successful orchestration for similar task used pattern "${best.pattern}" with ${best.agentCount} agents (${best.totalTokens} tokens, ${Math.round(best.durationMs / 1000)}s).`;
-                    }
-                    const failed = similar.find(t => t.status === 'failed');
-                    if (failed && failed.reflections.length > 0) {
-                        traceContext += ` Previous attempt failed. Lessons: ${failed.reflections[0]?.slice(0, 200)}`;
-                    }
-                    if (traceContext) {
-                        this.log.info('Found similar past traces', { context: traceContext.slice(0, 200) });
-                    }
-                }
-            }
-            catch { /* Don't fail planning if trace lookup fails */ }
+        const traceContext = this.learningEngine.findSimilarTraces(task);
+        if (traceContext) {
+            this.log.info('Found similar past traces', { context: traceContext.slice(0, 200) });
         }
         const id = generateId();
-        const phases = pattern.phases.map(phase => ({
+        const phases = pattern.phases.map((phase) => ({
             name: phase.name,
             parallel: phase.parallel,
             dependsOn: phase.dependsOn,
             requiresApproval: phase.requiresApproval,
             tokenBudget: phase.tokenBudget,
-            agents: phase.agents.map(a => {
+            agents: phase.agents.map((a) => {
                 const agentConfig = this.registry.getConfig(a.type);
                 return {
                     name: a.name,
@@ -462,14 +574,7 @@ export class SwarmEngine {
         }));
         // Substitute pattern parameters if provided
         if (options?.params && pattern.parameters) {
-            for (const phase of phases) {
-                for (const agent of phase.agents) {
-                    for (const [key, value] of Object.entries(options.params)) {
-                        const placeholder = '${' + key + '}';
-                        agent.prompt = agent.prompt.split(placeholder).join(String(value));
-                    }
-                }
-            }
+            this.substituteParams(phases, options.params);
         }
         const planResult = {
             id,
@@ -485,33 +590,58 @@ export class SwarmEngine {
         if (rewrites.length > 0) {
             this.log.info('Plan rewritten', { rewrites });
         }
-        // Retrieve relevant heuristics and inject into first phase
-        if (this.heuristicStore) {
+        // Inject learned heuristics and past solutions into the plan
+        this.learningEngine.injectHeuristics(rewrittenPlan);
+        this.learningEngine.injectSolutions(rewrittenPlan);
+        // Meta-pattern recommendation (Feature 4: Meta-Pattern Selector)
+        if (this.metaSelector) {
             try {
-                const heuristics = this.heuristicStore.retrieve(task);
-                if (heuristics.length > 0) {
-                    const context = this.heuristicStore.formatForContext(heuristics);
-                    for (const agent of rewrittenPlan.phases[0]?.agents ?? []) {
-                        agent.prompt = `${context}\n\n${agent.prompt}`;
-                    }
-                    this.log.info('Injected heuristics into plan', { count: heuristics.length });
+                const recommendation = this.metaSelector.selectPattern(task);
+                if (recommendation.confidence > 0.6) {
+                    this.log.info(`Meta-selector recommends: ${recommendation.pattern} (${Math.round(recommendation.confidence * 100)}% confidence)`);
                 }
             }
-            catch { /* Don't fail planning if heuristic retrieval fails */ }
+            catch (e) {
+                this.log.debug(`Meta-pattern selection failed: ${e}`);
+            }
         }
-        // Inject relevant past solutions from knowledge compounder (singleton)
-        if (this.compounder) {
+        // Task discovery: log discovered backlog items from historical failure patterns (skip in mock mode)
+        if (this.taskDiscovery && !this.options.mock) {
             try {
-                const solutions = this.compounder.findRelevant(task, 3);
-                if (solutions.length > 0) {
-                    const solutionContext = this.compounder.formatForContext(solutions);
-                    for (const agent of rewrittenPlan.phases[0]?.agents ?? []) {
-                        agent.prompt = `${solutionContext}\n\n${agent.prompt}`;
-                    }
-                    this.log.info('Injected past solutions into plan', { count: solutions.length });
+                const backlog = this.taskDiscovery.generateBacklog(3);
+                if (backlog.length > 0) {
+                    this.log.info(`Task discovery: ${backlog.length} actionable items found (top: ${backlog[0]?.title})`);
                 }
             }
-            catch { /* Don't fail planning if solution retrieval fails */ }
+            catch (e) {
+                this.log.debug(`Task discovery failed: ${e}`);
+            }
+        }
+        // Pattern synthesis: log insights from topology analysis (skip in mock mode)
+        if (this.patternSynthesizer && !this.options.mock) {
+            try {
+                const insights = this.patternSynthesizer.analyzeTopologyDiffs(3);
+                if (insights.length > 0) {
+                    const top = insights[0];
+                    this.log.info(`Pattern synthesis top insight: "${top.feature}" (correlation: ${top.successCorrelation.toFixed(2)}, n=${top.sampleSize})`);
+                }
+            }
+            catch (e) {
+                this.log.debug(`Pattern synthesis failed: ${e}`);
+            }
+        }
+        // Pattern synthesis: suggest novel pattern from historical success topology
+        if (this.patternSynthesizer && !this.options.mock) {
+            try {
+                const synthesized = this.patternSynthesizer.synthesizePattern(task);
+                if (synthesized && synthesized.confidence > 0.5) {
+                    this.log.info(`Pattern synthesis suggests: ${synthesized.name} (${Math.round(synthesized.confidence * 100)}% confidence)`);
+                    this.bus.emit('system:warning', { type: 'pattern-synthesis', pattern: synthesized }, 'engine');
+                }
+            }
+            catch (e) {
+                this.log.debug(`Pattern synthesis suggestion failed: ${e}`);
+            }
         }
         // Search over candidate plans to find the best one
         const searcher = new PlanSearcher(this.costModel, this.traceStore ?? undefined);
@@ -525,7 +655,7 @@ export class SwarmEngine {
         bestPlan.estimates = this.costModel.estimate(bestPlan);
         bestPlan.rewrites = rewrites;
         if (candidates.length > 1) {
-            bestPlan._alternatives = candidates.slice(1).map(c => ({
+            bestPlan._alternatives = candidates.slice(1).map((c) => ({
                 source: c.source,
                 costUsd: c.cost.costUsd,
                 quality: c.cost.qualityScore,
@@ -544,11 +674,31 @@ export class SwarmEngine {
             const dropout = this.replanner.evaluateDropout(orchestration, phase.config);
             if (dropout) {
                 this.log.info(`Agent dropout: ${dropout.reason}`);
-                phase.config.agents = phase.config.agents.filter(a => dropout.agentsToKeep.includes(a.name));
+                phase.config.agents = phase.config.agents.filter((a) => dropout.agentsToKeep.includes(a.name));
                 this.bus.emit('system:warning', { type: 'agent-dropout', ...dropout }, 'adaptive');
             }
         }
+        // Predictive dropout from ML model (Feature 5: Predictive Dropout)
+        if (this.predictiveDropout && phase.config.agents.length > 1) {
+            try {
+                const recommendations = this.predictiveDropout.predictRedundant(phase.config.agents.map(a => ({ id: a.name || a.type, agentType: a.type })), phase.config.kind ?? 'implement', 0, orchestration.config.pattern || '');
+                const toDrop = recommendations.filter(r => r.shouldDropOut);
+                if (toDrop.length > 0) {
+                    this.log.info(`Predictive dropout: skipping ${toDrop.map(d => d.agentType).join(', ')}`);
+                    phase.config.agents = phase.config.agents.filter(a => !toDrop.some(d => d.agentType === a.type));
+                }
+            }
+            catch (e) {
+                this.log.debug(`Predictive dropout failed: ${e}`);
+            }
+        }
         const maxRetries = phase.config.maxRetries ?? 2;
+        const runContext = {
+            phaseOutputs: this.phaseOutputs,
+            livingSpec: this.livingSpec,
+            sharedContextFiles: this.sharedContextFiles,
+            cwd: this.options.cwd,
+        };
         this.log.info(`Phase started: ${phase.config.name}`, {
             agents: phase.config.agents.length,
             parallel: phase.config.parallel,
@@ -562,7 +712,7 @@ export class SwarmEngine {
         try {
             if (phase.config.parallel) {
                 // Run all agents in parallel (with retry)
-                const results = await Promise.allSettled(phase.config.agents.map(assignment => this.executeAgentWithRetry(assignment, phase, orchestration, maxRetries)));
+                const results = await Promise.allSettled(phase.config.agents.map((assignment) => this.agentRunner.executeAgentWithRetry(assignment, phase, orchestration, maxRetries, runContext)));
                 // Log cache savings estimate for parallel phases
                 if (phase.config.agents.length > 1) {
                     const phaseContext = this.compaction.buildContextPrefix(this.phaseOutputs);
@@ -573,7 +723,7 @@ export class SwarmEngine {
                     }
                 }
                 // Check for failures
-                const failures = results.filter(r => r.status === 'rejected');
+                const failures = results.filter((r) => r.status === 'rejected');
                 if (failures.length > 0) {
                     const failureCount = failures.length;
                     this.log.warn(`Phase had ${failureCount} agent failures`, {
@@ -592,9 +742,16 @@ export class SwarmEngine {
                 }
             }
             else {
-                // Run agents sequentially (with retry)
+                // Run agents sequentially (with retry) — accumulate intra-phase outputs for live re-routing (Feature 1)
+                const intraPhaseOutputs = new Map();
                 for (const assignment of phase.config.agents) {
-                    await this.executeAgentWithRetry(assignment, phase, orchestration, maxRetries);
+                    const agentInstance = await this.agentRunner.executeAgentWithRetry(assignment, phase, orchestration, maxRetries, { ...runContext, intraPhaseOutputs });
+                    // After agent completes, add its output to intra-phase buffer for next sequential agent
+                    if (agentInstance?.result?.output) {
+                        const existing = intraPhaseOutputs.get(phase.config.name) || [];
+                        existing.push(agentInstance.result.output);
+                        intraPhaseOutputs.set(phase.config.name, existing);
+                    }
                     // Check phase token budget after each agent
                     if (phase.config.tokenBudget) {
                         const phaseTokens = phase.agents.reduce((sum, a) => sum + a.usage.totalTokens, 0);
@@ -615,8 +772,23 @@ export class SwarmEngine {
         }
         phase.completedAt = new Date();
         phase.usage = phase.agents.reduce((total, agent) => mergeUsageStats(total, agent.usage), emptyUsageStats());
+        // Record review feedback into the knowledge graph (Feature 7)
+        if (phase.config.kind === 'review' && this.reviewFeedback) {
+            for (const agent of phase.agents) {
+                if (agent.result?.output) {
+                    try {
+                        this.reviewFeedback.recordFindings(orchestration.config.id, phase.config.name, agent.result.output, agent.config.name);
+                        const findings = parseReviewOutput(agent.result.output, agent.config.name);
+                        this.reviewFeedback.adjustContextWeights(orchestration.config.id, findings);
+                    }
+                    catch (e) {
+                        this.log.debug(`Review feedback recording failed: ${e}`);
+                    }
+                }
+            }
+        }
         // Run verification after implementation phases
-        if (this.verifier && (phase.config.name.includes('implement') || phase.config.name.includes('integrate'))) {
+        if (phase.config.kind === 'implement' || phase.config.kind === 'integrate') {
             const results = await this.verifier.verify();
             const summary = Verifier.summarize(results);
             this.log.info(`Verification: ${summary}`, { phase: phase.config.name });
@@ -647,8 +819,8 @@ export class SwarmEngine {
         });
         // Collect agent outputs for inter-phase context
         const outputs = phase.agents
-            .filter(a => a.result?.output)
-            .map(a => `[${a.config.name}]: ${a.result.output.slice(0, 2000)}`);
+            .filter((a) => a.result?.output)
+            .map((a) => `[${a.config.name}]: ${a.result.output.slice(0, 2000)}`);
         // Update living spec with phase outputs
         if (this.livingSpec) {
             this.livingSpec = updateSpecFromPhase(this.livingSpec, phase.config.name, outputs);
@@ -665,121 +837,6 @@ export class SwarmEngine {
         }
         return outputs;
     }
-    /**
-     * Execute a single agent within a phase.
-     */
-    async executeAgent(assignment, phase, orchestration) {
-        // Spawn from registry, passing all assignment overrides
-        const instance = this.registry.spawn(assignment.type, {
-            name: assignment.name,
-            model: assignment.model,
-            backend: assignment.backend,
-            backendModel: assignment.backendModel,
-        });
-        phase.agents.push(instance);
-        // Build task prompt with cache-friendly ordering:
-        // Shared prefix (orchestration + phase context) comes first for API cache hits,
-        // agent-specific content (definition + task) comes last.
-        const orchestrationContext = `## Orchestration: ${orchestration.config.name}\nPattern: ${orchestration.config.pattern}`;
-        const phaseContext = this.compaction.buildContextPrefix(this.phaseOutputs);
-        const agentPrompt = instance.config.prompt;
-        const baseTask = assignment.prompt || `Execute task as ${assignment.type}`;
-        const optimized = optimizeForCache(orchestrationContext, phaseContext, agentPrompt, baseTask);
-        // For parallel phases with large shared prefix, use shared context file
-        const sharedContextPath = phase.config.parallel
-            ? this.writeSharedContext(optimized.sharedPrefix)
-            : null;
-        // Cascade model selection: if no explicit model override, pick starting tier by complexity
-        if (!assignment.model) {
-            const complexity = classifyTaskComplexity(baseTask);
-            const startTier = selectStartingTier(complexity);
-            if (startTier !== instance.config.model) {
-                instance.config.model = startTier;
-            }
-        }
-        // Inject living spec into task context
-        const specPrefix = this.livingSpec ? formatLivingSpec(this.livingSpec) + '\n\n' : '';
-        const task = sharedContextPath
-            ? `Read the shared context file at ${sharedContextPath} first, then:\n\n${specPrefix}${optimized.agentSuffix}`
-            : `${specPrefix}${optimized.full}`;
-        // Transition to running
-        this.registry.updateStatus(instance.id, 'running');
-        // Create worktree if enabled and assignment requests it
-        let worktreeId;
-        if (this.worktreeManager && assignment.worktree) {
-            try {
-                const wt = this.worktreeManager.create(instance.id, assignment.name);
-                worktreeId = wt.id;
-                this.bus.emit('worktree:created', { agentId: instance.id, worktreeId: wt.id, path: wt.path }, instance.id);
-            }
-            catch (err) {
-                this.log.warn(`Failed to create worktree for ${assignment.name}: ${err}`);
-            }
-        }
-        // Execute with timeout
-        const timeoutMs = (instance.config.timeout ?? 300) * 1000; // default 5 min
-        let timer;
-        const timeoutPromise = new Promise((_, reject) => {
-            timer = setTimeout(() => reject(new Error(`Agent timeout after ${timeoutMs / 1000}s`)), timeoutMs);
-        });
-        try {
-            await Promise.race([
-                this.executor.execute(instance, task),
-                timeoutPromise,
-            ]);
-            clearTimeout(timer);
-            // Merge worktree if created
-            if (worktreeId && this.worktreeManager) {
-                try {
-                    this.worktreeManager.merge(worktreeId);
-                    this.bus.emit('worktree:merged', { agentId: instance.id, worktreeId }, instance.id);
-                }
-                catch (err) {
-                    this.log.warn(`Failed to merge worktree ${worktreeId}: ${err}`);
-                }
-            }
-            // Transition to idle (completed successfully)
-            this.registry.updateStatus(instance.id, 'idle');
-            // Then shutdown
-            this.registry.updateStatus(instance.id, 'shutdown');
-            return instance;
-        }
-        catch (error) {
-            clearTimeout(timer);
-            if (String(error).includes('timeout')) {
-                this.registry.updateStatus(instance.id, 'timeout');
-            }
-            else {
-                this.registry.updateStatus(instance.id, 'error');
-            }
-            instance.error = error instanceof Error ? error.message : String(error);
-            throw error;
-        }
-    }
-    async executeAgentWithRetry(assignment, phase, orchestration, maxRetries = 2) {
-        let lastError;
-        for (let attempt = 0; attempt <= maxRetries; attempt++) {
-            try {
-                return await this.executeAgent(assignment, phase, orchestration);
-            }
-            catch (error) {
-                lastError = error instanceof Error ? error : new Error(String(error));
-                // Remove the failed instance from phase agents to prevent duplicates on retry
-                const failedIndex = phase.agents.findIndex(a => a.config.name === assignment.name && (a.status === 'error' || a.status === 'timeout'));
-                if (failedIndex >= 0)
-                    phase.agents.splice(failedIndex, 1);
-                if (attempt < maxRetries) {
-                    const delayMs = Math.min(1000 * Math.pow(2, attempt) + Math.random() * 1000, 30000);
-                    this.log.warn(`Agent failed, retrying in ${Math.round(delayMs / 1000)}s (attempt ${attempt + 1}/${maxRetries})`, {
-                        agent: assignment.name,
-                        error: lastError.message,
-                    });
-                    await new Promise(resolve => setTimeout(resolve, delayMs));
-                }
-            }
-        }
-        throw lastError;
-    }
     // ─── Resume ────────────────────────────────────────────────────
     /**
      * Resume an orchestration from a checkpoint.
@@ -817,10 +874,8 @@ export class SwarmEngine {
         const { createInterface } = await import('readline');
         const rl = createInterface({ input: process.stdin, output: process.stdout });
         return new Promise((resolve) => {
-            const agentList = agents.map(a => `  ${a.name} (${a.type})`).join('\n');
-            rl.question(`\n⏸  Phase "${phaseName}" requires approval.\n` +
-                `   Agents:\n${agentList}\n` +
-                `   Proceed? [Y/n] `, (answer) => {
+            const agentList = agents.map((a) => `  ${a.name} (${a.type})`).join('\n');
+            rl.question(`\n⏸  Phase "${phaseName}" requires approval.\n` + `   Agents:\n${agentList}\n` + `   Proceed? [Y/n] `, (answer) => {
                 rl.close();
                 resolve(answer.toLowerCase() !== 'n');
             });
@@ -856,26 +911,7 @@ export class SwarmEngine {
                     // Best effort — lifecycle may not allow transition
                 }
             }
-            try {
-                this.stats.close();
-            }
-            catch { /* best effort */ }
-            try {
-                this.traceStore?.close();
-            }
-            catch { /* best effort */ }
-            try {
-                this.heuristicStore?.close();
-            }
-            catch { /* best effort */ }
-            try {
-                this.modelRouter?.close();
-            }
-            catch { }
-            try {
-                this.autonomyTracker?.close();
-            }
-            catch { }
+            this.close();
             this.bus.emit('system:warning', { message: 'Graceful shutdown complete' }, 'engine');
         };
         process.on('SIGINT', handler);
@@ -889,28 +925,18 @@ export class SwarmEngine {
         }
         this.signalHandlers = [];
     }
-    /** Close all resources (stats, traces, heuristics, model router, autonomy tracker). */
+    /** Close all resources (stats, traces, heuristics, model router, autonomy tracker, database). */
     close() {
         try {
             this.stats.close();
         }
         catch { }
+        this.learningEngine.close();
         try {
-            this.traceStore?.close();
-        }
-        catch { }
-        try {
-            this.heuristicStore?.close();
-        }
-        catch { }
-        try {
-            this.modelRouter?.close();
-        }
-        catch { }
-        try {
-            this.autonomyTracker?.close();
+            this.executionGraph?.close();
         }
         catch { }
+        closeDatabase();
         this.removeSignalHandlers();
     }
     // ─── Queries ───────────────────────────────────────────────────
@@ -930,33 +956,46 @@ export class SwarmEngine {
      * Get active orchestrations.
      */
     getActiveOrchestrations() {
-        return [...this.orchestrations.values()].filter(o => o.status === 'running' || o.status === 'paused');
+        return [...this.orchestrations.values()].filter((o) => o.status === 'running' || o.status === 'paused');
     }
     get patternRegistry() {
         return this.patterns;
     }
-    // ─── Shared Context ────────────────────────────────────────────
+    get graph() {
+        return this.executionGraph;
+    }
+    // ─── Helpers ───────────────────────────────────────────────────
+    tryInit(factory, name) {
+        try {
+            return factory();
+        }
+        catch (e) {
+            this.log.warn(`${name} init failed`, { error: e instanceof Error ? e.message : e });
+            return null;
+        }
+    }
     /**
-     * Write shared context to a file for parallel agents to read,
-     * avoiding duplicating large context in each agent's prompt.
+     * Substitute ${key} placeholders in agent prompts.
+     * Uses string split/join instead of regex to avoid ReDoS from user-controlled keys.
      */
-    writeSharedContext(context) {
-        if (!context || context.length < 2000)
-            return null; // Not worth sharing for small contexts
-        const contextDir = join(this.options.cwd ?? process.cwd(), '.swarm');
-        if (!existsSync(contextDir))
-            mkdirSync(contextDir, { recursive: true });
-        const contextPath = join(contextDir, `shared-context-${randomUUID().slice(0, 8)}.md`);
-        writeFileSync(contextPath, redactSecrets(context), { mode: 0o600 });
-        this.sharedContextFiles.push(contextPath);
-        return contextPath;
+    substituteParams(phases, params) {
+        for (const phase of phases) {
+            for (const agent of phase.agents) {
+                if (agent.prompt) {
+                    for (const [key, value] of Object.entries(params)) {
+                        const placeholder = '${' + key + '}';
+                        agent.prompt = agent.prompt.split(placeholder).join(String(value));
+                    }
+                }
+            }
+        }
     }
     // ─── Factory ───────────────────────────────────────────────────
     createInstance(config) {
         return {
             config,
             status: 'pending',
-            phases: config.phases.map(phaseConfig => ({
+            phases: config.phases.map((phaseConfig) => ({
                 config: phaseConfig,
                 status: 'pending',
                 agents: [],