principles-disciple 1.7.6 → 1.7.8

package/dist/core/promotion-gate.js

@@ -0,0 +1,529 @@
+/**
+ * Promotion Gate — Checkpoint Promotion State Machine and Gate Logic
+ * ==================================================================
+ *
+ * PURPOSE: Control when a checkpoint can advance from training → shadow → promotion.
+ * Training success alone is not enough — a checkpoint must prove it improves
+ * bounded worker behavior under the existing offline benchmark and does not
+ * regress runtime safety signals.
+ *
+ * PROMOTION STATES:
+ *   - rejected:         The checkpoint must not be routed
+ *   - candidate_only:  The checkpoint is valid but not yet ready for shadow
+ *   - shadow_ready:   The checkpoint may enter controlled shadow rollout
+ *   - promotable:     The checkpoint may replace the active checkpoint
+ *
+ * STATE TRANSITIONS:
+ *   training_completed
+ *       ↓
+ *   candidate_only  ←── (eval attached, lineage complete)
+ *       ↓
+ *   shadow_ready   ←── (positive delta, safe constraints)
+ *       ↓
+ *   promotable      ←── (shadow window passed, orchestrator review passed)
+ *       ↓
+ *   deployed
+ *
+ * PRIMARY OBJECTIVE:
+ *   maximize reduced_prompt_holdout_delta
+ *
+ * CONSTRAINT METRICS (must all pass for promotion):
+ *   - arbiterRejectRate <= baseline + allowedMargin
+ *   - executabilityRejectRate <= baseline + allowedMargin
+ *   - reviewedSubsetQuality >= baseline
+ *   - routingScopeNotExpanded == true
+ *
+ * DESIGN CONSTRAINTS:
+ *   - No automatic promotion without explicit gate approval
+ *   - Orchestrator review remains mandatory for all promotions
+ *   - Rollback path must be always available
+ *   - First rollout limited to `local-reader` only
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { withLock } from '../utils/file-lock.js';
+import { getCheckpoint, getEvalSummary, } from './model-training-registry.js';
+import { computeShadowStats, } from './shadow-observation-registry.js';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/**
+ * Candidate delta must exceed this to enter shadow_ready.
+ */
+export const DEFAULT_MIN_DELTA = 0.05;
+/**
+ * Default allowed margin for constraint metrics.
+ * Constraint metrics can regress by at most this amount.
+ */
+export const DEFAULT_ALLOWED_MARGIN = 0.05;
+/**
+ * Allowed worker profiles for Phase 7 shadow rollout.
+ * Only bounded local workers eligible. local-reader first, local-editor deferred.
+ */
+const ALLOWED_ROLLOUT_PROFILES = ['local-reader'];
+/**
+ * Registry file for promotion records.
+ */
+const PROMOTION_REGISTRY_FILE = 'promotion-registry.json';
+/**
+ * Minimum shadow window duration in milliseconds.
+ * A checkpoint must remain in shadow_ready for at least this duration
+ * before it can be promoted to promotable.
+ *
+ * Phase 7 default: 1 hour (3600000 ms)
+ * This gives time for real-world feedback before full promotion.
+ */
+export const MIN_SHADOW_WINDOW_MS = 60 * 60 * 1000; // 1 hour
+// ---------------------------------------------------------------------------
+// Registry Path
+// ---------------------------------------------------------------------------
+function getRegistryPath(stateDir) {
+    return path.join(stateDir, PROMOTION_REGISTRY_FILE);
+}
+/**
+ * Ensure the registry directory exists.
+ */
+function ensureRegistryDir(stateDir) {
+    const registryPath = getRegistryPath(stateDir);
+    const dir = path.dirname(registryPath);
+    if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+    }
+}
+// ---------------------------------------------------------------------------
+// File Operations
+// ---------------------------------------------------------------------------
+/**
+ * Read the registry from disk. Returns empty registry if missing.
+ */
+function readRegistry(stateDir) {
+    const registryPath = getRegistryPath(stateDir);
+    if (!fs.existsSync(registryPath)) {
+        return { promotions: [] };
+    }
+    try {
+        const content = fs.readFileSync(registryPath, 'utf-8');
+        return JSON.parse(content);
+    }
+    catch (err) {
+        console.warn(`[promotion-gate] Registry corrupted at ${registryPath}, recovering with empty state: ${String(err)}`);
+        return { promotions: [] };
+    }
+}
+/**
+ * Write the registry to disk atomically.
+ */
+function writeRegistry(stateDir, registry) {
+    ensureRegistryDir(stateDir);
+    const registryPath = getRegistryPath(stateDir);
+    const tmpPath = `${registryPath}.tmp`;
+    fs.writeFileSync(tmpPath, JSON.stringify(registry, null, 2), 'utf-8');
+    fs.renameSync(tmpPath, registryPath);
+}
+/**
+ * Execute a read-modify-write under an exclusive file lock.
+ */
+function withPromotionRegistryLock(stateDir, fn) {
+    const registryPath = getRegistryPath(stateDir);
+    return withLock(registryPath, () => {
+        const registry = readRegistry(stateDir);
+        return fn(registry);
+    });
+}
+/**
+ * Evaluate whether a checkpoint passes the promotion gate.
+ *
+ * @param stateDir - Workspace state directory
+ * @param params - Evaluation parameters
+ * @returns PromotionGateResult with pass/fail and details
+ *
+ * FAIL-CLOSED: Returns { passes: false } if:
+ *   - No eval attached to checkpoint
+ *   - Delta is negative or below threshold
+ *   - Any constraint metric regresses beyond allowed margin
+ *   - Profile is not in allowed rollout list
+ */
+export function evaluatePromotionGate(stateDir, params) {
+    const { checkpointId, targetProfile, baselineMetrics, minDelta = DEFAULT_MIN_DELTA, allowedMargin = DEFAULT_ALLOWED_MARGIN, } = params;
+    const blockers = [];
+    const constraintChecks = [];
+    // --- Check 1: Checkpoint exists ---
+    const checkpoint = getCheckpoint(stateDir, checkpointId);
+    if (!checkpoint) {
+        blockers.push(`Checkpoint not found: ${checkpointId}`);
+        return {
+            passes: false,
+            blockers,
+            constraintChecks: [],
+            deltaCheck: { actual: 0, threshold: minDelta, passed: false },
+        };
+    }
+    // --- Check 2: Has eval attached ---
+    if (!checkpoint.lastEvalSummaryRef) {
+        blockers.push(`Checkpoint ${checkpointId} has no eval summary attached. ` +
+            `Run benchmark evaluation before promotion gate.`);
+        return {
+            passes: false,
+            blockers,
+            constraintChecks: [],
+            deltaCheck: { actual: 0, threshold: minDelta, passed: false },
+        };
+    }
+    // --- Check 3: Get eval summary ---
+    const evalSummary = getEvalSummary(stateDir, checkpoint.lastEvalSummaryRef);
+    if (!evalSummary) {
+        blockers.push(`Eval summary '${checkpoint.lastEvalSummaryRef}' not found. ` +
+            `Cannot evaluate promotion gate without valid eval.`);
+        return {
+            passes: false,
+            blockers,
+            constraintChecks: [],
+            deltaCheck: { actual: 0, threshold: minDelta, passed: false },
+        };
+    }
+    // --- Check 4: Delta must be positive and above threshold ---
+    const delta = evalSummary.delta;
+    const deltaCheck = {
+        actual: delta,
+        threshold: minDelta,
+        passed: delta >= minDelta,
+    };
+    if (!deltaCheck.passed) {
+        blockers.push(`Reduced-prompt holdout delta (${delta.toFixed(4)}) is below threshold (${minDelta}). ` +
+            `Checkpoint must show positive improvement to be promoted.`);
+    }
+    // --- Check 5: Arbiter reject rate constraint ---
+    // PREFER real shadow evidence over eval verdict proxy
+    // Shadow evidence comes from actual runtime routing decisions
+    const shadowStats = computeShadowStats(stateDir, { checkpointId });
+    let arbiterRejectRate;
+    let arbiterRejectSource;
+    if (shadowStats && shadowStats.isStatisticallySignificant) {
+        // Use real shadow evidence: reject rate from shadow routing
+        arbiterRejectRate = shadowStats.rejectRate;
+        arbiterRejectSource = 'shadow';
+    }
+    else {
+        // Fall back to eval verdict proxy (Phase 7 initial state)
+        // This is a coarse approximation: 'fail' verdict maps to 100% reject
+        arbiterRejectRate = evalSummary.verdict === 'fail' ? 1 : 0;
+        arbiterRejectSource = 'eval-proxy';
+    }
+    const arbiterRejectCheck = {
+        constraint: 'arbiterRejectRate',
+        actual: arbiterRejectRate,
+        baseline: baselineMetrics.arbiterRejectRate,
+        threshold: baselineMetrics.arbiterRejectRate + allowedMargin,
+        passed: arbiterRejectRate <= baselineMetrics.arbiterRejectRate + allowedMargin,
+        source: arbiterRejectSource,
+    };
+    constraintChecks.push(arbiterRejectCheck);
+    if (!arbiterRejectCheck.passed) {
+        blockers.push(`arbiterRejectRate regressed: ${arbiterRejectRate.toFixed(4)} > ${arbiterRejectCheck.threshold.toFixed(4)} ` +
+            `(baseline: ${baselineMetrics.arbiterRejectRate.toFixed(4)}, margin: ${allowedMargin}) ` +
+            `[source: ${arbiterRejectSource}${shadowStats ? `, n=${shadowStats.totalCount}` : ''}]`);
+    }
+    // --- Check 6: Executability reject rate constraint ---
+    // PREFER real shadow evidence: escalation rate + profile rejection rate
+    let executabilityRejectRate;
+    let executabilityRejectSource;
+    if (shadowStats && shadowStats.isStatisticallySignificant) {
+        // Use real shadow evidence: escalation + profile rejection from routing
+        executabilityRejectRate = shadowStats.escalationRate + shadowStats.profileRejectedRate;
+        executabilityRejectSource = 'shadow';
+    }
+    else {
+        // Fall back to eval verdict proxy
+        // This is a coarse approximation
+        executabilityRejectRate = evalSummary.verdict === 'fail' ? 0.1 : 0;
+        executabilityRejectSource = 'eval-proxy';
+    }
+    const executabilityRejectCheck = {
+        constraint: 'executabilityRejectRate',
+        actual: executabilityRejectRate,
+        baseline: baselineMetrics.executabilityRejectRate,
+        threshold: baselineMetrics.executabilityRejectRate + allowedMargin,
+        passed: executabilityRejectRate <= baselineMetrics.executabilityRejectRate + allowedMargin,
+        source: executabilityRejectSource,
+    };
+    constraintChecks.push(executabilityRejectCheck);
+    if (!executabilityRejectCheck.passed) {
+        blockers.push(`executabilityRejectRate regressed: ${executabilityRejectRate.toFixed(4)} > ${executabilityRejectCheck.threshold.toFixed(4)} ` +
+            `(baseline: ${baselineMetrics.executabilityRejectRate.toFixed(4)}, margin: ${allowedMargin}) ` +
+            `[source: ${executabilityRejectSource}${shadowStats ? `, n=${shadowStats.totalCount}` : ''}]`);
+    }
+    // --- Check 7: Reviewed subset quality constraint ---
+    // Use eval score as proxy for quality
+    const reviewedSubsetQuality = evalSummary.candidateScore;
+    const qualityCheck = {
+        constraint: 'reviewedSubsetQuality',
+        actual: reviewedSubsetQuality,
+        baseline: baselineMetrics.reviewedSubsetQuality,
+        threshold: baselineMetrics.reviewedSubsetQuality - allowedMargin,
+        passed: reviewedSubsetQuality >= baselineMetrics.reviewedSubsetQuality - allowedMargin,
+    };
+    constraintChecks.push(qualityCheck);
+    if (!qualityCheck.passed) {
+        blockers.push(`reviewedSubsetQuality regressed: ${reviewedSubsetQuality.toFixed(4)} < ${qualityCheck.threshold.toFixed(4)} ` +
+            `(baseline: ${baselineMetrics.reviewedSubsetQuality.toFixed(4)})`);
+    }
+    // --- Determine if passes ---
+    const allPassed = deltaCheck.passed &&
+        arbiterRejectCheck.passed &&
+        executabilityRejectCheck.passed &&
+        qualityCheck.passed;
+    // --- Suggest state based on checks ---
+    let suggestedState;
+    if (allPassed) {
+        suggestedState = 'candidate_only';
+        // If delta is strong enough, could be shadow_ready directly
+        if (delta >= minDelta * 2) {
+            suggestedState = 'shadow_ready';
+        }
+    }
+    else {
+        suggestedState = 'rejected';
+    }
+    return {
+        passes: allPassed,
+        suggestedState,
+        blockers,
+        constraintChecks,
+        deltaCheck,
+    };
+}
+/**
+ * Advance a checkpoint's promotion state.
+ *
+ * @param stateDir - Workspace state directory
+ * @param params - Advancement parameters
+ * @returns The updated PromotionRecord
+ *
+ * @throws Error if gate evaluation fails
+ * @throws Error if state transition is not allowed
+ */
+export function advancePromotion(stateDir, params) {
+    const { checkpointId, targetProfile, baselineMetrics, orchestratorReviewPassed = false, reviewNote, minDelta = DEFAULT_MIN_DELTA, allowedMargin = DEFAULT_ALLOWED_MARGIN, } = params;
+    // First, evaluate the gate
+    const gateResult = evaluatePromotionGate(stateDir, {
+        checkpointId,
+        targetProfile,
+        baselineMetrics,
+        minDelta,
+        allowedMargin,
+    });
+    // Find existing promotion record (if any) - need this to know current state
+    return withPromotionRegistryLock(stateDir, (registry) => {
+        const now = new Date().toISOString();
+        const existingIdx = registry.promotions.findIndex((p) => p.checkpointId === checkpointId);
+        const currentState = existingIdx >= 0 ? registry.promotions[existingIdx].state : 'candidate_only';
+        // Determine the target state based on current state, gate result, and review
+        //
+        // STATE TRANSITION RULES:
+        // - Any state → rejected: if gate fails
+        // - rejected/candidate_only → candidate_only: if gate passes but no review yet
+        // - shadow_ready → promotable: if gate passes + review + shadow window elapsed
+        // - rejected → candidate_only/shadow_ready: allowed via re-evaluation
+        //   (new eval data may reverse a previous rejection)
+        //
+        let targetState;
+        if (!gateResult.passes) {
+            targetState = 'rejected';
+        }
+        else if (!orchestratorReviewPassed) {
+            // Gate passed but need orchestrator review before shadow_ready
+            // Review is ALWAYS required to reach shadow_ready, regardless of delta strength
+            targetState = 'candidate_only';
+        }
+        else {
+            // Gate passed and orchestrator review passed: advance one level
+            // Only go to promotable if already at shadow_ready; otherwise advance to shadow_ready
+            if (currentState === 'shadow_ready') {
+                // Check shadow window duration before allowing promotion
+                const existing = existingIdx >= 0 ? registry.promotions[existingIdx] : null;
+                const shadowStartedAt = existing?.shadowStartedAt;
+                if (shadowStartedAt) {
+                    const shadowElapsed = Date.now() - new Date(shadowStartedAt).getTime();
+                    if (shadowElapsed < MIN_SHADOW_WINDOW_MS) {
+                        // Shadow window not elapsed yet — stay at shadow_ready
+                        targetState = 'shadow_ready';
+                    }
+                    else {
+                        // Shadow window elapsed — allow promotion to promotable
+                        targetState = 'promotable';
+                    }
+                }
+                else {
+                    // No shadowStartedAt, allow promotion (backward compat)
+                    targetState = 'promotable';
+                }
+            }
+            else {
+                // At candidate_only (or new), advance to shadow_ready
+                targetState = 'shadow_ready';
+            }
+        }
+        // Get previous promotion ID for chain
+        const previousPromotionId = existingIdx >= 0
+            ? registry.promotions[existingIdx].promotionId
+            : undefined;
+        // Get checkpoint info for lineage
+        const checkpoint = getCheckpoint(stateDir, checkpointId);
+        const evalSummary = checkpoint?.lastEvalSummaryRef
+            ? getEvalSummary(stateDir, checkpoint.lastEvalSummaryRef)
+            : null;
+        // Get current delta
+        const reducedPromptDelta = evalSummary?.delta ?? 0;
+        // Create/update promotion record
+        const promotion = {
+            promotionId: existingIdx >= 0
+                ? registry.promotions[existingIdx].promotionId
+                : crypto.randomUUID(),
+            checkpointId,
+            state: targetState,
+            targetProfile,
+            targetModelFamily: checkpoint?.targetModelFamily ?? 'unknown',
+            reducedPromptDelta,
+            constraintMetrics: {
+                arbiterRejectRate: evalSummary?.verdict === 'fail' ? 1 : 0,
+                executabilityRejectRate: evalSummary?.verdict === 'fail' ? 0.1 : 0,
+                reviewedSubsetQuality: evalSummary?.candidateScore ?? 0,
+                routingScopeNotExpanded: true, // Always true in Phase 7
+            },
+            baselineMetrics,
+            orchestratorReviewPassed,
+            reviewNote,
+            stateChangedAt: now,
+            createdAt: existingIdx >= 0
+                ? registry.promotions[existingIdx].createdAt
+                : now,
+            shadowStartedAt: (targetState === 'shadow_ready' || targetState === 'promotable')
+                ? (() => {
+                    const existing = existingIdx >= 0 ? registry.promotions[existingIdx] : null;
+                    // Only preserve shadowStartedAt if the checkpoint was already on the
+                    // shadow path (shadow_ready or promotable). A demotion to candidate_only
+                    // or rejected means the next shadow entry is a fresh start — use now.
+                    if (existing?.shadowStartedAt &&
+                        (existing.state === 'shadow_ready' || existing.state === 'promotable')) {
+                        return existing.shadowStartedAt;
+                    }
+                    return now;
+                })()
+                : existingIdx >= 0
+                    ? registry.promotions[existingIdx].shadowStartedAt
+                    : undefined,
+            promotableAt: targetState === 'promotable'
+                ? now
+                : existingIdx >= 0
+                    ? registry.promotions[existingIdx].promotableAt
+                    : undefined,
+            previousPromotionId,
+        };
+        if (existingIdx >= 0) {
+            registry.promotions[existingIdx] = promotion;
+        }
+        else {
+            registry.promotions.push(promotion);
+        }
+        writeRegistry(stateDir, registry);
+        return promotion;
+    });
+}
+// ---------------------------------------------------------------------------
+// Promotion Queries
+// ---------------------------------------------------------------------------
+/**
+ * Get the current promotion state for a checkpoint.
+ */
+export function getPromotionState(stateDir, checkpointId) {
+    const registry = readRegistry(stateDir);
+    const promotion = registry.promotions.find((p) => p.checkpointId === checkpointId);
+    return promotion?.state ?? null;
+}
+/**
+ * Get the promotion record for a checkpoint.
+ */
+export function getPromotionRecord(stateDir, checkpointId) {
+    const registry = readRegistry(stateDir);
+    return registry.promotions.find((p) => p.checkpointId === checkpointId) ?? null;
+}
+/**
+ * List promotions by state.
+ */
+export function listPromotionsByState(stateDir, state) {
+    const registry = readRegistry(stateDir);
+    return registry.promotions.filter((p) => p.state === state);
+}
+/**
+ * List all promotions for a profile.
+ */
+export function listPromotionsForProfile(stateDir, targetProfile) {
+    const registry = readRegistry(stateDir);
+    return registry.promotions.filter((p) => p.targetProfile === targetProfile);
+}
+// ---------------------------------------------------------------------------
+// Rollback Support
+// ---------------------------------------------------------------------------
+/**
+ * Reject a checkpoint, preventing it from being promoted.
+ *
+ * @param stateDir - Workspace state directory
+ * @param checkpointId - Checkpoint to reject
+ * @param reason - Reason for rejection
+ * @returns The updated PromotionRecord
+ */
+export function rejectCheckpoint(stateDir, checkpointId, reason) {
+    return withPromotionRegistryLock(stateDir, (registry) => {
+        const now = new Date().toISOString();
+        const existingIdx = registry.promotions.findIndex((p) => p.checkpointId === checkpointId);
+        const checkpoint = getCheckpoint(stateDir, checkpointId);
+        const promotion = {
+            promotionId: existingIdx >= 0
+                ? registry.promotions[existingIdx].promotionId
+                : crypto.randomUUID(),
+            checkpointId,
+            state: 'rejected',
+            targetProfile: 'local-reader', // Default, should be overridden
+            targetModelFamily: checkpoint?.targetModelFamily ?? 'unknown',
+            reducedPromptDelta: 0,
+            constraintMetrics: {
+                arbiterRejectRate: 1,
+                executabilityRejectRate: 1,
+                reviewedSubsetQuality: 0,
+                routingScopeNotExpanded: true,
+            },
+            baselineMetrics: {
+                arbiterRejectRate: 0,
+                executabilityRejectRate: 0,
+                reviewedSubsetQuality: 0,
+            },
+            orchestratorReviewPassed: false,
+            reviewNote: reason,
+            stateChangedAt: now,
+            createdAt: existingIdx >= 0
+                ? registry.promotions[existingIdx].createdAt
+                : now,
+        };
+        if (existingIdx >= 0) {
+            registry.promotions[existingIdx] = promotion;
+        }
+        else {
+            registry.promotions.push(promotion);
+        }
+        writeRegistry(stateDir, registry);
+        return promotion;
+    });
+}
+// ---------------------------------------------------------------------------
+// Default Baseline Metrics
+// ---------------------------------------------------------------------------
+/**
+ * Default baseline metrics for Phase 7.
+ * These represent the "acceptable" thresholds that new checkpoints must meet.
+ */
+export const DEFAULT_BASELINE_METRICS = {
+    arbiterRejectRate: 0.15, // 15% max arbiter rejection
+    executabilityRejectRate: 0.10, // 10% max executability rejection
+    reviewedSubsetQuality: 0.70, // 70% minimum quality score
+};

package/dist/core/session-tracker.d.ts

@@ -73,6 +73,16 @@ export declare function clearInjectedProbationIds(sessionId: string, workspaceDi
 export declare function getSession(sessionId: string): SessionState | undefined;
 export declare function listSessions(workspaceDir?: string): SessionState[];
 export declare function clearSession(sessionId: string): void;
+/**
+ * Seed a session directly into SessionTracker.sessions for testing.
+ * This bypasses the normal tool-call flow to set up test data for
+ * checkWorkspaceIdle without requiring full integration test setup.
+ *
+ * @param sessionId - Session ID
+ * @param workspaceDir - Workspace directory (optional, for filtering)
+ * @param lastActivityAt - Unix timestamp in ms (default: now)
+ */
+export declare function seedSessionForTest(sessionId: string, workspaceDir?: string, lastActivityAt?: number): void;
 export declare function garbageCollectSessions(): void;
 /**
  * Get daily statistics summary for a session.

package/dist/core/session-tracker.js

@@ -325,6 +325,20 @@ export function clearSession(sessionId) {
     }
     sessions.delete(sessionId);
 }
+/**
+ * Seed a session directly into SessionTracker.sessions for testing.
+ * This bypasses the normal tool-call flow to set up test data for
+ * checkWorkspaceIdle without requiring full integration test setup.
+ *
+ * @param sessionId - Session ID
+ * @param workspaceDir - Workspace directory (optional, for filtering)
+ * @param lastActivityAt - Unix timestamp in ms (default: now)
+ */
+export function seedSessionForTest(sessionId, workspaceDir, lastActivityAt) {
+    const state = getOrCreateSession(sessionId, workspaceDir);
+    state.lastActivityAt = lastActivityAt ?? Date.now();
+    state.lastControlActivityAt = state.lastActivityAt;
+}
 // Memory cleanup for abandoned sessions (older than 2 hours)
 export function garbageCollectSessions() {
     const twoHoursAgo = Date.now() - 2 * 60 * 60 * 1000;