principles-disciple 1.7.6 → 1.7.8

package/dist/core/model-training-registry.d.ts

@@ -0,0 +1,295 @@
+/**
+ * Model Training Registry — Training Run, Checkpoint, and Eval Summary Lineage
+ * =============================================================================
+ *
+ * PURPOSE: Establish strict auditable lineage from training run → checkpoint → eval
+ * so that "deployable" is a controlled state, not a free-text field.
+ *
+ * ARCHITECTURE:
+ *   - Registry file: {stateDir}/.state/nocturnal/training-registry.json
+ *   - Three record types in one store: TrainingRun, Checkpoint, EvalSummary
+ *   - File locking on all write operations
+ *   - Family alignment enforced at every transition
+ *
+ * LINEAGE CHAIN (enforced):
+ *   TrainingRun → Checkpoint → EvalSummary
+ *   DatasetFingerprint → TrainingRun → Checkpoint → EvalSummary
+ *
+ * DEPLOYABILITY RULE:
+ *   A Checkpoint can only be marked deployable if:
+ *     1. It has an attached EvalSummary
+ *     2. The EvalSummary has a verdict of 'pass' or 'compare_only' (not 'fail')
+ *     3. The EvalSummary's targetModelFamily matches the Checkpoint's targetModelFamily
+ *     4. The Checkpoint's trainRun is in 'completed' status
+ *
+ * DESIGN CONSTRAINTS:
+ *   - No real training invocation (Phase 4 only)
+ *   - No checkpoint deploy routing (Phase 5)
+ *   - No automatic promotion
+ *   - Registry is append-only for runs and checkpoints
+ *   - EvalSummary attachment is the only mutable operation on a Checkpoint
+ */
+/**
+ * Training run status — lifecycle enforced transitions.
+ */
+export type TrainingRunStatus = 'pending' | 'running' | 'completed' | 'failed';
+/**
+ * A training run record — tracks one training job.
+ */
+export interface TrainingRun {
+    /** Unique identifier for this training run */
+    trainRunId: string;
+    /**
+     * Experiment ID this run belongs to.
+     * Enables lookup by experimentId in addition to trainRunId.
+     */
+    experimentId: string;
+    /** Target model family this run produces checkpoints for */
+    targetModelFamily: string;
+    /**
+     * Fingerprint of the dataset used for this run.
+     * Links back to the ORPO export's datasetFingerprint.
+     */
+    datasetFingerprint: string;
+    /**
+     * Reference to the ORPO export that provided the training data.
+     * Format: {exportId}
+     */
+    exportId: string;
+    /** Number of samples from the export used in this run */
+    sampleCount: number;
+    /**
+     * Fingerprint of the training configuration used.
+     * For Phase 4 this is a placeholder (e.g., 'default-v0.1.0').
+     * Future: references an actual config artifact.
+     */
+    configFingerprint: string;
+    /** ISO-8601 creation timestamp */
+    createdAt: string;
+    /** ISO-8601 completion timestamp (set when status becomes completed/failed) */
+    completedAt?: string;
+    /** Current status */
+    status: TrainingRunStatus;
+    /** Human-readable reason for failure (if status === 'failed') */
+    failureReason?: string;
+    /**
+     * Checkpoint IDs produced by this run.
+     * A run may produce multiple checkpoints (e.g., epoch saves).
+     */
+    checkpointIds: string[];
+}
+/**
+ * A checkpoint record — a deployable artifact from a training run.
+ */
+export interface Checkpoint {
+    /** Unique identifier for this checkpoint */
+    checkpointId: string;
+    /** The training run that produced this checkpoint */
+    trainRunId: string;
+    /** Target model family (must match the TrainingRun's targetModelFamily) */
+    targetModelFamily: string;
+    /**
+     * Path to the checkpoint artifact.
+     * In Phase 4 this is a placeholder path.
+     * Future: path to adapter weights, config, etc.
+     */
+    artifactPath: string;
+    /** ISO-8601 creation timestamp */
+    createdAt: string;
+    /**
+     * Whether this checkpoint can be routed to a worker.
+     * MUST be false until an EvalSummary is attached with verdict 'pass' or 'compare_only'.
+     * Cannot be set back to true after false without a new passing eval.
+     */
+    deployable: boolean;
+    /**
+     * Reference to the attached EvalSummary (evalId).
+     * Required for deployable === true.
+     */
+    lastEvalSummaryRef?: string;
+}
+/**
+ * An eval summary record — result of benchmarking a checkpoint.
+ */
+export interface EvalSummary {
+    /** Unique identifier for this eval */
+    evalId: string;
+    /** The checkpoint this eval was run against */
+    checkpointId: string;
+    /** The benchmark run this eval came from */
+    benchmarkId: string;
+    /**
+     * Target model family — MUST match the checkpoint's targetModelFamily.
+     * Enforced at attachEvalSummary() time.
+     * This field prevents a gpt-4 checkpoint from being validated by a claude-3 eval.
+     */
+    targetModelFamily: string;
+    /** Evaluation mode: prompt-assisted or reduced-prompt */
+    mode: 'prompt_assisted' | 'reduced_prompt';
+    /** Baseline score from the benchmark (mean score of baseline checkpoint) */
+    baselineScore: number;
+    /** Candidate score from the benchmark (mean score of this checkpoint) */
+    candidateScore: number;
+    /** delta = candidateScore - baselineScore */
+    delta: number;
+    /** Verdict from the benchmark: pass | fail | compare_only */
+    verdict: 'pass' | 'fail' | 'compare_only';
+    /** ISO-8601 creation timestamp */
+    createdAt: string;
+}
+/**
+ * The complete training registry — all record types in one store.
+ */
+export interface ModelTrainingRegistry {
+    trainingRuns: TrainingRun[];
+    checkpoints: Checkpoint[];
+    evalSummaries: EvalSummary[];
+}
+/**
+ * Register a new training run.
+ *
+ * @param stateDir - Workspace state directory
+ * @param params - Run parameters
+ * @returns The registered TrainingRun
+ */
+export declare function registerTrainingRun(stateDir: string, params: {
+    experimentId: string;
+    targetModelFamily: string;
+    datasetFingerprint: string;
+    exportId: string;
+    sampleCount: number;
+    configFingerprint: string;
+}): TrainingRun;
+/**
+ * Update a training run's status.
+ *
+ * @throws Error if run not found or transition is invalid
+ */
+export declare function updateTrainingRunStatus(stateDir: string, trainRunId: string, newStatus: TrainingRunStatus, failureReason?: string): TrainingRun;
+/**
+ * Complete a training run (convenience wrapper).
+ */
+export declare function completeTrainingRun(stateDir: string, trainRunId: string): TrainingRun;
+/**
+ * Fail a training run (convenience wrapper).
+ */
+export declare function failTrainingRun(stateDir: string, trainRunId: string, reason: string): TrainingRun;
+/**
+ * Start a training run (convenience wrapper).
+ */
+export declare function startTrainingRun(stateDir: string, trainRunId: string): TrainingRun;
+/**
+ * Get a training run by ID.
+ */
+export declare function getTrainingRun(stateDir: string, trainRunId: string): TrainingRun | null;
+/**
+ * List all training runs, optionally filtered by status or family.
+ */
+export declare function listTrainingRuns(stateDir: string, filter?: {
+    status?: TrainingRunStatus;
+    targetModelFamily?: string;
+}): TrainingRun[];
+/**
+ * Register a checkpoint produced by a training run.
+ *
+ * @throws Error if the training run is not found
+ * @throws Error if the targetModelFamily does not match the run's family
+ */
+export declare function registerCheckpoint(stateDir: string, params: {
+    trainRunId: string;
+    targetModelFamily: string;
+    artifactPath: string;
+}): Checkpoint;
+/**
+ * Get a checkpoint by ID.
+ */
+export declare function getCheckpoint(stateDir: string, checkpointId: string): Checkpoint | null;
+/**
+ * List all checkpoints, optionally filtered.
+ */
+export declare function listCheckpoints(stateDir: string, filter?: {
+    trainRunId?: string;
+    targetModelFamily?: string;
+    deployable?: boolean;
+}): Checkpoint[];
+/**
+ * List all deployable checkpoints for a target model family.
+ */
+export declare function listDeployableCheckpoints(stateDir: string, targetModelFamily: string): Checkpoint[];
+/**
+ * Attach an eval summary to a checkpoint.
+ *
+ * @param stateDir - Workspace state directory
+ * @param checkpointId - The checkpoint to attach to
+ * @param summary - The eval summary to attach
+ *
+ * @throws Error if checkpoint not found
+ * @throws Error if targetModelFamily mismatch between summary and checkpoint
+ */
+export declare function attachEvalSummary(stateDir: string, checkpointId: string, summary: Omit<EvalSummary, 'createdAt'>): EvalSummary;
+/**
+ * Get an eval summary by ID.
+ */
+export declare function getEvalSummary(stateDir: string, evalId: string): EvalSummary | null;
+/**
+ * List eval summaries, optionally filtered.
+ */
+export declare function listEvalSummaries(stateDir: string, filter?: {
+    checkpointId?: string;
+    benchmarkId?: string;
+    verdict?: EvalSummary['verdict'];
+    targetModelFamily?: string;
+}): EvalSummary[];
+/**
+ * MARK the deployability status of a checkpoint.
+ *
+ * DEPLOYABILITY RULE (fail-closed):
+ *   A checkpoint can only be marked deployable if ALL of:
+ *     1. It has an attached EvalSummary (lastEvalSummaryRef is set)
+ *     2. The attached EvalSummary has verdict 'pass' or 'compare_only' (not 'fail')
+ *     3. The EvalSummary's targetModelFamily matches the Checkpoint's targetModelFamily
+ *        NOTE: This is enforced at attachEvalSummary() time (see attachEvalSummary).
+ *        If a mismatched-family eval is attached, attachEvalSummary throws before
+ *        the registry is modified, so no eval with wrong family can ever reach here.
+ *     4. The parent TrainingRun is in 'completed' status
+ *
+ * @param stateDir - Workspace state directory
+ * @param checkpointId - The checkpoint to mark
+ * @param deployable - true to mark as deployable; false to revoke
+ *
+ * @throws Error if checkpoint not found
+ * @throws Error if preconditions for deployable=true are not met
+ */
+export declare function markCheckpointDeployable(stateDir: string, checkpointId: string, deployable: boolean): Checkpoint;
+/**
+ * Convenience: check if a checkpoint is deployable.
+ */
+export declare function isCheckpointDeployable(stateDir: string, checkpointId: string): boolean;
+/**
+ * Get the full lineage chain for a checkpoint.
+ * Returns: { run, checkpoint, eval? } or null if not found.
+ */
+export declare function getCheckpointLineage(stateDir: string, checkpointId: string): {
+    run: TrainingRun;
+    checkpoint: Checkpoint;
+    eval: EvalSummary | null;
+} | null;
+/**
+ * Get the complete registry (for debugging/admin purposes).
+ */
+export declare function getFullRegistry(stateDir: string): ModelTrainingRegistry;
+/**
+ * Compute stats for the training registry.
+ */
+export declare function getTrainingRegistryStats(stateDir: string): {
+    totalRuns: number;
+    completedRuns: number;
+    failedRuns: number;
+    pendingRuns: number;
+    runningRuns: number;
+    totalCheckpoints: number;
+    deployableCheckpoints: number;
+    totalEvals: number;
+    passingEvals: number;
+    failingEvals: number;
+};