principles-disciple 1.7.6 → 1.8.0

package/dist/core/external-training-contract.d.ts

@@ -0,0 +1,276 @@
+/**
+ * External Training Contract — Normalized Experiment Spec and Result Schema
+ * ========================================================================
+ *
+ * PURPOSE: Define the stable contract between the plugin and external trainer
+ * backends. The plugin produces a constrained experiment specification that an
+ * external trainer consumes. The trainer returns a normalized result that the
+ * plugin can register, evaluate, and gate for rollout.
+ *
+ * ARCHITECTURE:
+ *   - Plugin is responsible for creating the experiment spec
+ *   - Plugin is responsible for validating the trainer result
+ *   - Plugin is responsible for registering lineage (train run → checkpoint → eval)
+ *   - Plugin is responsible for invoking benchmark evaluation
+ *   - Plugin is responsible for invoking promotion gate logic
+ *   - Plugin is responsible for binding deployment only after gate approval
+ *
+ * DESIGN CONSTRAINTS:
+ *   - ORPO-first: trainingMode must be 'orpo' for production runs
+ *   - No real training inside the plugin
+ *   - No direct deployment promotion from trainer output
+ *   - No direct trainer writes to review/eval/deployment state
+ *   - Backend-pluggable: same contract works for all backends
+ *
+ * CONTRACT GOALS:
+ *   - support ORPO training for approved nocturnal exports
+ *   - support multiple backend implementations behind one schema
+ *   - preserve dataset / config / checkpoint lineage
+ *   - remain valid on consumer hardware
+ *   - fail closed when inputs are incomplete or inconsistent
+ */
+/**
+ * Allowed backend identifiers.
+ *
+ * - `peft-trl-orpo`: primary reference implementation using PEFT + TRL ORPO
+ * - `unsloth-orpo`: compatible accelerated implementation using Unsloth
+ * - `dry-run`: validates paths/spec/environment only, no real training
+ */
+export type TrainerBackendKind = 'peft-trl-orpo' | 'unsloth-orpo' | 'dry-run';
+/**
+ * Hardware tier for training.
+ *
+ * - `consumer-gpu`: RTX 4090 24GB or equivalent (production target)
+ * - `small-gpu`: 8GB-16GB VRAM (compatibility target)
+ * - `cpu-experimental`: CPU-only experimental runs (dry-run or tiny models only)
+ */
+export type HardwareTier = 'consumer-gpu' | 'small-gpu' | 'cpu-experimental';
+/**
+ * Worker profiles supported for training.
+ *
+ * Phase 7 first rollout: `local-reader` only.
+ * `local-editor` requires explicit human approval to enable.
+ */
+export type TrainableWorkerProfile = 'local-reader' | 'local-editor';
+/**
+ * Training mode — Phase 7 production is ORPO-only.
+ */
+export type TrainingMode = 'orpo';
+/**
+ * Hyperparameters for ORPO training.
+ */
+export interface TrainingHyperparameters {
+    learningRate: number;
+    batchSize: number;
+    gradientAccumulation: number;
+    loraRank: number;
+    loraAlpha: number;
+    loraDropout: number;
+    warmupRatio: number;
+    maxSteps: number;
+    maxSeqLength: number;
+}
+/**
+ * Budget constraints for a training experiment.
+ */
+export interface TrainingBudget {
+    maxWallClockMinutes: number;
+    maxTrainTokens?: number;
+}
+/**
+ * Expected artifact from a successful training run.
+ */
+export interface ExpectedArtifact {
+    checkpointName: string;
+    adapterFormat: 'peft-adapter';
+}
+/**
+ * The experiment specification sent to an external trainer.
+ * This defines WHAT to train, not HOW to train (backend-specific).
+ */
+export interface TrainingExperimentSpec {
+    /** Unique identifier for this experiment */
+    experimentId: string;
+    /** Which backend to use */
+    backend: TrainerBackendKind;
+    /** Training mode — only 'orpo' is supported in Phase 7 */
+    trainingMode: TrainingMode;
+    /** Target worker profile for this experiment */
+    targetWorkerProfile: TrainableWorkerProfile;
+    /** Target model family to train */
+    targetModelFamily: string;
+    /** Hardware tier for this experiment */
+    hardwareTier: HardwareTier;
+    /** Reference to the ORPO export providing training data */
+    datasetExportId: string;
+    datasetExportPath: string;
+    /** Fingerprint of the dataset for lineage verification */
+    datasetFingerprint: string;
+    /** Reference to the benchmark export for eval */
+    benchmarkExportId: string;
+    /** Output directory for checkpoint artifacts */
+    outputDir: string;
+    /** Fingerprint of the training configuration */
+    configFingerprint: string;
+    /** Hash of the training code/contract version */
+    codeHash: string;
+    /** Training hyperparameters */
+    hyperparameters: TrainingHyperparameters;
+    /** Budget constraints */
+    budget: TrainingBudget;
+    /** Expected artifact from training */
+    expectedArtifact: ExpectedArtifact;
+}
+/**
+ * Training metrics recorded by the backend.
+ */
+export interface TrainingMetrics {
+    wallClockMinutes: number;
+    finalLoss?: number;
+    tokensSeen?: number;
+}
+/**
+ * Artifact produced by a successful training run.
+ */
+export interface TrainingArtifact {
+    adapterFormat: 'peft-adapter';
+    artifactPath: string;
+}
+/**
+ * Status of a training experiment.
+ */
+export type ExperimentStatus = 'completed' | 'failed' | 'dry_run';
+/**
+ * The result returned by an external trainer after execution.
+ * This defines the output contract — all backends must return the same shape.
+ */
+export interface TrainingExperimentResult {
+    /** Experiment ID (must match the spec's experimentId) */
+    experimentId: string;
+    /** Which backend was used */
+    backend: TrainerBackendKind;
+    /** Final status of the experiment */
+    status: ExperimentStatus;
+    /** Registered training run ID (plugin-side) */
+    trainRunId?: string;
+    /** Registered checkpoint ID (plugin-side) */
+    checkpointId?: string;
+    /** Checkpoint reference string (for lineage) */
+    checkpointRef?: string;
+    /** Target worker profile */
+    targetWorkerProfile: TrainableWorkerProfile;
+    /** Target model family */
+    targetModelFamily: string;
+    /** Dataset fingerprint (for lineage verification) */
+    datasetFingerprint: string;
+    /** Config fingerprint (for lineage verification) */
+    configFingerprint: string;
+    /** Code hash (for lineage verification) */
+    codeHash: string;
+    /** Training metrics */
+    metrics?: TrainingMetrics;
+    /** Produced artifact (only if status === 'completed') */
+    artifact?: TrainingArtifact;
+    /** Failure reason (only if status === 'failed') */
+    failureReason?: string;
+    /** ISO-8601 creation timestamp */
+    createdAt: string;
+}
+/**
+ * Validation error for trainer result verification.
+ */
+export interface ValidationError {
+    field: string;
+    expected: string;
+    actual: string;
+    reason: string;
+}
+/**
+ * Result of validating a trainer result against the experiment spec.
+ */
+export interface ValidationResult {
+    valid: boolean;
+    errors: ValidationError[];
+}
+/**
+ * Validate that a trainer result matches the experiment spec.
+ *
+ * FAILS CLOSED on any mismatch — a checkpoint with invalid lineage must not
+ * be registered or promoted.
+ *
+ * Validation rules:
+ * 1. experimentId must match
+ * 2. backend must match
+ * 3. targetWorkerProfile must match
+ * 4. targetModelFamily must match
+ * 5. datasetFingerprint must match
+ * 6. configFingerprint must match
+ * 7. codeHash must match
+ * 8. dry-run must not produce a deployable checkpoint
+ *
+ * @param spec - The original experiment spec
+ * @param result - The trainer result to validate
+ * @returns ValidationResult indicating pass/fail and any errors
+ */
+export declare function validateTrainerResult(spec: TrainingExperimentSpec, result: TrainingExperimentResult): ValidationResult;
+/**
+ * Generate a fingerprint for a configuration object.
+ * Used for configFingerprint in the experiment spec.
+ */
+export declare function computeConfigFingerprint(config: Partial<TrainingHyperparameters>): string;
+/**
+ * Generate a fingerprint for a dataset export.
+ * Used for datasetFingerprint in the experiment spec.
+ *
+ * Combines file content hash with sampleCount to detect:
+ * - Content changes (file modified/replaced)
+ * - Sample count changes (different export)
+ *
+ * If the file cannot be read, falls back to path+count hash (legacy behavior).
+ */
+export declare function computeDatasetFingerprint(exportPath: string, sampleCount: number): string;
+/**
+ * Generate a code hash for the training contract version.
+ * Used for codeHash in the experiment spec.
+ *
+ * Hashes the actual contract source file content so any change to the
+ * contract produces a different hash, ensuring lineage integrity.
+ *
+ * Falls back to version string + timestamp if source cannot be read.
+ */
+export declare function computeCodeHash(): string;
+/**
+ * Generate a new experiment ID.
+ */
+export declare function generateExperimentId(): string;
+/**
+ * Validate that a hardware tier is appropriate for the backend.
+ *
+ * @param backend - The backend being used
+ * @param tier - The hardware tier
+ * @throws Error if the combination is not supported
+ */
+export declare function validateHardwareTier(backend: TrainerBackendKind, tier: HardwareTier): void;
+/**
+ * Get the default hardware tier for a backend.
+ */
+export declare function getDefaultHardwareTier(backend: TrainerBackendKind): HardwareTier;
+/**
+ * Valid model family patterns for local-reader profile.
+ * Used for family validation in the training contract.
+ */
+export declare const READER_FAMILY_PATTERNS: string[];
+/**
+ * Valid model family patterns for local-editor profile.
+ * Used for family validation in the training contract.
+ */
+export declare const EDITOR_FAMILY_PATTERNS: string[];
+/**
+ * Check if a model family is valid for a worker profile.
+ */
+export declare function isValidModelFamilyForProfile(family: string, profile: TrainableWorkerProfile): boolean;
+/**
+ * Phase 7 first rollout is limited to local-reader.
+ * This flag controls whether local-editor is allowed.
+ */
+export declare const LOCAL_EDITOR_ENABLED = false;

package/dist/core/external-training-contract.js

@@ -0,0 +1,269 @@
+/**
+ * External Training Contract — Normalized Experiment Spec and Result Schema
+ * ========================================================================
+ *
+ * PURPOSE: Define the stable contract between the plugin and external trainer
+ * backends. The plugin produces a constrained experiment specification that an
+ * external trainer consumes. The trainer returns a normalized result that the
+ * plugin can register, evaluate, and gate for rollout.
+ *
+ * ARCHITECTURE:
+ *   - Plugin is responsible for creating the experiment spec
+ *   - Plugin is responsible for validating the trainer result
+ *   - Plugin is responsible for registering lineage (train run → checkpoint → eval)
+ *   - Plugin is responsible for invoking benchmark evaluation
+ *   - Plugin is responsible for invoking promotion gate logic
+ *   - Plugin is responsible for binding deployment only after gate approval
+ *
+ * DESIGN CONSTRAINTS:
+ *   - ORPO-first: trainingMode must be 'orpo' for production runs
+ *   - No real training inside the plugin
+ *   - No direct deployment promotion from trainer output
+ *   - No direct trainer writes to review/eval/deployment state
+ *   - Backend-pluggable: same contract works for all backends
+ *
+ * CONTRACT GOALS:
+ *   - support ORPO training for approved nocturnal exports
+ *   - support multiple backend implementations behind one schema
+ *   - preserve dataset / config / checkpoint lineage
+ *   - remain valid on consumer hardware
+ *   - fail closed when inputs are incomplete or inconsistent
+ */
+import * as crypto from 'crypto';
+import * as fs from 'fs';
+import { fileURLToPath } from 'url';
+// ---------------------------------------------------------------------------
+// Contract Validation
+// ---------------------------------------------------------------------------
+/**
+ * Validate that a trainer result matches the experiment spec.
+ *
+ * FAILS CLOSED on any mismatch — a checkpoint with invalid lineage must not
+ * be registered or promoted.
+ *
+ * Validation rules:
+ * 1. experimentId must match
+ * 2. backend must match
+ * 3. targetWorkerProfile must match
+ * 4. targetModelFamily must match
+ * 5. datasetFingerprint must match
+ * 6. configFingerprint must match
+ * 7. codeHash must match
+ * 8. dry-run must not produce a deployable checkpoint
+ *
+ * @param spec - The original experiment spec
+ * @param result - The trainer result to validate
+ * @returns ValidationResult indicating pass/fail and any errors
+ */
+export function validateTrainerResult(spec, result) {
+    const errors = [];
+    // Rule 1: experimentId must match
+    if (spec.experimentId !== result.experimentId) {
+        errors.push({
+            field: 'experimentId',
+            expected: spec.experimentId,
+            actual: result.experimentId,
+            reason: 'Trainer result experimentId does not match the experiment spec',
+        });
+    }
+    // Rule 2: backend must match
+    if (spec.backend !== result.backend) {
+        errors.push({
+            field: 'backend',
+            expected: spec.backend,
+            actual: result.backend,
+            reason: 'Trainer result backend does not match the experiment spec',
+        });
+    }
+    // Rule 3: targetWorkerProfile must match
+    if (spec.targetWorkerProfile !== result.targetWorkerProfile) {
+        errors.push({
+            field: 'targetWorkerProfile',
+            expected: spec.targetWorkerProfile,
+            actual: result.targetWorkerProfile,
+            reason: 'Trainer result targetWorkerProfile does not match the experiment spec',
+        });
+    }
+    // Rule 4: targetModelFamily must match
+    if (spec.targetModelFamily !== result.targetModelFamily) {
+        errors.push({
+            field: 'targetModelFamily',
+            expected: spec.targetModelFamily,
+            actual: result.targetModelFamily,
+            reason: 'Trainer result targetModelFamily does not match the experiment spec',
+        });
+    }
+    // Rule 5: datasetFingerprint must match
+    if (spec.datasetFingerprint !== result.datasetFingerprint) {
+        errors.push({
+            field: 'datasetFingerprint',
+            expected: spec.datasetFingerprint,
+            actual: result.datasetFingerprint,
+            reason: 'Dataset fingerprint mismatch — possible dataset tampering or wrong export used',
+        });
+    }
+    // Rule 6: configFingerprint must match
+    if (spec.configFingerprint !== result.configFingerprint) {
+        errors.push({
+            field: 'configFingerprint',
+            expected: spec.configFingerprint,
+            actual: result.configFingerprint,
+            reason: 'Config fingerprint mismatch — training config may have changed since spec was created',
+        });
+    }
+    // Rule 7: codeHash must match
+    if (spec.codeHash !== result.codeHash) {
+        errors.push({
+            field: 'codeHash',
+            expected: spec.codeHash,
+            actual: result.codeHash,
+            reason: 'Code hash mismatch — training code or contract version may have changed',
+        });
+    }
+    // Rule 8: dry-run must not produce a deployable checkpoint
+    if (spec.backend === 'dry-run') {
+        if (result.status === 'completed' && result.artifact) {
+            errors.push({
+                field: 'artifact',
+                expected: 'no artifact for dry-run',
+                actual: 'artifact present',
+                reason: 'Dry-run backend must not produce a deployable checkpoint',
+            });
+        }
+    }
+    return {
+        valid: errors.length === 0,
+        errors,
+    };
+}
+// ---------------------------------------------------------------------------
+// Spec Creation Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Generate a fingerprint for a configuration object.
+ * Used for configFingerprint in the experiment spec.
+ */
+export function computeConfigFingerprint(config) {
+    const normalized = JSON.stringify(config, Object.keys(config).sort());
+    return crypto.createHash('sha256').update(normalized).digest('hex').slice(0, 16);
+}
+/**
+ * Generate a fingerprint for a dataset export.
+ * Used for datasetFingerprint in the experiment spec.
+ *
+ * Combines file content hash with sampleCount to detect:
+ * - Content changes (file modified/replaced)
+ * - Sample count changes (different export)
+ *
+ * If the file cannot be read, falls back to path+count hash (legacy behavior).
+ */
+export function computeDatasetFingerprint(exportPath, sampleCount) {
+    let contentHash;
+    try {
+        const content = fs.readFileSync(exportPath, 'utf-8');
+        contentHash = crypto.createHash('sha256').update(content, 'utf8').digest('hex').slice(0, 16);
+    }
+    catch {
+        // Fallback: include path in hash so different paths still differ
+        // (even if files don't exist during spec creation)
+        const fallbackContent = `${exportPath}:${sampleCount}`;
+        return crypto.createHash('sha256').update(fallbackContent).digest('hex').slice(0, 16);
+    }
+    // Combine content hash with sample count for additional safety
+    const combined = `${contentHash}:${sampleCount}`;
+    return crypto.createHash('sha256').update(combined).digest('hex').slice(0, 16);
+}
+/**
+ * Generate a code hash for the training contract version.
+ * Used for codeHash in the experiment spec.
+ *
+ * Hashes the actual contract source file content so any change to the
+ * contract produces a different hash, ensuring lineage integrity.
+ *
+ * Falls back to version string + timestamp if source cannot be read.
+ */
+export function computeCodeHash() {
+    try {
+        // Hash the actual contract source file content using ESM-safe resolution
+        const sourcePath = fileURLToPath(import.meta.url);
+        const sourceContent = fs.readFileSync(sourcePath, 'utf-8');
+        // Include only the relevant contract definitions (first 500 lines)
+        // to avoid hash changes from comments/timestamps
+        const relevantContent = sourceContent.split('\n').slice(0, 500).join('\n');
+        return crypto.createHash('sha256').update(relevantContent).digest('hex').slice(0, 16);
+    }
+    catch {
+        // Fallback if source cannot be read (should not happen in normal operation)
+        // Use a deterministic version string — NOT Date.now() — so the hash is stable
+        const fallback = 'nocturnal-phase7-v1:deterministic-fallback';
+        return crypto.createHash('sha256').update(fallback).digest('hex').slice(0, 16);
+    }
+}
+/**
+ * Generate a new experiment ID.
+ */
+export function generateExperimentId() {
+    return crypto.randomUUID();
+}
+// ---------------------------------------------------------------------------
+// Hardware Tier Helpers
+// ---------------------------------------------------------------------------
+/**
+ * Validate that a hardware tier is appropriate for the backend.
+ *
+ * @param backend - The backend being used
+ * @param tier - The hardware tier
+ * @throws Error if the combination is not supported
+ */
+export function validateHardwareTier(backend, tier) {
+    // cpu-experimental is only allowed for dry-run
+    if (tier === 'cpu-experimental' && backend !== 'dry-run') {
+        throw new Error(`Hardware tier 'cpu-experimental' is only allowed for 'dry-run' backend. ` +
+            `For real training on GPU, use 'consumer-gpu' or 'small-gpu'.`);
+    }
+}
+/**
+ * Get the default hardware tier for a backend.
+ */
+export function getDefaultHardwareTier(backend) {
+    if (backend === 'dry-run') {
+        return 'cpu-experimental';
+    }
+    return 'consumer-gpu';
+}
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/**
+ * Valid model family patterns for local-reader profile.
+ * Used for family validation in the training contract.
+ */
+export const READER_FAMILY_PATTERNS = [
+    'reader', 'read', 'claude-haiku', 'qwen-lite', 'phi-mini',
+    'gpt-4o-mini', 'gpt-4o-nano',
+];
+/**
+ * Valid model family patterns for local-editor profile.
+ * Used for family validation in the training contract.
+ */
+export const EDITOR_FAMILY_PATTERNS = [
+    'editor', 'edit', 'code', 'claude-sonnet', 'gpt-4o-mini',
+];
+/**
+ * Check if a model family is valid for a worker profile.
+ */
+export function isValidModelFamilyForProfile(family, profile) {
+    const lower = family.toLowerCase();
+    if (profile === 'local-reader') {
+        return READER_FAMILY_PATTERNS.some((p) => lower.includes(p));
+    }
+    if (profile === 'local-editor') {
+        return EDITOR_FAMILY_PATTERNS.some((p) => lower.includes(p));
+    }
+    return false;
+}
+/**
+ * Phase 7 first rollout is limited to local-reader.
+ * This flag controls whether local-editor is allowed.
+ */
+export const LOCAL_EDITOR_ENABLED = false;