principles-disciple 1.7.6 → 1.7.8

package/dist/core/nocturnal-export.js

@@ -0,0 +1,275 @@
+/**
+ * Nocturnal ORPO Export — Approved Dataset to Decision-Point JSONL
+ * =================================================================
+ *
+ * PURPOSE: Export approved nocturnal samples as ORPO-formatted decision-point
+ * training JSONL, strictly separated from legacy correction export.
+ *
+ * ARCHITECTURE:
+ *   - Export output: .state/exports/orpo/{exportId}.jsonl
+ *   - Export manifest: .state/exports/orpo/{exportId}-manifest.json
+ *   - Legacy corrections: untouched, separate path
+ *
+ * ORPO FORMAT (each line):
+ *   {
+ *     sampleFingerprint: string,
+ *     artifactId: string,
+ *     sessionId: string,
+ *     principleId: string,
+ *     targetModelFamily: string,
+ *     prompt: string,        // badDecision (the wrong choice)
+ *     chosen: string,        // betterDecision (the right choice)
+ *     rejected: string,       // badDecision (for ORPO)
+ *     rationale: string,
+ *     datasetMetadata: {
+ *       sampleFingerprint: string,
+ *       artifactPath: string,
+ *       createdAt: string,
+ *       exportedAt: string,
+ *       exportId: string,
+ *       datasetFingerprint: string
+ *     }
+ *   }
+ *
+ * EXPORT GATING (fail-closed):
+ *   - reviewStatus === 'approved_for_training'
+ *   - targetModelFamily matches requested target (or any if not specified)
+ *   - Lineage fields complete (sampleFingerprint, artifactId, sessionId, principleId)
+ *   - Source artifact file exists and is approved
+ *
+ * DESIGN CONSTRAINTS:
+ *   - No trainer invocation
+ *   - No automatic training
+ *   - No checkpoint deploy
+ *   - Export is read-only from dataset perspective
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+import { listDatasetRecords, readDatasetArtifact, } from './nocturnal-dataset.js';
+import { NocturnalPathResolver } from './nocturnal-paths.js';
+// ---------------------------------------------------------------------------
+// Dataset Fingerprint (for reproducibility)
+// ---------------------------------------------------------------------------
+/**
+ * Compute a deterministic dataset fingerprint from a sorted list of sample fingerprints.
+ * This allows reproducible exports — same dataset always produces same fingerprint.
+ */
+function computeDatasetFingerprint(sampleFingerprints) {
+    const sorted = [...sampleFingerprints].sort();
+    const combined = sorted.join('|');
+    return crypto.createHash('sha256').update(combined, 'utf8').digest('hex');
+}
+// ---------------------------------------------------------------------------
+// Individual Sample Serialization
+// ---------------------------------------------------------------------------
+/**
+ * Serialize a single dataset record + artifact to ORPO JSONL line.
+ * Caller guarantees record.targetModelFamily is non-null.
+ */
+function serializeORPOSample(record, artifact, exportId, datasetFingerprint) {
+    const now = new Date().toISOString();
+    return {
+        sampleFingerprint: record.sampleFingerprint,
+        artifactId: record.artifactId,
+        sessionId: record.sessionId,
+        principleId: record.principleId,
+        targetModelFamily: record.targetModelFamily, // validated non-null by caller
+        // For ORPO: prompt = badDecision, chosen = betterDecision, rejected = badDecision
+        // This teaches the model to prefer betterDecision over badDecision
+        prompt: artifact.badDecision,
+        chosen: artifact.betterDecision,
+        rejected: artifact.badDecision,
+        rationale: artifact.rationale,
+        datasetMetadata: {
+            sampleFingerprint: record.sampleFingerprint,
+            artifactPath: record.artifactPath,
+            createdAt: record.createdAt,
+            exportedAt: now,
+            exportId,
+            datasetFingerprint,
+        },
+    };
+}
+// ---------------------------------------------------------------------------
+// Core Export Function
+// ---------------------------------------------------------------------------
+/**
+ * Export approved nocturnal samples as ORPO decision-point JSONL.
+ *
+ * @param workspaceDir - Workspace directory
+ * @param targetModelFamily - Specific model family to export, or undefined for all
+ * @param options - Additional export options
+ * @returns ExportResult
+ */
+export function exportORPOSamples(workspaceDir, targetModelFamily, _options = {}) {
+    const exportId = crypto.randomUUID();
+    const now = new Date().toISOString();
+    // Step 1: Collect eligible records
+    // Use listDatasetRecords directly to have full control over the family filter
+    // (listExportReadyRecords uses ?? which maps null→undefined, losing the null distinction)
+    const allApprovedRecords = listDatasetRecords(workspaceDir, {
+        reviewStatus: 'approved_for_training',
+    });
+    let eligibleRecords;
+    if (targetModelFamily !== undefined && targetModelFamily !== null) {
+        // Specific family: check if ANY records (regardless of status) have this family
+        const allRecords = listDatasetRecords(workspaceDir);
+        const hasAnyWithFamily = allRecords.some((r) => r.targetModelFamily === targetModelFamily);
+        if (!hasAnyWithFamily) {
+            // Family doesn't exist in any record
+            return {
+                success: false,
+                error: 'No samples found for the requested target model family',
+                emptyReason: 'family_mismatch',
+            };
+        }
+        // Family exists but none are approved
+        eligibleRecords = allApprovedRecords.filter((r) => r.targetModelFamily === targetModelFamily);
+    }
+    else {
+        // All families
+        eligibleRecords = allApprovedRecords;
+    }
+    // Step 2: Validate we have records
+    if (eligibleRecords.length === 0) {
+        return {
+            success: false,
+            error: 'No approved samples found for export',
+            emptyReason: 'no_approved_samples',
+        };
+    }
+    // Step 3: Verify lineage completeness and read artifacts
+    const orpoSamples = [];
+    const failedFingerprints = [];
+    for (const record of eligibleRecords) {
+        // Enforce targetModelFamily binding — samples without a family cannot enter training
+        if (record.targetModelFamily === null) {
+            failedFingerprints.push(record.sampleFingerprint);
+            continue;
+        }
+        // Verify lineage completeness
+        if (!record.sampleFingerprint || !record.artifactId || !record.sessionId || !record.principleId) {
+            failedFingerprints.push(record.sampleFingerprint);
+            continue;
+        }
+        // Read artifact (throws on error — distinguishes read failure from missing artifact)
+        let artifact;
+        try {
+            artifact = readDatasetArtifact(workspaceDir, record.sampleFingerprint);
+        }
+        catch {
+            failedFingerprints.push(record.sampleFingerprint);
+            continue;
+        }
+        // Serialize
+        orpoSamples.push(serializeORPOSample(record, artifact, exportId, ''));
+    }
+    // Step 4: Fail if all samples failed validation
+    if (orpoSamples.length === 0) {
+        return {
+            success: false,
+            error: `All ${eligibleRecords.length} eligible samples failed validation (missing artifacts or lineage)`,
+            emptyReason: 'all_samples_missing_artifacts',
+        };
+    }
+    // Step 5: Compute dataset fingerprint for manifest
+    const datasetFingerprint = computeDatasetFingerprint(orpoSamples.map((s) => s.sampleFingerprint));
+    // Step 6: Fill in dataset fingerprint in all samples
+    for (const sample of orpoSamples) {
+        sample.datasetMetadata.datasetFingerprint = datasetFingerprint;
+    }
+    // Step 7: Write JSONL file
+    const exportsDir = NocturnalPathResolver.exportsDir(workspaceDir);
+    const jsonlPath = path.join(exportsDir, `${exportId}.jsonl`);
+    const lines = orpoSamples.map((s) => JSON.stringify(s)).join('\n') + '\n';
+    fs.writeFileSync(jsonlPath, lines, 'utf-8');
+    // Step 8: Write manifest
+    const manifest = {
+        exportId,
+        createdAt: now,
+        sampleCount: orpoSamples.length,
+        targetModelFamily: targetModelFamily ?? 'all',
+        datasetFingerprint,
+        exportPath: jsonlPath,
+        manifestPath: path.join(exportsDir, `${exportId}-manifest.json`),
+        samples: orpoSamples.map((s) => ({
+            sampleFingerprint: s.sampleFingerprint,
+            artifactId: s.artifactId,
+            sessionId: s.sessionId,
+            principleId: s.principleId,
+        })),
+    };
+    fs.writeFileSync(manifest.manifestPath, JSON.stringify(manifest, null, 2), 'utf-8');
+    return {
+        success: true,
+        manifest,
+    };
+}
+/**
+ * Verify an existing export by re-computing its dataset fingerprint.
+ * Returns true if the export is intact and reproducible.
+ */
+export function verifyExportIntegrity(workspaceDir, exportId) {
+    const exportsDir = NocturnalPathResolver.exportsDir(workspaceDir);
+    const manifestPath = path.join(exportsDir, `${exportId}-manifest.json`);
+    if (!fs.existsSync(manifestPath)) {
+        return null;
+    }
+    try {
+        const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+        const computedFingerprint = computeDatasetFingerprint(manifest.samples.map((s) => s.sampleFingerprint));
+        return {
+            valid: computedFingerprint === manifest.datasetFingerprint,
+            computedFingerprint,
+            manifestFingerprint: manifest.datasetFingerprint,
+        };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * List all exports in the exports directory.
+ */
+export function listExports(workspaceDir) {
+    const exportsDir = NocturnalPathResolver.exportsDir(workspaceDir);
+    if (!fs.existsSync(exportsDir)) {
+        return [];
+    }
+    try {
+        const files = fs.readdirSync(exportsDir);
+        const manifests = [];
+        for (const file of files) {
+            if (!file.endsWith('-manifest.json'))
+                continue;
+            try {
+                const manifest = JSON.parse(fs.readFileSync(path.join(exportsDir, file), 'utf-8'));
+                manifests.push(manifest);
+            }
+            catch {
+                // Skip malformed manifest
+            }
+        }
+        return manifests.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Read an export manifest by ID.
+ */
+export function getExportManifest(workspaceDir, exportId) {
+    const exportsDir = NocturnalPathResolver.exportsDir(workspaceDir);
+    const manifestPath = path.join(exportsDir, `${exportId}-manifest.json`);
+    if (!fs.existsSync(manifestPath)) {
+        return null;
+    }
+    try {
+        return JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}

package/dist/core/nocturnal-paths.d.ts

@@ -0,0 +1,124 @@
+/**
+ * Nocturnal Paths — Canonical Path Registry for Sleep-Mode Reflection Artifacts
+ * =============================================================================
+ *
+ * PURPOSE: Establishes a single source of truth for all nocturnal artifact paths.
+ * Prevents path fragmentation and ensures consistent resolution across modules.
+ *
+ * ARCHITECTURE:
+ *   Operator-facing  (read by agent, injected into prompts):
+ *     memory/reflection-log.md       ← human-readable lessons, NOT training data
+ *
+ *   Nocturnal artifacts (structured, NOT injected into prompts):
+ *     .state/nocturnal/samples/     ← decision-point JSON artifacts (Phase 2+)
+ *     .state/nocturnal/memory/      ← short-term reflection memory (Phase 2+)
+ *     .state/exports/orpo/          ← approved training pairs, immutable (Phase 3+)
+ *
+ * DESIGN CONSTRAINTS:
+ * - Nocturnal samples are NEVER written to memory/reflection-log.md
+ * - Prompt injection reads ONLY from memory/reflection-log.md
+ * - Each nocturnal artifact category has its own subdirectory
+ * - All paths go through this registry — no ad-hoc path construction
+ *
+ * USAGE:
+ *   import { NocturnalPathResolver, NOCTURNAL_DIRS, NOCTURNAL_FILES } from './nocturnal-paths.js';
+ *   const sampleDir = NocturnalPathResolver.samplesDir(workspaceDir);
+ */
+export declare const NOCTURNAL_DIRS: {
+    /** Root directory for all nocturnal reflection artifacts */
+    readonly ROOT: ".state/nocturnal";
+    /**
+     * Structured decision-point samples from nocturnal reflection.
+     * Each file is a JSON artifact containing:
+     *   - session snapshot reference
+     *   - target principle
+     *   - decision-point contrastive pair
+     *   - arbiter approval status
+     */
+    readonly SAMPLES: ".state/nocturnal/samples";
+    /**
+     * Short-term operator-facing reflection memory.
+     * Written by nocturnal service, NOT injected into prompts directly.
+     * Consumed by operator on next session start.
+     */
+    readonly MEMORY: ".state/nocturnal/memory";
+    /**
+     * Nocturnal runtime bookkeeping (cooldowns, quotas).
+     * NOTE: nocturnal-runtime.json is written to stateDir directly
+     * (not under NOCTURNAL_DIRS.ROOT) for simpler migration.
+     */
+    /**
+     * Approved training pairs ready for export.
+     * Immutable once written — never modified in place.
+     * Consumed by external trainer (not the plugin).
+     */
+    readonly EXPORTS: ".state/exports/orpo";
+};
+export declare const NOCTURNAL_FILES: {
+    /**
+     * Arbiter review queue for pending samples.
+     * Not written by nocturnal service directly — created during arbiter review.
+     * Format: JSON array of sample refs pending approval.
+     */
+    readonly REVIEW_QUEUE: ".state/nocturnal/review-queue.json";
+    /**
+     * Lineage metadata for all samples.
+     * Written alongside each sample file for traceability.
+     */
+    readonly LINEAGE_INDEX: ".state/nocturnal/samples/lineage-index.json";
+};
+/**
+ * Resolves a nocturnal directory path within a workspace.
+ */
+export declare function resolveNocturnalDir(workspaceDir: string, dirKey: keyof typeof NOCTURNAL_DIRS): string;
+/**
+ * Resolves a nocturnal file path within a workspace.
+ */
+export declare function resolveNocturnalFile(workspaceDir: string, fileKey: keyof typeof NOCTURNAL_FILES): string;
+export declare const NocturnalPathResolver: {
+    /**
+     * Returns the samples directory path.
+     * Creates the directory if it does not exist.
+     */
+    readonly samplesDir: (workspaceDir: string) => string;
+    /**
+     * Returns the memory directory path.
+     * Creates the directory if it does not exist.
+     */
+    readonly memoryDir: (workspaceDir: string) => string;
+    /**
+     * Returns the exports directory path.
+     * Creates the directory if it does not exist.
+     */
+    readonly exportsDir: (workspaceDir: string) => string;
+    /**
+     * Returns the path for a named sample file.
+     * File is NOT created — caller decides when to write.
+     */
+    readonly samplePath: (workspaceDir: string, sampleId: string) => string;
+    /**
+     * Returns the path for nocturnal reflection memory.
+     * This is the operator-facing summary written after each nocturnal run.
+     */
+    readonly reflectionMemoryPath: (workspaceDir: string) => string;
+    /**
+     * Lists all sample files in the samples directory.
+     * Returns absolute paths sorted by modification time (newest first).
+     */
+    readonly listSamples: (workspaceDir: string) => string[];
+    /**
+     * Lists all approved sample files ready for export.
+     * Filters to samples with status === 'approved'.
+     */
+    readonly listApprovedSamples: (workspaceDir: string) => string[];
+};
+/**
+ * Complete path map for reference.
+ * These mirror the keys in NOCTURNAL_DIRS and NOCTURNAL_FILES.
+ */
+export declare const NOCTURNAL_PATH_DESCRIPTIONS: Record<string, string>;
+/**
+ * IMPORTANT: memory/reflection-log.md is NOT a nocturnal artifact.
+ * It is the pre-existing operator-facing reflection log, defined in paths.ts.
+ * It is kept separate from nocturnal outputs by design.
+ */

package/dist/core/nocturnal-paths.js

@@ -0,0 +1,214 @@
+/**
+ * Nocturnal Paths — Canonical Path Registry for Sleep-Mode Reflection Artifacts
+ * =============================================================================
+ *
+ * PURPOSE: Establishes a single source of truth for all nocturnal artifact paths.
+ * Prevents path fragmentation and ensures consistent resolution across modules.
+ *
+ * ARCHITECTURE:
+ *   Operator-facing  (read by agent, injected into prompts):
+ *     memory/reflection-log.md       ← human-readable lessons, NOT training data
+ *
+ *   Nocturnal artifacts (structured, NOT injected into prompts):
+ *     .state/nocturnal/samples/     ← decision-point JSON artifacts (Phase 2+)
+ *     .state/nocturnal/memory/      ← short-term reflection memory (Phase 2+)
+ *     .state/exports/orpo/          ← approved training pairs, immutable (Phase 3+)
+ *
+ * DESIGN CONSTRAINTS:
+ * - Nocturnal samples are NEVER written to memory/reflection-log.md
+ * - Prompt injection reads ONLY from memory/reflection-log.md
+ * - Each nocturnal artifact category has its own subdirectory
+ * - All paths go through this registry — no ad-hoc path construction
+ *
+ * USAGE:
+ *   import { NocturnalPathResolver, NOCTURNAL_DIRS, NOCTURNAL_FILES } from './nocturnal-paths.js';
+ *   const sampleDir = NocturnalPathResolver.samplesDir(workspaceDir);
+ */
+import * as path from 'path';
+import * as fs from 'fs';
+// ---------------------------------------------------------------------------
+// Directory Constants
+// ---------------------------------------------------------------------------
+export const NOCTURNAL_DIRS = {
+    /** Root directory for all nocturnal reflection artifacts */
+    ROOT: '.state/nocturnal',
+    /**
+     * Structured decision-point samples from nocturnal reflection.
+     * Each file is a JSON artifact containing:
+     *   - session snapshot reference
+     *   - target principle
+     *   - decision-point contrastive pair
+     *   - arbiter approval status
+     */
+    SAMPLES: '.state/nocturnal/samples',
+    /**
+     * Short-term operator-facing reflection memory.
+     * Written by nocturnal service, NOT injected into prompts directly.
+     * Consumed by operator on next session start.
+     */
+    MEMORY: '.state/nocturnal/memory',
+    /**
+     * Nocturnal runtime bookkeeping (cooldowns, quotas).
+     * NOTE: nocturnal-runtime.json is written to stateDir directly
+     * (not under NOCTURNAL_DIRS.ROOT) for simpler migration.
+     */
+    // RUNTIME is in {stateDir}/nocturnal-runtime.json (not here)
+    /**
+     * Approved training pairs ready for export.
+     * Immutable once written — never modified in place.
+     * Consumed by external trainer (not the plugin).
+     */
+    EXPORTS: '.state/exports/orpo',
+};
+// ---------------------------------------------------------------------------
+// File Path Constants (within their respective directories)
+// ---------------------------------------------------------------------------
+export const NOCTURNAL_FILES = {
+    /**
+     * Arbiter review queue for pending samples.
+     * Not written by nocturnal service directly — created during arbiter review.
+     * Format: JSON array of sample refs pending approval.
+     */
+    REVIEW_QUEUE: '.state/nocturnal/review-queue.json',
+    /**
+     * Lineage metadata for all samples.
+     * Written alongside each sample file for traceability.
+     */
+    LINEAGE_INDEX: '.state/nocturnal/samples/lineage-index.json',
+};
+// ---------------------------------------------------------------------------
+// Path Resolution
+// ---------------------------------------------------------------------------
+/**
+ * Cross-platform path join for workspace-relative paths.
+ * Handles Windows vs POSIX differences.
+ */
+function joinWorkspacePath(workspaceDir, relativePath) {
+    const normalized = relativePath.replace(/\\/g, '/');
+    if (/^[A-Za-z]:/.test(workspaceDir)) {
+        // Windows
+        return path.win32.join(workspaceDir, ...normalized.split('/'));
+    }
+    return path.posix.join(workspaceDir, ...normalized.split('/'));
+}
+/**
+ * Resolves a nocturnal directory path within a workspace.
+ */
+export function resolveNocturnalDir(workspaceDir, dirKey) {
+    return joinWorkspacePath(workspaceDir, NOCTURNAL_DIRS[dirKey]);
+}
+/**
+ * Resolves a nocturnal file path within a workspace.
+ */
+export function resolveNocturnalFile(workspaceDir, fileKey) {
+    return joinWorkspacePath(workspaceDir, NOCTURNAL_FILES[fileKey]);
+}
+// ---------------------------------------------------------------------------
+// NocturnalPathResolver — Fluent API for common resolutions
+// ---------------------------------------------------------------------------
+export const NocturnalPathResolver = {
+    /**
+     * Returns the samples directory path.
+     * Creates the directory if it does not exist.
+     */
+    samplesDir(workspaceDir) {
+        const dir = resolveNocturnalDir(workspaceDir, 'SAMPLES');
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        return dir;
+    },
+    /**
+     * Returns the memory directory path.
+     * Creates the directory if it does not exist.
+     */
+    memoryDir(workspaceDir) {
+        const dir = resolveNocturnalDir(workspaceDir, 'MEMORY');
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        return dir;
+    },
+    /**
+     * Returns the exports directory path.
+     * Creates the directory if it does not exist.
+     */
+    exportsDir(workspaceDir) {
+        const dir = resolveNocturnalDir(workspaceDir, 'EXPORTS');
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        return dir;
+    },
+    /**
+     * Returns the path for a named sample file.
+     * File is NOT created — caller decides when to write.
+     */
+    samplePath(workspaceDir, sampleId) {
+        const dir = resolveNocturnalDir(workspaceDir, 'SAMPLES');
+        // Sanitize sampleId for filesystem
+        const safeId = sampleId.replace(/[/\\:]/g, '_');
+        return path.join(dir, `${safeId}.json`);
+    },
+    /**
+     * Returns the path for nocturnal reflection memory.
+     * This is the operator-facing summary written after each nocturnal run.
+     */
+    reflectionMemoryPath(workspaceDir) {
+        return path.join(resolveNocturnalDir(workspaceDir, 'MEMORY'), 'reflection-memory.md');
+    },
+    /**
+     * Lists all sample files in the samples directory.
+     * Returns absolute paths sorted by modification time (newest first).
+     */
+    listSamples(workspaceDir) {
+        const dir = resolveNocturnalDir(workspaceDir, 'SAMPLES');
+        if (!fs.existsSync(dir)) {
+            return [];
+        }
+        try {
+            return fs.readdirSync(dir)
+                .filter(f => f.endsWith('.json'))
+                .map(f => path.join(dir, f))
+                .sort((a, b) => fs.statSync(b).mtime.getTime() - fs.statSync(a).mtime.getTime());
+        }
+        catch {
+            return [];
+        }
+    },
+    /**
+     * Lists all approved sample files ready for export.
+     * Filters to samples with status === 'approved'.
+     */
+    listApprovedSamples(workspaceDir) {
+        return this.listSamples(workspaceDir).filter(samplePath => {
+            try {
+                const content = fs.readFileSync(samplePath, 'utf-8');
+                const sample = JSON.parse(content);
+                return sample.status === 'approved';
+            }
+            catch {
+                return false;
+            }
+        });
+    },
+};
+// ---------------------------------------------------------------------------
+// Constants for documentation / external reference
+// ---------------------------------------------------------------------------
+/**
+ * Complete path map for reference.
+ * These mirror the keys in NOCTURNAL_DIRS and NOCTURNAL_FILES.
+ */
+export const NOCTURNAL_PATH_DESCRIPTIONS = {
+    '.state/nocturnal/samples/': 'Structured decision-point JSON artifacts (not injected into prompts)',
+    '.state/nocturnal/memory/': 'Short-term operator-facing reflection memory (not prompt-injected)',
+    '.state/exports/orpo/': 'Approved training pairs, immutable after export',
+    '.state/nocturnal/review-queue.json': 'Pending sample review queue',
+    'memory/reflection-log.md': 'Operator-facing human-readable lessons (INJECTED into prompts)',
+};
+/**
+ * IMPORTANT: memory/reflection-log.md is NOT a nocturnal artifact.
+ * It is the pre-existing operator-facing reflection log, defined in paths.ts.
+ * It is kept separate from nocturnal outputs by design.
+ */