principles-disciple 1.52.0 → 1.53.0

package/tests/core/file-storage-adapter.test.ts

@@ -0,0 +1,285 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { FileStorageAdapter } from '../../src/core/file-storage-adapter.js';
+import type { HybridLedgerStore } from '../../src/core/principle-tree-ledger.js';
+import { TREE_NAMESPACE, loadLedger } from '../../src/core/principle-tree-ledger.js';
+import { safeRmDir } from '../test-utils.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function createTmpDir(): string {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'pd-file-storage-adapter-test-'));
+}
+
+function createEmptyStore(): HybridLedgerStore {
+  return {
+    trainingStore: {},
+    tree: {
+      principles: {},
+      rules: {},
+      implementations: {},
+      metrics: {},
+      lastUpdated: new Date(0).toISOString(),
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe('FileStorageAdapter', () => {
+  let tmpDir: string;
+  let adapter: FileStorageAdapter;
+
+  beforeEach(() => {
+    tmpDir = createTmpDir();
+    adapter = new FileStorageAdapter(tmpDir, tmpDir);
+  });
+
+  afterEach(() => {
+    safeRmDir(tmpDir);
+  });
+
+  // -------------------------------------------------------------------------
+  // loadLedger
+  // -------------------------------------------------------------------------
+
+  describe('loadLedger', () => {
+    it('returns empty store when no file exists', async () => {
+      const store = await adapter.loadLedger();
+      expect(store.trainingStore).toEqual({});
+      expect(store.tree.principles).toEqual({});
+      expect(store.tree.rules).toEqual({});
+    });
+
+    it('loads existing persisted store', async () => {
+      const original = createEmptyStore();
+      original.tree.principles['P-001'] = {
+        id: 'P-001',
+        version: 1,
+        text: 'Write before delete',
+        triggerPattern: 'delete',
+        action: 'write first',
+        status: 'active',
+        priority: 'P1',
+        scope: 'general',
+        evaluability: 'deterministic',
+        valueScore: 0,
+        adherenceRate: 0,
+        painPreventedCount: 0,
+        derivedFromPainIds: [],
+        ruleIds: [],
+        conflictsWithPrincipleIds: [],
+        createdAt: '2026-04-17T00:00:00.000Z',
+        updatedAt: '2026-04-17T00:00:00.000Z',
+      };
+
+      // Persist using the low-level ledger to seed the file
+      await adapter.saveLedger(original);
+      const loaded = await adapter.loadLedger();
+      expect(loaded.tree.principles['P-001']).toBeDefined();
+      expect(loaded.tree.principles['P-001'].text).toBe('Write before delete');
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // saveLedger
+  // -------------------------------------------------------------------------
+
+  describe('saveLedger', () => {
+    it('persists store to disk', async () => {
+      const store = createEmptyStore();
+      store.trainingStore['test-principle'] = {
+        principleId: 'test-principle',
+        evaluability: 'manual_only',
+        applicableOpportunityCount: 0,
+        observedViolationCount: 0,
+        complianceRate: 0,
+        violationTrend: 0,
+        generatedSampleCount: 0,
+        approvedSampleCount: 0,
+        includedTrainRunIds: [],
+        deployedCheckpointIds: [],
+        internalizationStatus: 'prompt_only',
+      };
+
+      await adapter.saveLedger(store);
+
+      // Verify file exists and contains the data
+      const filePath = path.join(tmpDir, 'principle_training_state.json');
+      expect(fs.existsSync(filePath)).toBe(true);
+      const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+      expect(raw['test-principle']).toBeDefined();
+      expect(raw[TREE_NAMESPACE]).toBeDefined();
+    });
+
+    it('round-trips data through save and load', async () => {
+      const store = createEmptyStore();
+      store.trainingStore['p-1'] = {
+        principleId: 'p-1',
+        evaluability: 'weak_heuristic',
+        applicableOpportunityCount: 5,
+        observedViolationCount: 2,
+        complianceRate: 0.6,
+        violationTrend: -0.1,
+        generatedSampleCount: 3,
+        approvedSampleCount: 2,
+        includedTrainRunIds: ['run-1'],
+        deployedCheckpointIds: [],
+        internalizationStatus: 'in_training',
+      };
+
+      await adapter.saveLedger(store);
+      const loaded = await adapter.loadLedger();
+      expect(loaded.trainingStore['p-1'].evaluability).toBe('weak_heuristic');
+      expect(loaded.trainingStore['p-1'].applicableOpportunityCount).toBe(5);
+    });
+  });
+
+  // -------------------------------------------------------------------------
+  // mutateLedger
+  // -------------------------------------------------------------------------
+
+  describe('mutateLedger', () => {
+    it('reads, mutates, and writes atomically', async () => {
+      // Start with empty store
+      await adapter.saveLedger(createEmptyStore());
+
+      const result = await adapter.mutateLedger((store) => {
+        store.tree.principles['P-002'] = {
+          id: 'P-002',
+          version: 1,
+          text: 'Test principle',
+          triggerPattern: 'test',
+          action: 'do something',
+          status: 'candidate',
+          priority: 'P2',
+          scope: 'general',
+          evaluability: 'manual_only',
+          valueScore: 0,
+          adherenceRate: 0,
+          painPreventedCount: 0,
+          derivedFromPainIds: [],
+          ruleIds: [],
+          conflictsWithPrincipleIds: [],
+          createdAt: '2026-04-17T00:00:00.000Z',
+          updatedAt: '2026-04-17T00:00:00.000Z',
+        };
+        return 42;
+      });
+
+      expect(result).toBe(42);
+      const loaded = await adapter.loadLedger();
+      expect(loaded.tree.principles['P-002']).toBeDefined();
+      expect(loaded.tree.principles['P-002'].text).toBe('Test principle');
+    });
+
+    it('returns the value from the mutate function', async () => {
+      await adapter.saveLedger(createEmptyStore());
+
+      const count = await adapter.mutateLedger((store) => {
+        return Object.keys(store.tree.principles).length;
+      });
+
+      expect(count).toBe(0);
+    });
+
+    it('supports async mutate functions', async () => {
+      await adapter.saveLedger(createEmptyStore());
+
+      const result = await adapter.mutateLedger(async (store) => {
+        // Simulate async work
+        await new Promise((r) => setTimeout(r, 10));
+        store.trainingStore['async-test'] = {
+          principleId: 'async-test',
+          evaluability: 'deterministic',
+          applicableOpportunityCount: 1,
+          observedViolationCount: 0,
+          complianceRate: 1.0,
+          violationTrend: 0,
+          generatedSampleCount: 0,
+          approvedSampleCount: 0,
+          includedTrainRunIds: [],
+          deployedCheckpointIds: [],
+          internalizationStatus: 'prompt_only',
+        };
+        return 'async-done';
+      });
+
+      expect(result).toBe('async-done');
+      const loaded = await adapter.loadLedger();
+      expect(loaded.trainingStore['async-test']).toBeDefined();
+    });
+
+    it('persists via atomicWriteFileSync (file is not corrupted)', async () => {
+      await adapter.saveLedger(createEmptyStore());
+
+      await adapter.mutateLedger((store) => {
+        store.tree.principles['P-003'] = {
+          id: 'P-003',
+          version: 1,
+          text: 'Atomic write test',
+          triggerPattern: 'test',
+          action: 'verify atomicity',
+          status: 'candidate',
+          priority: 'P1',
+          scope: 'general',
+          evaluability: 'manual_only',
+          valueScore: 0,
+          adherenceRate: 0,
+          painPreventedCount: 0,
+          derivedFromPainIds: [],
+          ruleIds: [],
+          conflictsWithPrincipleIds: [],
+          createdAt: '2026-04-17T00:00:00.000Z',
+          updatedAt: '2026-04-17T00:00:00.000Z',
+        };
+      });
+
+      // Verify no leftover temp file
+      const filePath = path.join(tmpDir, 'principle_training_state.json');
+      expect(fs.existsSync(filePath + '.tmp')).toBe(false);
+      expect(fs.existsSync(filePath)).toBe(true);
+
+      // File is valid JSON
+      const raw = JSON.parse(fs.readFileSync(filePath, 'utf8'));
+      expect(raw[TREE_NAMESPACE].principles['P-003']).toBeDefined();
+    });
+
+    it('is compatible with low-level loadLedger', async () => {
+      await adapter.saveLedger(createEmptyStore());
+
+      await adapter.mutateLedger((store) => {
+        store.tree.principles['P-COMPAT'] = {
+          id: 'P-COMPAT',
+          version: 1,
+          text: 'Compatibility test',
+          triggerPattern: 'compat',
+          action: 'verify',
+          status: 'active',
+          priority: 'P1',
+          scope: 'general',
+          evaluability: 'deterministic',
+          valueScore: 10,
+          adherenceRate: 0.8,
+          painPreventedCount: 5,
+          derivedFromPainIds: ['pain-1'],
+          ruleIds: [],
+          conflictsWithPrincipleIds: [],
+          createdAt: '2026-04-17T00:00:00.000Z',
+          updatedAt: '2026-04-17T00:00:00.000Z',
+        };
+      });
+
+      // Load with the low-level function — should see the same data
+      const ledger = loadLedger(tmpDir);
+      expect(ledger.tree.principles['P-COMPAT']).toBeDefined();
+      expect(ledger.tree.principles['P-COMPAT'].text).toBe('Compatibility test');
+    });
+  });
+});

package/tests/core/nocturnal-trinity.test.ts

@@ -12,6 +12,7 @@ import {
   formatReasoningContext,
   invokeStubDreamer,
   invokeStubPhilosopher,
+  validateExtraction,
   type TrinityConfig,
   type DreamerOutput,
   type DreamerCandidate,
@@ -1815,3 +1816,238 @@ describe('Scribe Backward Compatibility (SCRIBE-04)', () => {
     expect(result.artifact!.chosenJustification).toBeUndefined();
   });
 });
+
+// ---------------------------------------------------------------------------
+// Tests: validateExtraction — Hallucination Detection (SDK-QUAL-02)
+// ---------------------------------------------------------------------------
+
+describe('validateExtraction — Hallucination Detection (SDK-QUAL-02)', () => {
+  function makeArtifact(badDecision: string, overrides: Record<string, unknown> = {}): TrinityDraftArtifact {
+    return {
+      selectedCandidateIndex: 0,
+      badDecision,
+      betterDecision: 'Do it right instead',
+      rationale: 'Because the principle says so and this is the correct approach',
+      sessionId: 'session-test-123',
+      principleId: 'T-01',
+      sourceSnapshotRef: 'snapshot-test-001',
+      telemetry: {
+        chainMode: 'trinity',
+        usedStubs: true,
+        dreamerPassed: true,
+        philosopherPassed: true,
+        scribePassed: true,
+        candidateCount: 1,
+        selectedCandidateIndex: 0,
+        stageFailures: [],
+      },
+      ...overrides,
+    };
+  }
+
+  function makeSnapshotWithEvidence(overrides: {
+    failedToolCalls?: Array<{ toolName: string; filePath?: string; errorMessage?: string }>;
+    painEvents?: Array<{ source: string; score: number; reason?: string }>;
+    gateBlocks?: Array<{ toolName: string; reason: string }>;
+    userCorrections?: number;
+  } = {}) {
+    const toolCalls = (overrides.failedToolCalls ?? []).map(tc => ({
+      toolName: tc.toolName,
+      outcome: 'failure' as const,
+      filePath: tc.filePath ?? null,
+      durationMs: null,
+      exitCode: 1,
+      errorType: 'runtime_error',
+      errorMessage: tc.errorMessage ?? 'unknown error',
+      createdAt: '2026-04-17T00:00:00.000Z',
+    }));
+
+    const painEvents = (overrides.painEvents ?? []).map(pe => ({
+      source: pe.source,
+      score: pe.score,
+      severity: 'medium' as const,
+      reason: pe.reason ?? null,
+      createdAt: '2026-04-17T00:00:00.000Z',
+    }));
+
+    const gateBlocks = (overrides.gateBlocks ?? []).map(gb => ({
+      toolName: gb.toolName,
+      filePath: null,
+      reason: gb.reason,
+      planStatus: null,
+      createdAt: '2026-04-17T00:00:00.000Z',
+    }));
+
+    const userTurns = Array.from({ length: overrides.userCorrections ?? 0 }, (_, i) => ({
+      turnIndex: i,
+      correctionDetected: true,
+      correctionCue: 'wrong approach',
+      createdAt: '2026-04-17T00:00:00.000Z',
+    }));
+
+    return {
+      sessionId: 'session-test-123',
+      startedAt: '2026-04-17T00:00:00.000Z',
+      updatedAt: '2026-04-17T00:05:00.000Z',
+      assistantTurns: [],
+      userTurns,
+      toolCalls: toolCalls,
+      painEvents,
+      gateBlocks,
+      stats: {
+        failureCount: toolCalls.length,
+        totalPainEvents: painEvents.length,
+        totalGateBlocks: gateBlocks.length,
+        totalAssistantTurns: 5,
+        totalToolCalls: 10,
+      },
+    };
+  }
+
+  it('passes when badDecision references a tool failure from the snapshot', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      failedToolCalls: [{ toolName: 'Edit', filePath: 'src/config.ts', errorMessage: 'permission denied' }],
+    });
+    const artifact = makeArtifact('Proceeded with Edit on src/config.ts without checking permission');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(true);
+    expect(result.evidenceTypes).toContain('tool_failures');
+  });
+
+  it('passes when badDecision references a pain event from the snapshot', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      painEvents: [{ source: 'gate', score: 70, reason: 'accumulated friction from repeated file operation failures' }],
+    });
+    const artifact = makeArtifact('Ignored accumulated friction from file operations');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(true);
+    expect(result.evidenceTypes).toContain('pain_events');
+  });
+
+  it('passes when badDecision references a gate block from the snapshot', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      gateBlocks: [{ toolName: 'Bash', reason: 'destructive command blocked by safety gate' }],
+    });
+    const artifact = makeArtifact('Attempted to execute a destructive Bash command that was blocked by the gate');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(true);
+    expect(result.evidenceTypes).toContain('gate_blocks');
+  });
+
+  it('passes when badDecision references user corrections', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      userCorrections: 2,
+    });
+    const artifact = makeArtifact('Continued with the wrong approach despite user corrections');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(true);
+    expect(result.evidenceTypes).toContain('user_corrections');
+  });
+
+  it('detects hallucination when badDecision has no overlap with snapshot evidence', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      failedToolCalls: [{ toolName: 'Read', filePath: 'package.json', errorMessage: 'file not found' }],
+    });
+    const artifact = makeArtifact('Deployed production database without running migration scripts first');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(false);
+    expect(result.reason).toContain('Hallucinated extraction');
+  });
+
+  it('passes when snapshot has no evidence at all (no signal to validate against)', () => {
+    const snapshot = makeSnapshotWithEvidence();
+    const artifact = makeArtifact('Made an incorrect decision during the session');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    // No evidence means we cannot validate -- allow through
+    expect(result.isGrounded).toBe(true);
+    expect(result.evidenceTypes).toHaveLength(0);
+  });
+
+  it('provides evidence preview for telemetry', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      failedToolCalls: [{ toolName: 'Write', filePath: 'output.log', errorMessage: 'permission denied for write operation' }],
+      painEvents: [{ source: 'hook', score: 80, reason: 'repeated permission denied failures during write operation' }],
+    });
+    const artifact = makeArtifact('Proceeded with write operation on output.log despite permission denied error');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(true);
+    expect(result.evidencePreview.length).toBeGreaterThan(0);
+    expect(result.evidenceTypes).toContain('tool_failures');
+    expect(result.evidenceTypes).toContain('pain_events');
+  });
+
+  it('detects hallucination with unrelated but specific badDecision text', () => {
+    const snapshot = makeSnapshotWithEvidence({
+      painEvents: [{ source: 'gate', score: 60, reason: 'rate limit exceeded for API calls' }],
+    });
+    const artifact = makeArtifact('Deleted the primary database without creating a backup first');
+
+    const result = validateExtraction(artifact, snapshot as any);
+
+    expect(result.isGrounded).toBe(false);
+  });
+
+  it('runTrinity stub path fails when hallucination is detected', () => {
+    // Create a snapshot with failure signals so stub candidates are generated
+    // but override the tool calls to be something completely unrelated to what
+    // the stub Dreamer generates (which mentions "failing operation")
+    const snapshot = {
+      sessionId: 'session-hallucination-test',
+      startedAt: '2026-04-17T00:00:00.000Z',
+      updatedAt: '2026-04-17T00:05:00.000Z',
+      assistantTurns: [],
+      userTurns: [],
+      toolCalls: [
+        {
+          toolName: 'Grep',
+          outcome: 'failure' as const,
+          filePath: null,
+          durationMs: null,
+          exitCode: 1,
+          errorType: 'timeout',
+          errorMessage: 'search timed out after 30 seconds',
+          createdAt: '2026-04-17T00:00:00.000Z',
+        },
+      ],
+      painEvents: [],
+      gateBlocks: [],
+      stats: {
+        failureCount: 1,
+        totalPainEvents: 0,
+        totalGateBlocks: 0,
+        totalAssistantTurns: 2,
+        totalToolCalls: 1,
+      },
+    };
+
+    const config: TrinityConfig = {
+      useTrinity: true,
+      maxCandidates: 3,
+      useStubs: true,
+    };
+
+    const result = runTrinity({ snapshot: snapshot as any, principleId: 'T-08', config });
+
+    // The stub Dreamer generates candidates mentioning "failing operation" and "config.json"
+    // The snapshot has a Grep failure with "search timed out"
+    // With the normalized token matching: badDecisionTokens = {retry,faili,oper,diagnos,root,caus}
+    // and evidenceTokens = {search,timed,after,seconds,timedout} — no overlap → extraction fails
+    // So result.success must be false with a Hallucinated failure.
+    expect(result.success).toBe(false);
+    expect(result.failures.some(f => f.reason?.includes('Hallucinated'))).toBe(true);
+  });
+});