principles-disciple 1.46.0 → 1.48.0

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "principles-disciple",
   "name": "Principles Disciple",
   "description": "Evolutionary programming agent framework with strategic guardrails and reflection loops.",
-  "version": "1.46.0",
+  "version": "1.48.0",
   "skills": [
     "./skills"
   ],
@@ -76,8 +76,8 @@
     }
   },
   "buildFingerprint": {
-    "gitSha": "9ae5cc1407da",
-    "bundleMd5": "68bc85f0121a780b83931b7ed5491b97",
-    "builtAt": "2026-04-16T02:19:50.219Z"
+    "gitSha": "70500e1475ef",
+    "bundleMd5": "607cfbcb4534d2cfdc45cb0cd019cd0b",
+    "builtAt": "2026-04-16T03:41:17.317Z"
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "principles-disciple",
-  "version": "1.46.0",
+  "version": "1.48.0",
   "description": "Native OpenClaw plugin for Principles Disciple",
   "type": "module",
   "main": "./dist/bundle.js",

package/src/core/evolution-reducer.ts

@@ -21,9 +21,23 @@ import type {
   PrincipleSuggestedRule,
 } from './evolution-types.js';
 import { isCompleteDetectorMetadata } from './evolution-types.js';
-import { updateTrainingStore, addPrincipleToLedger, type LedgerPrinciple } from './principle-tree-ledger.js';
+import { updateTrainingStore, addPrincipleToLedger, updatePrinciple, type LedgerPrinciple } from './principle-tree-ledger.js';
+import { PrincipleCompiler } from './principle-compiler/index.js';
+
+/**
+ * Wrapper for updatePrinciple calls in the compilation retry path.
+ * If updatePrinciple throws, logs the error instead of propagating —
+ * compilation retry state is best-effort and should not crash principle creation.
+ */
+function updateRetryCount(stateDir: string, workspaceDir: string, principleId: string, count: number): void {
+  try {
+    updatePrinciple(stateDir, principleId, { compilationRetryCount: count });
+  } catch (err) {
+    SystemLogger.log(workspaceDir, 'RETRY_COUNT_UPDATE_FAILED',
+      `Failed to update compilationRetryCount for ${principleId}: ${String(err)}`);
+  }
+}
 
- 
 export interface EvolutionReducer {
 
   emit(_event: EvolutionLoopEvent): void;
@@ -420,6 +434,36 @@ export class EvolutionReducerImpl implements EvolutionReducer {
         };
         addPrincipleToLedger(this.stateDir, ledgerPrinciple);
         SystemLogger.log(this.workspaceDir, 'LEDGER_PRINCIPLE_ADDED', `Principle ${principleId} added to ledger tree`);
+
+        // Sync compile: attempt to compile immediately unless evaluability is manual_only.
+        // Failures are not fatal — heartbeat backfill will retry automatically.
+        if (evaluability !== 'manual_only' && this.stateDir) {
+          const trajectory = TrajectoryRegistry.get(this.workspaceDir);
+          const compiler = new PrincipleCompiler(this.stateDir, trajectory);
+          try {
+            const result = compiler.compileOne(principleId);
+            if (result.success) {
+              // Reset retry count on success
+              updatePrinciple(this.stateDir, principleId, { compilationRetryCount: undefined });
+              SystemLogger.log(this.workspaceDir, 'COMPILE_SUCCESS', `Principle ${principleId} compiled successfully`);
+            } else {
+              // Compile returned failure — queue for backfill retry (count=0 means "queued", Phase 2 will pick it up).
+              // This gives exactly 5 total attempts before exhaustion (backfill: 0-4, sync: 0-4).
+              updateRetryCount(this.stateDir, this.workspaceDir, principleId, 0);
+              SystemLogger.log(
+                this.workspaceDir, 'COMPILE_FAILED',
+                `Principle ${principleId} compile failed: ${result.reason ?? 'unknown'} (attempt 1/5)`
+              );
+            }
+          } catch (compileErr) {
+            // Unexpected error during compilation — queue for backfill retry
+            updateRetryCount(this.stateDir, this.workspaceDir, principleId, 0);
+            SystemLogger.log(
+              this.workspaceDir, 'COMPILE_FAILED',
+              `Principle ${principleId} compile threw: ${String(compileErr)} (attempt 1/5)`
+            );
+          }
+        }
       } catch (err) {
         SystemLogger.log(this.workspaceDir, 'LEDGER_PRINCIPLE_ADD_FAILED', `Failed to add ${principleId} to ledger tree: ${String(err)}`);
       }

package/src/service/evolution-worker.ts

@@ -15,6 +15,7 @@ import { initPersistence, flushAllSessions } from '../core/session-tracker.js';
 import { addDiagnosticianTask, completeDiagnosticianTask } from '../core/diagnostician-task-store.js';
 import { getEvolutionLogger } from '../core/evolution-logger.js';
 import type { TaskKind, TaskPriority } from '../core/trajectory-types.js';
+import type { PrincipleEvaluability } from '../types/principle-tree-schema.js';
 export type { TaskKind, TaskPriority } from '../core/trajectory-types.js';
 import { atomicWriteFileSync } from '../utils/io.js';
 
@@ -37,6 +38,8 @@ import {
     type NocturnalSessionSnapshot,
 } from '../core/nocturnal-trajectory-extractor.js';
 import { validateNocturnalSnapshotIngress } from '../core/nocturnal-snapshot-contract.js';
+import { PrincipleCompiler } from '../core/principle-compiler/index.js';
+import { loadLedger, updatePrinciple } from '../core/principle-tree-ledger.js';
 import { isExpectedSubagentError } from './subagent-workflow/subagent-error-utils.js';
 import { readPainFlagContract } from '../core/pain.js';
 import { CorrectionObserverWorkflowManager, correctionObserverWorkflowSpec } from './subagent-workflow/correction-observer-workflow-manager.js';
@@ -567,8 +570,166 @@ async function checkPainFlag(wctx: WorkspaceContext, logger: PluginLogger): Prom
     return result;
 }
 
- 
- 
+/**
+ * Process compilation backfill and retry loop.
+ * Phase 1 — Backfill: on first call, scan for old principles (compilationRetryCount === undefined)
+ *            with evaluability !== 'manual_only' and no active implementation, queue them (set to 0).
+ * Phase 2 — Retry: compile all principles with compilationRetryCount >= 0.
+ * After 5 consecutive failures, downgrades to manual_only and logs COMPILE_EXHAUSTED.
+ */
+export async function processCompilationBackfill(
+    wctx: WorkspaceContext,
+    logger: PluginLogger,
+): Promise<void> {
+    if (!wctx.stateDir) return;
+
+    let ledger: ReturnType<typeof loadLedger>;
+    try {
+        ledger = loadLedger(wctx.stateDir);
+    } catch (err) {
+        logger?.warn?.(`[PD:EvolutionWorker] CompilationBackfill: failed to load ledger: ${String(err)}`);
+        return;
+    }
+
+    // ── Phase 1: Backfill old principles (runs once per process) ─────────────────
+    const backfillMarkerPath = path.join(wctx.stateDir, 'COMPILATION_BACKFILL_DONE');
+    const hasBackfillRun = fs.existsSync(backfillMarkerPath);
+    if (!hasBackfillRun) {
+        let backfillQueued = 0;
+        for (const [principleId, principle] of Object.entries(ledger.tree.principles)) {
+            if (principle.compilationRetryCount !== undefined) continue; // already processed
+            if (principle.evaluability === 'manual_only') continue;
+            // Check if already has active implementation
+            const hasActiveImpl = Object.values(ledger.tree.implementations).some(
+                (impl) => impl.lifecycleState === 'active' && (
+                    ledger.tree.rules[impl.ruleId]?.principleId === principleId
+                )
+            );
+            if (hasActiveImpl) {
+                // Already compiled — mark as done
+                try {
+                    updatePrinciple(wctx.stateDir, principleId, { compilationRetryCount: undefined });
+                } catch (err) {
+                    SystemLogger.log(wctx.workspaceDir, 'BACKFILL_UPDATE_FAILED',
+                        `Failed to mark principle ${principleId} as done: ${String(err)}`);
+                    continue;
+                }
+            } else {
+                // Needs compilation — queue it
+                try {
+                    updatePrinciple(wctx.stateDir, principleId, { compilationRetryCount: 0 });
+                    backfillQueued++;
+                } catch (err) {
+                    SystemLogger.log(wctx.workspaceDir, 'BACKFILL_UPDATE_FAILED',
+                        `Failed to queue principle ${principleId}: ${String(err)}`);
+                    continue;
+                }
+            }
+        }
+        if (backfillQueued > 0) {
+            SystemLogger.log(wctx.workspaceDir, 'COMPILE_BACKFILL_QUEUED',
+                `Queued ${backfillQueued} old principles for compilation`);
+        }
+        // Write marker so we don't backfill again in this process
+        try {
+            atomicWriteFileSync(backfillMarkerPath, new Date().toISOString());
+        } catch (err) {
+            SystemLogger.log(wctx.workspaceDir, 'BACKFILL_MARKER_WRITE_FAILED',
+                `Failed to write backfill marker: ${String(err)}`);
+        }
+    }
+
+    // ── Phase 2: Retry pending compilations ───────────────────────────────────
+    const trajectory = TrajectoryRegistry.get(wctx.workspaceDir);
+    const compiler = new PrincipleCompiler(wctx.stateDir, trajectory);
+
+    // Re-load ledger after potential backfill updates
+    ledger = loadLedger(wctx.stateDir);
+
+    for (const [principleId, principle] of Object.entries(ledger.tree.principles)) {
+        const count = principle.compilationRetryCount;
+
+        // Skip: not in retry queue (undefined = done/succeeded)
+        if (count === undefined) continue;
+
+        // Skip: already exhausted (count >= 5 means 5 attempts already made)
+        if (count >= 5) continue;
+
+        // Error-isolate each principle so one failure doesn't stop all other retries
+        try {
+            const result = compiler.compileOne(principleId);
+            if (result.success) {
+                tryUpdateRetryCount(wctx.stateDir, wctx.workspaceDir, principleId, undefined);
+                SystemLogger.log(wctx.workspaceDir, 'COMPILE_SUCCESS',
+                    `Principle ${principleId} compiled successfully (attempt ${count + 1})`);
+            } else {
+                const nextCount = count + 1;
+                if (nextCount >= 5) {
+                    // Exhausted: single write to set manual_only (no intermediate count write)
+                    tryUpdatePrinciple(wctx.stateDir, wctx.workspaceDir, principleId, {
+                        evaluability: 'manual_only',
+                        compilationRetryCount: undefined,
+                    });
+                    SystemLogger.log(wctx.workspaceDir, 'COMPILE_EXHAUSTED',
+                        `Principle ${principleId} compilation exhausted after 5 attempts: ${result.reason ?? 'unknown'}`);
+                } else {
+                    tryUpdateRetryCount(wctx.stateDir, wctx.workspaceDir, principleId, nextCount);
+                    SystemLogger.log(wctx.workspaceDir, 'COMPILE_FAILED',
+                        `Principle ${principleId} compile failed: ${result.reason ?? 'unknown'} (attempt ${nextCount}/5)`);
+                }
+            }
+        } catch (compileErr) {
+            const nextCount = count + 1;
+            if (nextCount >= 5) {
+                // Exhausted: single write to set manual_only (no intermediate count write)
+                tryUpdatePrinciple(wctx.stateDir, wctx.workspaceDir, principleId, {
+                    evaluability: 'manual_only',
+                    compilationRetryCount: undefined,
+                });
+                SystemLogger.log(wctx.workspaceDir, 'COMPILE_EXHAUSTED',
+                    `Principle ${principleId} compilation exhausted after 5 attempts: threw ${String(compileErr)}`);
+            } else {
+                tryUpdateRetryCount(wctx.stateDir, wctx.workspaceDir, principleId, nextCount);
+                SystemLogger.log(wctx.workspaceDir, 'COMPILE_FAILED',
+                    `Principle ${principleId} compile threw: ${String(compileErr)} (attempt ${nextCount}/5)`);
+            }
+        }
+    }
+}
+
+/**
+ * Wrapper for updateRetryCount — logs but does not propagate errors.
+ * Errors are silently swallowed: if update fails, the principle stays in its
+ * current retry state and will be picked up again on the next heartbeat.
+ */
+function tryUpdateRetryCount(stateDir: string, workspaceDir: string, principleId: string, count: number | undefined): void {
+    try {
+        updatePrinciple(stateDir, principleId, { compilationRetryCount: count });
+    } catch (err) {
+        SystemLogger.log(workspaceDir, 'RETRY_COUNT_UPDATE_FAILED',
+            `Failed to update retry count for ${principleId}: ${String(err)}`);
+    }
+}
+
+/**
+ * Wrapper for updatePrinciple with multiple fields — logs but does not propagate errors.
+ * Errors are silently swallowed: if update fails, the principle stays in its
+ * current state and will be picked up again on the next heartbeat.
+ */
+function tryUpdatePrinciple(
+    stateDir: string,
+    workspaceDir: string,
+    principleId: string,
+    updates: { evaluability?: PrincipleEvaluability; compilationRetryCount?: number },
+): void {
+    try {
+        updatePrinciple(stateDir, principleId, updates);
+    } catch (err) {
+        SystemLogger.log(workspaceDir, 'RETRY_PRINCIPLE_UPDATE_FAILED',
+            `Failed to update principle ${principleId}: ${String(err)}`);
+    }
+}
+
 async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogger, eventLog: EventLog, api?: OpenClawPluginApi) {
     const queuePath = wctx.resolve('EVOLUTION_QUEUE');
     if (!fs.existsSync(queuePath)) {
@@ -773,7 +934,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
                                 } else {
                                     logger.info(`[PD:EvolutionWorker] Creating principle from report for task ${task.id}`);
                                     const principleId = wctx.evolutionReducer.createPrincipleFromDiagnosis({
-                                        painId: task.id,
+                                        painId: task.painEventId !== undefined ? String(task.painEventId) : task.id,
                                         painType: task.source === 'Human Intervention' ? 'user_frustration' : 'tool_failure',
                                         triggerPattern: principle.trigger_pattern,
                                         action: principle.action,
@@ -871,7 +1032,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
                                 logger.info(`[PD:EvolutionWorker] Diagnostician marked principle as duplicate: ${principle.duplicate_of || 'unknown'} — skipping for task ${task.id}`);
                             } else {
                                 const principleId = wctx.evolutionReducer.createPrincipleFromDiagnosis({
-                                    painId: task.id,
+                                    painId: task.painEventId !== undefined ? String(task.painEventId) : task.id,
                                     painType: task.source === 'Human Intervention' ? 'user_frustration' : 'tool_failure',
                                     triggerPattern: principle.trigger_pattern,
                                     action: principle.action,
@@ -1957,6 +2118,12 @@ export const EvolutionWorkerService: ExtendedEvolutionWorkerService = {
                 const mergedConfig = loadNocturnalConfigMerged(wctx.stateDir);
                 const { sleepReflection: sleepConfig, keywordOptimization: kwOptConfig } = mergedConfig;
 
+                // Compilation backfill: runs on every heartbeat to retry failed compilations.
+                // Fire-and-forget — errors are logged within the function.
+                processCompilationBackfill(wctx, logger).catch((err) => {
+                    logger?.error?.(`[PD:EvolutionWorker] CompilationBackfill threw: ${String(err)}`);
+                });
+
                 const idleResult = checkWorkspaceIdle(wctx.workspaceDir, {});
                 logger?.info?.(`[PD:EvolutionWorker] HEARTBEAT cycle=${new Date().toISOString()} idle=${idleResult.isIdle} idleForMs=${idleResult.idleForMs} userActiveSessions=${idleResult.userActiveSessions} abandonedSessions=${idleResult.abandonedSessionIds.length} lastActivityEpoch=${idleResult.mostRecentActivityAt} triggerMode=${sleepConfig.trigger_mode}`);
 

package/src/types/principle-tree-schema.ts

@@ -76,6 +76,10 @@ export interface Principle {
 
   // Detector metadata (for auto-training eligibility)
   detectorMetadata?: PrincipleDetectorSpec;
+
+  // Compilation retry tracking (for runtime auto-trigger)
+  // undefined = not yet attempted or succeeded; 0 = queued; n >= 1 = retry attempt n
+  compilationRetryCount?: number;
 }
 
 // =========================================================================

package/tests/core/evolution-reducer.compilation-retry.test.ts

@@ -0,0 +1,188 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { EvolutionReducerImpl } from '../../src/core/evolution-reducer.js';
+import { loadLedger } from '../../src/core/principle-tree-ledger.js';
+
+const tempDirs: string[] = [];
+
+function makeTempDir(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pd-evolution-compile-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+// Minimal state dir structure
+function makeStateDir(workspace: string): string {
+  const stateDir = path.join(workspace, '.state');
+  fs.mkdirSync(stateDir, { recursive: true });
+  fs.writeFileSync(path.join(stateDir, 'EVOLUTION_STREAM'), '', 'utf8');
+  fs.writeFileSync(path.join(stateDir, 'PRINCIPLES'), '', 'utf8');
+  fs.writeFileSync(path.join(stateDir, 'evolution_queue.json'), '[]', 'utf8');
+  fs.writeFileSync(path.join(stateDir, 'ledger.json'), JSON.stringify({
+    trainingStore: {},
+    tree: { principles: {}, rules: {}, implementations: {}, metrics: {}, lastUpdated: new Date().toISOString() },
+  }), 'utf8');
+  return stateDir;
+}
+
+afterEach(() => {
+  for (const dir of tempDirs.splice(0)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// ---------------------------------------------------------------------------
+// createPrincipleFromDiagnosis — compilationRetryCount initialization
+// ---------------------------------------------------------------------------
+
+describe('createPrincipleFromDiagnosis — compilationRetryCount initialization', () => {
+  it('sets compilationRetryCount=0 when evaluability is weak_heuristic (queued for compilation)', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+    const reducer = new EvolutionReducerImpl({ workspaceDir: workspace, stateDir });
+
+    const id = reducer.createPrincipleFromDiagnosis({
+      painId: `pain-weak-heuristic-${Date.now()}`,
+      painType: 'tool_failure',
+      triggerPattern: 'bash rm fails',
+      action: 'verify file exists before rm',
+      source: 'test-compilation-retry',
+      evaluability: 'weak_heuristic',
+    });
+
+    expect(id).not.toBeNull();
+    const ledger = loadLedger(stateDir);
+    const principle = ledger.tree.principles[id as string];
+    expect(principle).toBeDefined();
+    // Compilation queued: count >= 0 means queued
+    expect(typeof principle?.compilationRetryCount).toBe('number');
+    expect(principle?.compilationRetryCount).toBeGreaterThanOrEqual(0);
+  });
+
+  it('sets compilationRetryCount=0 when evaluability is deterministic', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+    const reducer = new EvolutionReducerImpl({ workspaceDir: workspace, stateDir });
+
+    const id = reducer.createPrincipleFromDiagnosis({
+      painId: `pain-deterministic-${Date.now()}`,
+      painType: 'tool_failure',
+      triggerPattern: 'edit without read',
+      action: 'always read before edit',
+      source: 'test-compilation-retry',
+      evaluability: 'deterministic',
+    });
+
+    expect(id).not.toBeNull();
+    const ledger = loadLedger(stateDir);
+    const principle = ledger.tree.principles[id as string];
+    expect(principle).toBeDefined();
+    expect(typeof principle?.compilationRetryCount).toBe('number');
+    expect(principle?.compilationRetryCount).toBeGreaterThanOrEqual(0);
+  });
+
+  it('does NOT set compilationRetryCount when evaluability is manual_only', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+    const reducer = new EvolutionReducerImpl({ workspaceDir: workspace, stateDir });
+
+    const id = reducer.createPrincipleFromDiagnosis({
+      painId: `pain-manual-only-${Date.now()}`,
+      painType: 'tool_failure',
+      triggerPattern: 'generic pain',
+      action: 'be more careful',
+      source: 'test-compilation-retry',
+      evaluability: 'manual_only',
+    });
+
+    expect(id).not.toBeNull();
+    const ledger = loadLedger(stateDir);
+    const principle = ledger.tree.principles[id as string];
+    expect(principle).toBeDefined();
+    // manual_only principles should NOT be queued for compilation
+    expect(principle?.compilationRetryCount).toBeUndefined();
+    expect(principle?.evaluability).toBe('manual_only');
+  });
+
+  it('defaults to weak_heuristic and queues for compilation when no evaluability provided', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+    const reducer = new EvolutionReducerImpl({ workspaceDir: workspace, stateDir });
+
+    const id = reducer.createPrincipleFromDiagnosis({
+      painId: `pain-default-${Date.now()}`,
+      painType: 'tool_failure',
+      triggerPattern: 'some pattern',
+      action: 'some action',
+      source: 'test-compilation-retry',
+    });
+
+    expect(id).not.toBeNull();
+    const ledger = loadLedger(stateDir);
+    const principle = ledger.tree.principles[id as string];
+    expect(principle).toBeDefined();
+    // default evaluability is weak_heuristic, which should queue for compilation
+    expect(typeof principle?.compilationRetryCount).toBe('number');
+    expect(principle?.compilationRetryCount).toBeGreaterThanOrEqual(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// createPrincipleFromDiagnosis — compilationRetryCount increments on compile failure
+// ---------------------------------------------------------------------------
+
+describe('createPrincipleFromDiagnosis — compilationRetryCount increments on failure', () => {
+  it('increments to 1 when compilation fails (no trajectory data)', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+    const reducer = new EvolutionReducerImpl({ workspaceDir: workspace, stateDir });
+
+    const id = reducer.createPrincipleFromDiagnosis({
+      painId: `pain-fail-${Date.now()}`,
+      painType: 'tool_failure',
+      triggerPattern: 'unknown tool',
+      action: 'do nothing',
+      source: 'test-compilation-retry',
+      evaluability: 'weak_heuristic',
+    });
+
+    expect(id).not.toBeNull();
+    const ledger = loadLedger(stateDir);
+    const principle = ledger.tree.principles[id as string];
+    expect(principle).toBeDefined();
+    // Compilation was attempted and failed (no trajectory data) → count should be 0
+    // (sync failure sets count=0 so Phase 2 gets exactly 5 total attempts)
+    expect(principle?.compilationRetryCount).toBe(0);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Principle schema — compilationRetryCount field exists and persists
+// ---------------------------------------------------------------------------
+
+describe('Principle schema — compilationRetryCount field persists', () => {
+  it('compilationRetryCount is stored and retrieved correctly', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+    const reducer = new EvolutionReducerImpl({ workspaceDir: workspace, stateDir });
+
+    const id = reducer.createPrincipleFromDiagnosis({
+      painId: `pain-schema-${Date.now()}`,
+      painType: 'tool_failure',
+      triggerPattern: 'test pattern',
+      action: 'test action',
+      source: 'test-compilation-retry',
+      evaluability: 'weak_heuristic',
+    });
+
+    expect(id).not.toBeNull();
+    // Reload ledger to verify persistence
+    const ledger = loadLedger(stateDir);
+    const principle = ledger.tree.principles[id as string];
+    expect(principle).toBeDefined();
+    expect(typeof principle?.compilationRetryCount).toBe('number');
+    expect(principle?.compilationRetryCount).toBeGreaterThanOrEqual(0);
+  });
+});

package/tests/integration/pain-id-chain-e2e.test.ts

@@ -0,0 +1,253 @@
+/**
+ * E2E Test: Pain ID Chain — pain event → createPrincipleFromDiagnosis → compile → RuleHost
+ *
+ * Tests the complete chain:
+ *   1. recordPainEvent() returns AUTOINCREMENT row ID as number
+ *   2. createPrincipleFromDiagnosis(painId: String(painEventId))
+ *   3. derivedFromPainIds stores the stringified numeric ID
+ *   4. PrincipleCompiler.compileOne() succeeds (registers active implementation)
+ *   5. RuleHost.evaluate(matching input) → block
+ *   6. RuleHost.evaluate(non-matching input) → undefined (passthrough)
+ *
+ * Pain ID chain fixed in commits 4b0dce59 and 0146bbb7:
+ *   - recordPainEvent() now returns real AUTOINCREMENT ID (was -1)
+ *   - derivedFromPainIds now stores String(painId) correctly
+ *   - LedgerPrinciple.derivedFromPainIds used by compiler reflection
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { TrajectoryDatabase } from '../../src/core/trajectory.js';
+import { PrincipleCompiler } from '../../src/core/principle-compiler/compiler.js';
+import { RuleHost } from '../../src/core/rule-host.js';
+import { EvolutionReducerImpl } from '../../src/core/evolution-reducer.js';
+import {
+  loadLedger,
+} from '../../src/core/principle-tree-ledger.js';
+import type { RuleHostInput } from '../../src/core/rule-host-types.js';
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+interface TestWorkspace {
+  workspaceDir: string;
+  stateDir: string;
+  trajectory: TrajectoryDatabase;
+  reducer: EvolutionReducerImpl;
+}
+
+function createTestWorkspace(): TestWorkspace {
+  const workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pd-pain-chain-e2e-'));
+  const stateDir = path.join(workspaceDir, '.state');
+  fs.mkdirSync(stateDir, { recursive: true });
+
+  const trajectory = new TrajectoryDatabase({ workspaceDir });
+  const reducer = new EvolutionReducerImpl({ workspaceDir, stateDir });
+
+  return { workspaceDir, stateDir, trajectory, reducer };
+}
+
+function disposeTestWorkspace(ws: TestWorkspace): void {
+  ws.trajectory.dispose();
+  fs.rmSync(ws.workspaceDir, { recursive: true, force: true });
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe('Pain ID Chain E2E: pain event → principle → compile → RuleHost', () => {
+  let ws: TestWorkspace;
+
+  beforeEach(() => {
+    ws = createTestWorkspace();
+  });
+
+  afterEach(() => {
+    disposeTestWorkspace(ws);
+  });
+
+  it('full chain: pain event ID → createPrincipleFromDiagnosis → derivedFromPainIds → compile → block', () => {
+    const sessionId = 'session-pain-chain-001';
+
+    // ── Step 1: Record tool call + pain event, capture the returned AUTOINCREMENT ID ──
+    ws.trajectory.recordToolCall({
+      sessionId,
+      toolName: 'bash',
+      outcome: 'failure',
+      errorType: 'command_not_found',
+      errorMessage: 'heartbeat: command not found',
+      paramsJson: { command: 'heartbeat --status' },
+    });
+
+    // recordPainEvent() returns the real AUTOINCREMENT row ID as a number (fix from 4b0dce59)
+    const painEventId = ws.trajectory.recordPainEvent({
+      sessionId,
+      source: 'gate_block',
+      score: 80,
+      reason: 'Blocked bash heartbeat command due to unsafe operation',
+      severity: 'moderate',
+      origin: 'system_infer',
+    });
+
+    // Verify painEventId is a positive integer (real AUTOINCREMENT, not -1)
+    expect(typeof painEventId).toBe('number');
+    expect(painEventId).toBeGreaterThan(0);
+
+    // ── Step 2: Create principle via createPrincipleFromDiagnosis with stringified pain ID ──
+    const triggerPattern = 'heartbeat.*bash';
+    const action = 'Block heartbeat commands in bash';
+    const painIdStr = String(painEventId);
+
+    const principleId = ws.reducer.createPrincipleFromDiagnosis({
+      painId: painIdStr,
+      painType: 'tool_failure',
+      triggerPattern,
+      action,
+      source: 'pain-id-chain-e2e',
+      evaluability: 'deterministic',
+    });
+
+    expect(principleId).not.toBeNull();
+    expect(typeof principleId).toBe('string');
+
+    // ── Step 3: Verify derivedFromPainIds in the ledger contains the stringified pain ID ──
+    const ledger = loadLedger(ws.stateDir);
+    const ledgerPrinciple = ledger.tree.principles[principleId!];
+    expect(ledgerPrinciple).toBeDefined();
+    expect(ledgerPrinciple!.derivedFromPainIds).toContain(painIdStr);
+
+    // ── Step 4: Compile the principle (registers active implementation) ──
+    const compiler = new PrincipleCompiler(ws.stateDir, ws.trajectory);
+    const compileResult = compiler.compileOne(principleId!);
+
+    expect(compileResult.success).toBe(true);
+    expect(compileResult.principleId).toBe(principleId);
+    expect(compileResult.code).toBeDefined();
+    expect(compileResult.code).toContain('heartbeat');
+    expect(compileResult.ruleId).toBeDefined();
+    expect(compileResult.implementationId).toBeDefined();
+
+    // Verify implementation is active
+    const updatedLedger = loadLedger(ws.stateDir);
+    const impl = updatedLedger.tree.implementations[compileResult.implementationId!];
+    expect(impl.lifecycleState).toBe('active');
+
+    // ── Step 5: RuleHost.evaluate(matching input) → block ──
+    const host = new RuleHost(ws.stateDir, { warn: () => {} });
+
+    const matchingInput: RuleHostInput = {
+      action: {
+        toolName: 'bash',
+        normalizedPath: null,
+        paramsSummary: { command: 'heartbeat --status' },
+      },
+      workspace: {
+        isRiskPath: false,
+        planStatus: 'NONE',
+        hasPlanFile: false,
+      },
+      session: {
+        sessionId: 'session-eval-001',
+        currentGfi: 50,
+        recentThinking: false,
+      },
+      evolution: {
+        epTier: 0,
+      },
+      derived: {
+        estimatedLineChanges: 0,
+        bashRisk: 'unknown',
+      },
+    };
+
+    const blockResult = host.evaluate(matchingInput);
+
+    expect(blockResult).toBeDefined();
+    expect(blockResult!.decision).toBe('block');
+    expect(blockResult!.matched).toBe(true);
+    expect(blockResult!.reason).toContain(principleId);
+
+    // ── Step 6: RuleHost.evaluate(non-matching input) → undefined (passthrough) ──
+    const nonMatchingInput: RuleHostInput = {
+      action: {
+        toolName: 'Read',
+        normalizedPath: '/some/file.txt',
+        paramsSummary: {},
+      },
+      workspace: {
+        isRiskPath: false,
+        planStatus: 'NONE',
+        hasPlanFile: false,
+      },
+      session: {
+        sessionId: 'session-eval-002',
+        currentGfi: 50,
+        recentThinking: false,
+      },
+      evolution: {
+        epTier: 0,
+      },
+      derived: {
+        estimatedLineChanges: 0,
+        bashRisk: 'unknown',
+      },
+    };
+
+    const passResult = host.evaluate(nonMatchingInput);
+    expect(passResult).toBeUndefined();
+  });
+
+  it('compileOne returns failure for non-existent principle ID', () => {
+    const compiler = new PrincipleCompiler(ws.stateDir, ws.trajectory);
+    const badResult = compiler.compileOne('non-existent-principle-id');
+    expect(badResult.success).toBe(false);
+  });
+
+  it('recordPainEvent returns sequential IDs for multiple events', () => {
+    const sessionId = 'session-seq-001';
+
+    ws.trajectory.recordToolCall({
+      sessionId,
+      toolName: 'bash',
+      outcome: 'failure',
+      errorType: 'command_not_found',
+      errorMessage: 'test error 1',
+      paramsJson: { command: 'test1' },
+    });
+
+    const id1 = ws.trajectory.recordPainEvent({
+      sessionId,
+      source: 'gate_block',
+      score: 50,
+      reason: 'First pain event',
+      severity: 'low',
+      origin: 'system_infer',
+    });
+
+    ws.trajectory.recordToolCall({
+      sessionId,
+      toolName: 'bash',
+      outcome: 'failure',
+      errorType: 'command_not_found',
+      errorMessage: 'test error 2',
+      paramsJson: { command: 'test2' },
+    });
+
+    const id2 = ws.trajectory.recordPainEvent({
+      sessionId,
+      source: 'gate_block',
+      score: 60,
+      reason: 'Second pain event',
+      severity: 'moderate',
+      origin: 'system_infer',
+    });
+
+    expect(id2).toBeGreaterThan(id1);
+    expect(typeof id1).toBe('number');
+    expect(typeof id2).toBe('number');
+  });
+});

package/tests/service/evolution-worker.compilation-backfill.test.ts

@@ -0,0 +1,199 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { addPrincipleToLedger, loadLedger, type LedgerPrinciple } from '../../src/core/principle-tree-ledger.js';
+import type { WorkspaceContext } from '../../src/core/workspace-context.js';
+import type { PluginLogger } from '../../src/openclaw-sdk.js';
+import { processCompilationBackfill } from '../../src/service/evolution-worker.js';
+
+const tempDirs: string[] = [];
+
+function makeTempDir(): string {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'pd-backfill-test-'));
+  tempDirs.push(dir);
+  return dir;
+}
+
+function makeStateDir(workspace: string): string {
+  const stateDir = path.join(workspace, '.state');
+  fs.mkdirSync(stateDir, { recursive: true });
+  fs.writeFileSync(path.join(stateDir, 'EVOLUTION_STREAM'), '', 'utf8');
+  fs.writeFileSync(path.join(stateDir, 'PRINCIPLES'), '', 'utf8');
+  fs.writeFileSync(path.join(stateDir, 'evolution_queue.json'), '[]', 'utf8');
+  fs.writeFileSync(path.join(stateDir, 'ledger.json'), JSON.stringify({
+    trainingStore: {},
+    tree: { principles: {}, rules: {}, implementations: {}, metrics: {}, lastUpdated: new Date().toISOString() },
+  }), 'utf8');
+  return stateDir;
+}
+
+function makeWctx(workspace: string, stateDir: string): WorkspaceContext {
+  return {
+    workspaceDir: workspace,
+    stateDir,
+    resolve: (file: string) => path.join(stateDir, file),
+  } as unknown as WorkspaceContext;
+}
+
+function makePrinciple(id: string, overrides: Partial<LedgerPrinciple> = {}): LedgerPrinciple {
+  return {
+    id,
+    version: 1,
+    text: `principle ${id}`,
+    triggerPattern: 'test',
+    action: 'test action',
+    status: 'active',
+    priority: 'P1',
+    scope: 'general',
+    evaluability: 'weak_heuristic',
+    compilationRetryCount: undefined,
+    ruleIds: [],
+    conflictsWithPrincipleIds: [],
+    derivedFromPainIds: [],
+    valueScore: 0,
+    adherenceRate: 0,
+    painPreventedCount: 0,
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    ...overrides,
+  } as LedgerPrinciple;
+}
+
+const noopLogger: PluginLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+};
+
+afterEach(() => {
+  vi.restoreAllMocks();
+  for (const dir of tempDirs.splice(0)) {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+});
+
+// ---------------------------------------------------------------------------
+// Phase 1: Backfill
+// ---------------------------------------------------------------------------
+
+describe('processCompilationBackfill — Phase 1 backfill', () => {
+  it('sets compilationRetryCount=0 for old principles without retry count', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+
+    addPrincipleToLedger(stateDir, makePrinciple('P_001', {
+      evaluability: 'weak_heuristic',
+      compilationRetryCount: undefined,
+    }));
+
+    const wctx = makeWctx(workspace, stateDir);
+    processCompilationBackfill(wctx, noopLogger);
+
+    const ledger = loadLedger(stateDir);
+    // Phase 1 sets count=0, then Phase 2 runs and increments to 1 (compilation fails without trajectory data)
+    // The key assertion: count was set to 0 at some point (proves backfill ran)
+    expect(ledger.tree.principles['P_001'].compilationRetryCount).toBeGreaterThanOrEqual(0);
+  });
+
+  it('skips principles with manual_only evaluability', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+
+    addPrincipleToLedger(stateDir, makePrinciple('P_002', {
+      evaluability: 'manual_only',
+      compilationRetryCount: undefined,
+    }));
+
+    const wctx = makeWctx(workspace, stateDir);
+    processCompilationBackfill(wctx, noopLogger);
+
+    const ledger = loadLedger(stateDir);
+    expect(ledger.tree.principles['P_002'].compilationRetryCount).toBeUndefined();
+  });
+
+  it('writes COMPILATION_BACKFILL_DONE marker after backfill', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+
+    addPrincipleToLedger(stateDir, makePrinciple('P_003', {
+      evaluability: 'weak_heuristic',
+      compilationRetryCount: undefined,
+    }));
+
+    const wctx = makeWctx(workspace, stateDir);
+    processCompilationBackfill(wctx, noopLogger);
+
+    const markerPath = path.join(stateDir, 'COMPILATION_BACKFILL_DONE');
+    expect(fs.existsSync(markerPath)).toBe(true);
+  });
+
+  it('does not re-backfill if marker already exists', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+
+    // Pre-write the marker so Phase 1 (backfill) is skipped
+    const markerPath = path.join(stateDir, 'COMPILATION_BACKFILL_DONE');
+    fs.writeFileSync(markerPath, new Date().toISOString(), 'utf8');
+
+    // Add principle with compilationRetryCount already set to a high value
+    // (simulating already-processed-by-Phase2)
+    addPrincipleToLedger(stateDir, makePrinciple('P_004', {
+      evaluability: 'weak_heuristic',
+      compilationRetryCount: 2,
+    }));
+
+    const wctx = makeWctx(workspace, stateDir);
+    processCompilationBackfill(wctx, noopLogger);
+
+    const ledger = loadLedger(stateDir);
+    // Phase 1 was skipped (marker exists), but Phase 2 still ran and incremented count
+    // So count goes from 2 -> 3 (compilation fails without trajectory data)
+    expect(ledger.tree.principles['P_004'].compilationRetryCount).toBe(3);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// Phase 2: Retry loop
+// ---------------------------------------------------------------------------
+
+describe('processCompilationBackfill — Phase 2 retry loop', () => {
+  it('increments count on compile failure (below exhaustion)', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+
+    // Principle queued with count=1 — next failure should make it 2
+    addPrincipleToLedger(stateDir, makePrinciple('P_012', {
+      evaluability: 'weak_heuristic',
+      compilationRetryCount: 1,
+    }));
+
+    const wctx = makeWctx(workspace, stateDir);
+    processCompilationBackfill(wctx, noopLogger);
+
+    const ledger = loadLedger(stateDir);
+    // Compilation fails (no trajectory data), so count increments
+    expect(ledger.tree.principles['P_012'].compilationRetryCount).toBe(2);
+    expect(ledger.tree.principles['P_012'].evaluability).toBe('weak_heuristic');
+  });
+
+  it('downgrades to manual_only after 5 consecutive failures', () => {
+    const workspace = makeTempDir();
+    const stateDir = makeStateDir(workspace);
+
+    // Principle at count=4 — next failure exhausts it
+    addPrincipleToLedger(stateDir, makePrinciple('P_011', {
+      evaluability: 'weak_heuristic',
+      compilationRetryCount: 4,
+    }));
+
+    const wctx = makeWctx(workspace, stateDir);
+    processCompilationBackfill(wctx, noopLogger);
+
+    const ledger = loadLedger(stateDir);
+    // Compilation fails (no trajectory), count becomes 5 >= 5, downgrades to manual_only
+    expect(ledger.tree.principles['P_011'].evaluability).toBe('manual_only');
+    expect(ledger.tree.principles['P_011'].compilationRetryCount).toBeUndefined();
+  });
+});