principles-disciple 1.35.0 → 1.37.0

package/src/service/evolution-queue-migration.ts

@@ -0,0 +1,173 @@
+/**
+ * Evolution Queue Migration — V1 to V2 Schema
+ *
+ * Pure transformation functions and shared queue types.
+ */
+
+import type { TaskKind, TaskPriority } from '../core/trajectory-types.js';
+
+// Re-export TaskKind and TaskPriority for convenience
+export type { TaskKind, TaskPriority } from '../core/trajectory-types.js';
+
+/**
+ * Queue item status values.
+ */
+export type QueueStatus = 'pending' | 'in_progress' | 'completed' | 'failed' | 'canceled';
+
+/**
+ * Task resolution strings.
+ */
+export type TaskResolution =
+    | 'marker_detected'
+    | 'auto_completed_timeout'
+    | 'failed_max_retries'
+    | 'runtime_unavailable'
+    | 'canceled'
+    | 'late_marker_principle_created'
+    | 'late_marker_no_principle'
+    | 'stub_fallback'
+    | 'skipped_thin_violation';
+
+/**
+ * Recent pain context for sleep_reflection tasks.
+ * Attached to queue items to provide pain signal context.
+ */
+export interface RecentPainContext {
+    mostRecent: { score: number; source: string; reason: string; timestamp: string; sessionId: string } | null;
+    recentPainCount: number;
+    recentMaxPainScore: number;
+}
+
+/**
+ * Default values for new V2 fields when migrating legacy items.
+ */
+export const DEFAULT_TASK_KIND: TaskKind = 'pain_diagnosis';
+export const DEFAULT_PRIORITY: TaskPriority = 'medium';
+export const DEFAULT_MAX_RETRIES = 3;
+
+/**
+ * Legacy (pre-V2) queue item schema.
+ */
+export interface LegacyEvolutionQueueItem {
+    id: string;
+    source: string;
+    traceId?: string;
+    task?: string;
+    score: number;
+    reason: string;
+    timestamp: string;
+    enqueued_at?: string;
+    started_at?: string;
+    completed_at?: string;
+    assigned_session_key?: string;
+    trigger_text_preview?: string;
+    status?: string;
+    resolution?: string;
+    session_id?: string;
+    agent_id?: string;
+    taskKind?: string;
+    priority?: string;
+    retryCount?: number;
+    maxRetries?: number;
+    lastError?: string;
+    resultRef?: string;
+}
+
+/**
+ * V2 queue item schema.
+ */
+export interface EvolutionQueueItem {
+    id: string;
+    taskKind: TaskKind;
+    priority: TaskPriority;
+    source: string;
+    traceId?: string;
+    task?: string;
+    score: number;
+    reason: string;
+    timestamp: string;
+    enqueued_at?: string;
+    started_at?: string;
+    completed_at?: string;
+    assigned_session_key?: string;
+    trigger_text_preview?: string;
+    status: QueueStatus;
+    resolution?: TaskResolution;
+    session_id?: string;
+    agent_id?: string;
+    retryCount: number;
+    maxRetries: number;
+    lastError?: string;
+    resultRef?: string;
+    recentPainContext?: RecentPainContext;
+}
+
+export type RawQueueItem = Record<string, unknown>;
+
+/**
+ * Migrate a legacy queue item to V2 schema.
+ * Old items without taskKind are assumed to be pain_diagnosis for backward compatibility.
+ */
+export function migrateToV2(item: LegacyEvolutionQueueItem): {
+    id: string;
+    taskKind: TaskKind;
+    priority: TaskPriority;
+    source: string;
+    traceId?: string;
+    task?: string;
+    score: number;
+    reason: string;
+    timestamp: string;
+    enqueued_at?: string;
+    started_at?: string;
+    completed_at?: string;
+    assigned_session_key?: string;
+    trigger_text_preview?: string;
+    status: QueueStatus;
+    resolution?: TaskResolution;
+    session_id?: string;
+    agent_id?: string;
+    retryCount: number;
+    maxRetries: number;
+    lastError?: string;
+    resultRef?: string;
+} {
+    return {
+        id: item.id,
+        taskKind: (item.taskKind as TaskKind) || DEFAULT_TASK_KIND,
+        priority: (item.priority as TaskPriority) || DEFAULT_PRIORITY,
+        source: item.source,
+        traceId: item.traceId,
+        task: item.task,
+        score: item.score,
+        reason: item.reason,
+        timestamp: item.timestamp,
+        enqueued_at: item.enqueued_at,
+        started_at: item.started_at,
+        completed_at: item.completed_at,
+        assigned_session_key: item.assigned_session_key,
+        trigger_text_preview: item.trigger_text_preview,
+        status: (item.status as QueueStatus) || 'pending',
+        resolution: item.resolution as TaskResolution | undefined,
+        session_id: item.session_id,
+        agent_id: item.agent_id,
+        retryCount: item.retryCount || 0,
+        maxRetries: item.maxRetries || DEFAULT_MAX_RETRIES,
+        lastError: item.lastError,
+        resultRef: item.resultRef,
+    };
+}
+
+/**
+ * Check if an item is a legacy (pre-V2) queue item.
+ */
+export function isLegacyQueueItem(item: RawQueueItem): boolean {
+    return item && typeof item === 'object' && !('taskKind' in item);
+}
+
+/**
+ * Migrate entire queue to V2 schema if needed.
+ */
+export function migrateQueueToV2(queue: RawQueueItem[]): ReturnType<typeof migrateToV2>[] {
+    return queue.map(item => isLegacyQueueItem(item) ? migrateToV2(item as unknown as LegacyEvolutionQueueItem) : item as unknown as ReturnType<typeof migrateToV2>);
+}

package/src/service/evolution-worker.ts

@@ -16,11 +16,10 @@ import { getEvolutionLogger } from '../core/evolution-logger.js';
 import type { TaskKind, TaskPriority } from '../core/trajectory-types.js';
 export type { TaskKind, TaskPriority } from '../core/trajectory-types.js';
 import { LockUnavailableError } from '../config/index.js';
-import { PAIN_QUEUE_DEDUP_WINDOW_MS } from '../config/defaults/runtime.js';
 import { checkWorkspaceIdle, checkCooldown } from './nocturnal-runtime.js';
 import { loadNocturnalConfig } from './nocturnal-config.js';
 import { WorkflowStore } from './subagent-workflow/workflow-store.js';
-import type { WorkflowRow, RecentPainContext } from './subagent-workflow/types.js';
+import type { WorkflowRow } from './subagent-workflow/types.js';
 import { EmpathyObserverWorkflowManager } from './subagent-workflow/empathy-observer-workflow-manager.js';
 import { DeepReflectWorkflowManager } from './subagent-workflow/deep-reflect-workflow-manager.js';
 import { NocturnalWorkflowManager, nocturnalWorkflowSpec } from './subagent-workflow/nocturnal-workflow-manager.js';
@@ -32,22 +31,14 @@ import {
 import { validateNocturnalSnapshotIngress } from '../core/nocturnal-snapshot-contract.js';
 import { isExpectedSubagentError } from './subagent-workflow/subagent-error-utils.js';
 import { readPainFlagContract } from '../core/pain.js';
-import { CorrectionObserverWorkflowManager, correctionObserverWorkflowSpec } from './correction-observer-workflow-manager.js';
-import type { CorrectionObserverPayload } from './correction-observer-types.js';
+import { CorrectionObserverWorkflowManager, correctionObserverWorkflowSpec } from './subagent-workflow/correction-observer-workflow-manager.js';
+import type { CorrectionObserverPayload } from './subagent-workflow/correction-observer-types.js';
 import { KeywordOptimizationService } from './keyword-optimization-service.js';
 import { TrajectoryRegistry } from '../core/trajectory.js';
 import { CorrectionCueLearner } from '../core/correction-cue-learner.js';
-import { WORKFLOW_TTL_MS } from '../config/defaults/runtime.js';
-import { OpenClawTrinityRuntimeAdapter } from '../core/nocturnal-trinity.js';
 
-/**
- * Atomic file write — write to temp then rename to prevent partial writes on crash.
- */
-function atomicWriteFileSync(filePath: string, data: string): void {
-  const tmpPath = filePath + '.tmp';
-  fs.writeFileSync(tmpPath, data, 'utf8');
-  fs.renameSync(tmpPath, filePath);
-}
+const WORKFLOW_TTL_MS = 5 * 60 * 1000; // 5 minutes default TTL for helper workflows
+import { OpenClawTrinityRuntimeAdapter } from '../core/nocturnal-trinity.js';
 
 // ── Workflow Watchdog ────────────────────────────────────────────────────────
 // Detects stale/orphaned workflows, invalid results, and cleanup failures.
@@ -209,6 +200,27 @@ let timeoutId: NodeJS.Timeout | null = null;
 export type QueueStatus = 'pending' | 'in_progress' | 'completed' | 'failed' | 'canceled';
 export type TaskResolution = 'marker_detected' | 'auto_completed_timeout' | 'failed_max_retries' | 'runtime_unavailable' | 'canceled' | 'late_marker_principle_created' | 'late_marker_no_principle' | 'stub_fallback' | 'skipped_thin_violation';
 
+/**
+ * Recent pain context attached to sleep_reflection tasks.
+ * Carries explicit recent pain signal metadata without being a separate task kind.
+ * Used by NocturnalTargetSelector for ranking bias and context enrichment.
+ */
+export interface RecentPainContext {
+  /** Most recent unresolved pain event */
+  mostRecent: {
+    score: number;
+    source: string;
+    reason: string;
+    timestamp: string;
+    /** Session ID where the pain occurred */
+    sessionId: string;
+  } | null;
+  /** Count of pain events in the recent window (for signal strength) */
+  recentPainCount: number;
+  /** Highest pain score in the recent window */
+  recentMaxPainScore: number;
+}
+
 export interface EvolutionQueueItem {
     // Core identity
     id: string;
@@ -414,6 +426,7 @@ function buildFallbackNocturnalSnapshot(
     };
 }
 
+const PAIN_QUEUE_DEDUP_WINDOW_MS = 30 * 60 * 1000;
 
 // P0 fix: File lock constants and helper for queue operations (prevents TOCTOU race)
 export const EVOLUTION_QUEUE_LOCK_SUFFIX = '.lock';
@@ -711,7 +724,7 @@ function enqueueNewSleepReflectionTask(
         recentPainContext,
     });
 
-    atomicWriteFileSync(queuePath, JSON.stringify(queue, null, 2));
+    fs.writeFileSync(queuePath, JSON.stringify(queue, null, 2), 'utf8');
     logger?.info?.(`[PD:EvolutionWorker] Enqueued sleep_reflection task ${taskId}`);
 }
 
@@ -856,7 +869,7 @@ async function doEnqueuePainTask(
             retryCount: 0, maxRetries: 3,
         });
 
-        atomicWriteFileSync(queuePath, JSON.stringify(queue, null, 2));
+        fs.writeFileSync(queuePath, JSON.stringify(queue, null, 2), 'utf8');
         fs.appendFileSync(painFlagPath, `\nstatus: queued\ntask_id: ${taskId}\n`, 'utf8');
         result.enqueued = true;
 
@@ -1645,7 +1658,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
 
             // Write claimed state (includes any pain changes from above) and release lock
             if (queueChanged) {
-                atomicWriteFileSync(queuePath, JSON.stringify(queue, null, 2));
+                fs.writeFileSync(queuePath, JSON.stringify(queue, null, 2), 'utf8');
             }
             releaseLock();
             for (const sleepTask of sleepReflectionTasks) {
@@ -1899,7 +1912,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
                             freshQueue[idx] = sleepTask;
                         }
                     }
-                    atomicWriteFileSync(queuePath, JSON.stringify(freshQueue, null, 2));
+                    fs.writeFileSync(queuePath, JSON.stringify(freshQueue, null, 2), 'utf8');
 
                     // Log completions to EvolutionLogger
                     for (const sleepTask of sleepReflectionTasks) {
@@ -1992,14 +2005,10 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
                     };
 
                     // Dispatch LLM subagent via CorrectionObserverWorkflowManager
-                    const subagent = api?.runtime?.subagent;
-                    if (!subagent) {
-                        throw new Error('[PD:EvolutionWorker] subagent runtime not available for keyword_optimization');
-                    }
                     const manager = new CorrectionObserverWorkflowManager({
                         workspaceDir: wctx.workspaceDir,
                         logger,
-                        subagent,
+                        subagent: api?.runtime?.subagent!,
                         agentSession: api?.runtime?.agent?.session,
                     });
 
@@ -2013,11 +2022,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
                         workflowId = handle.workflowId;
                         koTask.resultRef = workflowId;
                     } else {
-                        // isPolling implies resultRef exists (checked above)
-                        workflowId = koTask.resultRef;
-                        if (!workflowId) {
-                            throw new Error(`[PD:EvolutionWorker] keyword_optimization task ${koTask.id} has no resultRef in polling mode`);
-                        }
+                        workflowId = koTask.resultRef!;
                     }
 
                     // Poll workflow state
@@ -2029,7 +2034,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
 
                             if (parsedResult?.updated) {
                                 koService.applyResult(parsedResult);
-                                await learner.recordOptimizationPerformed();
+                                learner.recordOptimizationPerformed();
                                 logger?.info?.(`[PD:EvolutionWorker] keyword_optimization applied mutations: ${parsedResult.summary}`);
                             } else {
                                 logger?.info?.(`[PD:EvolutionWorker] keyword_optimization completed with no updates`);
@@ -2082,7 +2087,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
                         freshQueue.push(koTask);
                     }
                 }
-                fs.writeFileSync(queuePath, JSON.stringify(freshQueue, null, 2));
+                fs.writeFileSync(queuePath, JSON.stringify(freshQueue, null, 2), 'utf8');
             } catch (koResultErr) {
                 logger?.warn?.(`[PD:EvolutionWorker] Failed to write keyword_optimization results: ${String(koResultErr)}`);
             } finally {
@@ -2092,7 +2097,7 @@ async function processEvolutionQueue(wctx: WorkspaceContext, logger: PluginLogge
         }
 
         if (queueChanged) {
-            atomicWriteFileSync(queuePath, JSON.stringify(queue, null, 2));
+            fs.writeFileSync(queuePath, JSON.stringify(queue, null, 2), 'utf8');
         }
 
         // Pipeline observability: log stage-level summary at end of cycle
@@ -2210,7 +2215,7 @@ export async function registerEvolutionTaskSession(
         if (!task.started_at) {
             task.started_at = new Date().toISOString();
         }
-        atomicWriteFileSync(queuePath, JSON.stringify(queue, null, 2));
+        fs.writeFileSync(queuePath, JSON.stringify(queue, null, 2), 'utf8');
         return true;
     } finally {
         releaseLock();
@@ -2250,7 +2255,7 @@ interface WorkerStatusReport {
 function writeWorkerStatus(stateDir: string, report: WorkerStatusReport): void {
     try {
         const statusPath = path.join(stateDir, 'worker-status.json');
-        atomicWriteFileSync(statusPath, JSON.stringify(report, null, 2));
+        fs.writeFileSync(statusPath, JSON.stringify(report, null, 2), 'utf8');
     } catch (statusErr) {
         // Non-critical: worker-status.json is for monitoring, failure is acceptable
         // (no logger available in this standalone helper)
@@ -2281,7 +2286,7 @@ async function processEvolutionQueueWithResult(
         const purgeResult = purgeStaleFailedTasks(queue, logger);
         if (purgeResult.purged > 0) {
             // Write back the cleaned queue
-            atomicWriteFileSync(queuePath, JSON.stringify(queue, null, 2));
+            fs.writeFileSync(queuePath, JSON.stringify(queue, null, 2), 'utf8');
         }
 
         queueResult.total = queue.length;
@@ -2558,6 +2563,7 @@ export const EvolutionWorkerService: ExtendedEvolutionWorkerService = {
             }
 
             timeoutId = setTimeout(runCycle, interval);
+            timeoutId.unref();
         }
 
         timeoutId = setTimeout(() => {
@@ -2573,11 +2579,14 @@ export const EvolutionWorkerService: ExtendedEvolutionWorkerService = {
                 }
                 // processPromotion removed (D-06)
                 timeoutId = setTimeout(runCycle, interval);
+                timeoutId.unref();
             })().catch((err) => {
                 if (logger) logger.error(`[PD:EvolutionWorker] Startup worker cycle failed: ${String(err)}`);
                 timeoutId = setTimeout(runCycle, interval);
+                timeoutId.unref();
             });
         }, initialDelay);
+        timeoutId.unref();
     },
 
     stop(ctx: OpenClawPluginServiceContext): void {

package/src/service/keyword-optimization-service.ts

@@ -7,7 +7,7 @@
  */
 
 import { CorrectionCueLearner } from '../core/correction-cue-learner.js';
-import type { CorrectionObserverResult } from './correction-observer-types.js';
+import type { CorrectionObserverResult } from './subagent-workflow/correction-observer-types.js';
 import type { PluginLogger } from '../openclaw-sdk.js';
 import { TrajectoryRegistry } from '../core/trajectory.js';
 
@@ -137,4 +137,4 @@ export type TrajectoryHistoryEntry = {
 };
 
 /** Re-export CorrectionObserverPayload for convenience */
-export type { CorrectionObserverPayload } from './correction-observer-types.js';
+export type { CorrectionObserverPayload } from './subagent-workflow/correction-observer-types.js';

package/src/service/subagent-workflow/correction-observer-types.ts

@@ -0,0 +1,69 @@
+/**
+ * Correction Observer Workflow - Type Definitions
+ *
+ * Types for the correction observer LLM optimization workflow.
+ * This workflow dispatches an LLM subagent to analyze keyword performance
+ * and recommend ADD/UPDATE/REMOVE actions for the correction keyword store.
+ */
+
+import type { SubagentWorkflowSpec } from './types.js';
+
+/**
+ * Input passed to the correction observer subagent.
+ */
+export interface CorrectionObserverPayload {
+  /** Parent session that triggered the optimization */
+  parentSessionId: string;
+  /** Workspace directory */
+  workspaceDir: string;
+  /** Current keyword store summary for context */
+  keywordStoreSummary: {
+    totalKeywords: number;
+    terms: Array<{
+      term: string;
+      weight: number;
+      hitCount: number;
+      truePositiveCount: number;
+      falsePositiveCount: number;
+    }>;
+  };
+  /** Recent user messages for pattern analysis */
+  recentMessages: string[];
+
+  /**
+   * Trajectory history: user turns where correctionDetected=true (D-40-08).
+   * Includes term matched, timestamp, sessionId for FPR trend analysis.
+   */
+  trajectoryHistory: Array<{
+    sessionId: string;
+    timestamp: string;
+    term: string;
+    userMessage: string;
+  }>;
+}
+
+/**
+ * Result from the correction observer subagent.
+ */
+export interface CorrectionObserverResult {
+  /** Whether any changes were made */
+  updated: boolean;
+  /** The optimization decisions returned by the LLM */
+  updates: Record<string, {
+    action: 'add' | 'update' | 'remove';
+    weight?: number;
+    falsePositiveRate?: number;
+    reasoning: string;
+  }>;
+  /** Human-readable summary */
+  summary: string;
+}
+
+/**
+ * Workflow spec for the correction observer optimization workflow.
+ */
+export interface CorrectionObserverWorkflowSpec extends SubagentWorkflowSpec<CorrectionObserverResult> {
+  workflowType: 'correction_observer';
+  payload: CorrectionObserverPayload;
+  result?: CorrectionObserverResult;
+}