principles-disciple 1.41.0 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.planning/codebase/ARCHITECTURE.md +157 -0
- package/.planning/codebase/CONCERNS.md +145 -0
- package/.planning/codebase/CONVENTIONS.md +148 -0
- package/.planning/codebase/INTEGRATIONS.md +81 -0
- package/.planning/codebase/STACK.md +87 -0
- package/.planning/codebase/STRUCTURE.md +193 -0
- package/.planning/codebase/TESTING.md +243 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/src/commands/archive-impl.ts +5 -3
- package/src/commands/context.ts +1 -0
- package/src/commands/disable-impl.ts +1 -1
- package/src/commands/evolution-status.ts +2 -2
- package/src/commands/pain.ts +12 -5
- package/src/commands/principle-rollback.ts +1 -1
- package/src/commands/promote-impl.ts +13 -7
- package/src/commands/rollback.ts +10 -4
- package/src/commands/samples.ts +1 -1
- package/src/commands/thinking-os.ts +1 -0
- package/src/commands/workflow-debug.ts +1 -1
- package/src/core/config.ts +1 -0
- package/src/core/dictionary.ts +1 -0
- package/src/core/event-log.ts +8 -6
- package/src/core/evolution-types.ts +33 -1
- package/src/core/external-training-contract.ts +1 -1
- package/src/core/merge-gate-audit.ts +3 -3
- package/src/core/nocturnal-arbiter.ts +1 -1
- package/src/core/nocturnal-compliance.ts +21 -21
- package/src/core/nocturnal-executability.ts +1 -1
- package/src/core/nocturnal-reasoning-deriver.ts +4 -4
- package/src/core/nocturnal-rule-implementation-validator.ts +1 -1
- package/src/core/nocturnal-snapshot-contract.ts +1 -1
- package/src/core/pain-context-extractor.ts +2 -2
- package/src/core/path-resolver.ts +1 -0
- package/src/core/pd-task-reconciler.ts +1 -0
- package/src/core/pd-task-service.ts +1 -1
- package/src/core/pd-task-store.ts +1 -0
- package/src/core/principle-internalization/deprecated-readiness.ts +1 -1
- package/src/core/principle-internalization/principle-lifecycle-service.ts +1 -1
- package/src/core/principle-training-state.ts +2 -2
- package/src/core/principle-tree-migration.ts +1 -1
- package/src/core/replay-engine.ts +1 -0
- package/src/core/risk-calculator.ts +2 -1
- package/src/core/rule-host.ts +1 -1
- package/src/core/session-tracker.ts +1 -0
- package/src/core/shadow-observation-registry.ts +1 -1
- package/src/core/thinking-models.ts +1 -1
- package/src/core/thinking-os-parser.ts +1 -1
- package/src/core/trajectory.ts +2 -0
- package/src/hooks/bash-risk.ts +2 -2
- package/src/hooks/edit-verification.ts +3 -3
- package/src/hooks/gate.ts +8 -8
- package/src/hooks/gfi-gate.ts +2 -2
- package/src/hooks/lifecycle-routing.ts +1 -1
- package/src/hooks/message-sanitize.ts +18 -5
- package/src/hooks/pain.ts +2 -2
- package/src/hooks/progressive-trust-gate.ts +3 -3
- package/src/hooks/prompt.ts +17 -4
- package/src/hooks/subagent.ts +2 -3
- package/src/hooks/thinking-checkpoint.ts +1 -1
- package/src/http/principles-console-route.ts +21 -4
- package/src/service/central-database.ts +3 -2
- package/src/service/central-health-service.ts +2 -1
- package/src/service/central-overview-service.ts +3 -2
- package/src/service/control-ui-query-service.ts +2 -2
- package/src/service/event-log-auditor.ts +2 -2
- package/src/service/evolution-query-service.ts +1 -1
- package/src/service/evolution-worker.ts +96 -370
- package/src/service/health-query-service.ts +11 -10
- package/src/service/monitoring-query-service.ts +4 -4
- package/src/service/nocturnal-target-selector.ts +2 -2
- package/src/service/queue-io.ts +375 -0
- package/src/service/queue-migration.ts +122 -0
- package/src/service/runtime-summary-service.ts +1 -1
- package/src/service/sleep-cycle.ts +157 -0
- package/src/service/subagent-workflow/empathy-observer-workflow-manager.ts +1 -0
- package/src/service/subagent-workflow/runtime-direct-driver.ts +1 -1
- package/src/service/subagent-workflow/subagent-error-utils.ts +1 -1
- package/src/service/subagent-workflow/workflow-store.ts +3 -2
- package/src/service/workflow-watchdog.ts +168 -0
- package/src/tools/critique-prompt.ts +1 -1
- package/src/tools/deep-reflect.ts +22 -11
- package/src/tools/model-index.ts +1 -1
- package/src/types/event-payload.ts +80 -0
- package/src/types/queue.ts +70 -0
- package/src/utils/file-lock.ts +2 -2
- package/src/utils/io.ts +11 -3
- package/tests/core/evolution-migration.test.ts +325 -1
- package/tests/core/queue-purge.test.ts +337 -0
- package/tests/fixtures/legacy-queue-v1.json +74 -0
- package/tests/queue/async-lock.test.ts +200 -0
- package/tests/service/evolution-worker.queue.test.ts +296 -0
- package/tests/service/queue-io.test.ts +229 -0
- package/tests/service/queue-migration.test.ts +147 -0
- package/tests/service/workflow-watchdog.test.ts +372 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sleep Cycle Orchestrator — extracted from evolution-worker.ts
|
|
3
|
+
*
|
|
4
|
+
* Responsibilities:
|
|
5
|
+
* - Idle workspace detection via nocturnal-runtime.js
|
|
6
|
+
* - Cooldown enforcement via nocturnal-runtime.js
|
|
7
|
+
* - Sleep reflection task enqueue orchestration (fire-and-forget)
|
|
8
|
+
* - Keyword optimization task enqueue orchestration (fire-and-forget)
|
|
9
|
+
* - Cycle heartbeat tracking and periodic trigger reset
|
|
10
|
+
*
|
|
11
|
+
* Does NOT include (remain in evolution-worker.ts facade):
|
|
12
|
+
* - checkPainFlag, processEvolutionQueueWithResult, processDetectionQueue
|
|
13
|
+
* - Workflow managers (EmpathyObserver, DeepReflect, Nocturnal)
|
|
14
|
+
* - Workflow watchdog (runWorkflowWatchdog)
|
|
15
|
+
* - Pain-flag-triggered immediate heartbeat
|
|
16
|
+
*
|
|
17
|
+
* Dependencies: nocturnal-runtime.js, nocturnal-config.js, queue-io.js
|
|
18
|
+
* Zero imports from evolution-worker.ts.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { WorkspaceContext } from '../core/workspace-context.js';
|
|
22
|
+
import type { OpenClawPluginApi, PluginLogger } from '../openclaw-sdk.js';
|
|
23
|
+
import type { EventLog } from '../core/event-log.js';
|
|
24
|
+
import { checkWorkspaceIdle, checkCooldown } from './nocturnal-runtime.js';
|
|
25
|
+
import { loadNocturnalConfigMerged } from './nocturnal-config.js';
|
|
26
|
+
import { enqueueSleepReflectionTask, enqueueKeywordOptimizationTask } from './queue-io.js';
|
|
27
|
+
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
// Types
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
export interface WorkerStatusReport {
|
|
33
|
+
timestamp: string;
|
|
34
|
+
cycle_start_ms: number;
|
|
35
|
+
duration_ms: number;
|
|
36
|
+
pain_flag: { exists: boolean; score: number | null; source: string | null; enqueued: boolean; skipped_reason: string | null };
|
|
37
|
+
queue: { total: number; pending: number; in_progress: number; completed_this_cycle: number; failed_this_cycle: number };
|
|
38
|
+
errors: string[];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface CycleOptions {
|
|
42
|
+
wctx: WorkspaceContext;
|
|
43
|
+
logger: PluginLogger | undefined;
|
|
44
|
+
eventLog: EventLog;
|
|
45
|
+
api: OpenClawPluginApi | undefined;
|
|
46
|
+
/** Mutable ref to the heartbeat counter — incremented by runCycle on each call */
|
|
47
|
+
heartbeatCounterRef: { value: number };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
// Cycle Orchestrator
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Execute one sleep-cycle heartbeat.
|
|
56
|
+
*
|
|
57
|
+
* Orchestrates:
|
|
58
|
+
* 1. Load merged nocturnal config
|
|
59
|
+
* 2. Check workspace idle state
|
|
60
|
+
* 3. Enqueue keyword_optimization task (independent periodic trigger)
|
|
61
|
+
* 4. Enqueue sleep_reflection task (idle-based OR periodic trigger + cooldown gate)
|
|
62
|
+
* 5. Cycle result reporting
|
|
63
|
+
*
|
|
64
|
+
* Does NOT directly call checkPainFlag, processEvolutionQueueWithResult, or
|
|
65
|
+
* processDetectionQueue — those remain in the evolution-worker.ts facade.
|
|
66
|
+
*
|
|
67
|
+
* @param options.wctx — workspace context
|
|
68
|
+
* @param options.logger — plugin logger
|
|
69
|
+
* @param options.eventLog — event log
|
|
70
|
+
* @param options.api — OpenClaw plugin API (optional)
|
|
71
|
+
* @param options.heartbeatCounterRef — mutable counter, incremented by runCycle
|
|
72
|
+
*/
|
|
73
|
+
export async function runCycle(options: CycleOptions): Promise<WorkerStatusReport> {
|
|
74
|
+
const { wctx, logger, eventLog: _eventLog, api: _api, heartbeatCounterRef } = options;
|
|
75
|
+
const cycleStart = Date.now();
|
|
76
|
+
heartbeatCounterRef.value++;
|
|
77
|
+
|
|
78
|
+
// ──── DEBUG: Verify subagent availability in heartbeat context ────
|
|
79
|
+
const hbSubagent = _api?.runtime?.subagent;
|
|
80
|
+
logger?.info?.(`[PD:DEBUG:SubagentCheck:Heartbeat] api_exists=${!!_api}, subagent_exists=${!!hbSubagent}, subagent.run_exists=${!!hbSubagent?.run}, heartbeatCounter=${heartbeatCounterRef.value}`);
|
|
81
|
+
if (hbSubagent?.run) {
|
|
82
|
+
logger?.info?.('[PD:DEBUG:SubagentCheck:Heartbeat] run entrypoint is callable');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const cycleResult: WorkerStatusReport = {
|
|
86
|
+
timestamp: new Date().toISOString(),
|
|
87
|
+
cycle_start_ms: cycleStart,
|
|
88
|
+
duration_ms: 0,
|
|
89
|
+
pain_flag: { exists: false, score: null, source: null, enqueued: false, skipped_reason: null },
|
|
90
|
+
queue: { total: 0, pending: 0, in_progress: 0, completed_this_cycle: 0, failed_this_cycle: 0 },
|
|
91
|
+
errors: [],
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
try {
|
|
95
|
+
// Load config on each cycle (supports runtime updates) — single file read
|
|
96
|
+
const mergedConfig = loadNocturnalConfigMerged(wctx.stateDir);
|
|
97
|
+
const { sleepReflection: sleepConfig, keywordOptimization: kwOptConfig } = mergedConfig;
|
|
98
|
+
|
|
99
|
+
const idleResult = checkWorkspaceIdle(wctx.workspaceDir, {});
|
|
100
|
+
logger?.info?.(`[PD:EvolutionWorker] HEARTBEAT cycle=${new Date().toISOString()} idle=${idleResult.isIdle} idleForMs=${idleResult.idleForMs} userActiveSessions=${idleResult.userActiveSessions} abandonedSessions=${idleResult.abandonedSessionIds.length} lastActivityEpoch=${idleResult.mostRecentActivityAt} triggerMode=${sleepConfig.trigger_mode}`);
|
|
101
|
+
|
|
102
|
+
let shouldTrySleepReflection = false;
|
|
103
|
+
|
|
104
|
+
// Path 1: Idle-based trigger (default mode)
|
|
105
|
+
if (idleResult.isIdle && sleepConfig.trigger_mode === 'idle') {
|
|
106
|
+
logger?.info?.(`[PD:EvolutionWorker] Workspace idle (${idleResult.idleForMs}ms since last activity)`);
|
|
107
|
+
shouldTrySleepReflection = true;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// keyword_optimization: Independent periodic trigger (CORR-07).
|
|
111
|
+
// Fires every kwOptConfig.period_heartbeats regardless of trigger_mode.
|
|
112
|
+
// Has its own dedicated config (default 24 heartbeats = 6 hours).
|
|
113
|
+
if (kwOptConfig.enabled && heartbeatCounterRef.value > 0 && heartbeatCounterRef.value % kwOptConfig.period_heartbeats === 0) {
|
|
114
|
+
logger?.info?.(`[PD:EvolutionWorker] keyword_optimization trigger at heartbeat ${heartbeatCounterRef.value} (trigger_mode=${sleepConfig.trigger_mode})`);
|
|
115
|
+
enqueueKeywordOptimizationTask(wctx, logger).catch((err) => {
|
|
116
|
+
logger?.error?.(`[PD:EvolutionWorker] Failed to enqueue keyword_optimization task: ${String(err)}`);
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Path 2: Periodic trigger for sleep_reflection (fires regardless of idle state)
|
|
121
|
+
if (sleepConfig.trigger_mode === 'periodic') {
|
|
122
|
+
if (heartbeatCounterRef.value >= sleepConfig.period_heartbeats) {
|
|
123
|
+
logger?.info?.(`[PD:EvolutionWorker] Periodic trigger: heartbeatCounter=${heartbeatCounterRef.value} >= period_heartbeats=${sleepConfig.period_heartbeats}`);
|
|
124
|
+
shouldTrySleepReflection = true;
|
|
125
|
+
heartbeatCounterRef.value = 0; // Reset counter
|
|
126
|
+
} else {
|
|
127
|
+
logger?.info?.(`[PD:EvolutionWorker] Periodic: ${heartbeatCounterRef.value}/${sleepConfig.period_heartbeats} heartbeats — waiting`);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (shouldTrySleepReflection) {
|
|
132
|
+
const cooldown = checkCooldown(wctx.stateDir, undefined, {
|
|
133
|
+
globalCooldownMs: sleepConfig.cooldown_ms,
|
|
134
|
+
maxRunsPerWindow: sleepConfig.max_runs_per_day,
|
|
135
|
+
quotaWindowMs: 24 * 60 * 60 * 1000,
|
|
136
|
+
});
|
|
137
|
+
logger?.info?.(`[PD:EvolutionWorker] Cooldown check: globalCooldownActive=${cooldown.globalCooldownActive} quotaExhausted=${cooldown.quotaExhausted} runsRemaining=${cooldown.runsRemaining}`);
|
|
138
|
+
if (!cooldown.globalCooldownActive && !cooldown.quotaExhausted) {
|
|
139
|
+
logger?.info?.('[PD:EvolutionWorker] Attempting to enqueue sleep_reflection task...');
|
|
140
|
+
enqueueSleepReflectionTask(wctx, logger).catch((err) => {
|
|
141
|
+
logger?.error?.(`[PD:EvolutionWorker] Failed to enqueue sleep_reflection task: ${String(err)}`);
|
|
142
|
+
});
|
|
143
|
+
} else {
|
|
144
|
+
logger?.info?.(`[PD:EvolutionWorker] Skipping sleep_reflection: globalCooldown=${cooldown.globalCooldownActive} quotaExhausted=${cooldown.quotaExhausted}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
cycleResult.duration_ms = Date.now() - cycleStart;
|
|
149
|
+
} catch (err) {
|
|
150
|
+
const errMsg = `Error in runCycle: ${String(err)}`;
|
|
151
|
+
if (logger) logger.error(`[PD:EvolutionWorker] ${errMsg}`);
|
|
152
|
+
cycleResult.errors.push(errMsg);
|
|
153
|
+
cycleResult.duration_ms = Date.now() - cycleStart;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return cycleResult;
|
|
157
|
+
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*
|
|
6
6
|
* Callers should suppress warnings for these errors — they are not real failures.
|
|
7
7
|
*/
|
|
8
|
-
|
|
8
|
+
|
|
9
9
|
export function isExpectedSubagentError(err: unknown): boolean {
|
|
10
10
|
const msg = String(err);
|
|
11
11
|
return (
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
/* eslint-disable no-console */
|
|
1
2
|
import Database from 'better-sqlite3';
|
|
2
3
|
import * as fs from 'fs';
|
|
3
4
|
import * as path from 'path';
|
|
@@ -233,7 +234,7 @@ export class WorkflowStore {
|
|
|
233
234
|
}
|
|
234
235
|
|
|
235
236
|
|
|
236
|
-
|
|
237
|
+
|
|
237
238
|
recordEvent(
|
|
238
239
|
workflowId: string,
|
|
239
240
|
eventType: string,
|
|
@@ -270,7 +271,7 @@ export class WorkflowStore {
|
|
|
270
271
|
* same idempotency_key already exists, this is a no-op (idempotent).
|
|
271
272
|
*/
|
|
272
273
|
|
|
273
|
-
|
|
274
|
+
|
|
274
275
|
recordStageOutput(
|
|
275
276
|
workflowId: string,
|
|
276
277
|
stage: 'dreamer' | 'philosopher',
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Workflow Watchdog - Extracted from evolution-worker.ts (lines 79-223)
|
|
3
|
+
*
|
|
4
|
+
* Detects stale/orphaned workflows, invalid results, and cleanup failures.
|
|
5
|
+
* Runs every heartbeat cycle, catching bugs like:
|
|
6
|
+
* #185 — orphaned active workflows
|
|
7
|
+
* #181 — structurally invalid results (all zeros)
|
|
8
|
+
* #180/#183 — expired workflows not swept
|
|
9
|
+
* #182 — unhandled rejections leaving workflows in limbo
|
|
10
|
+
*
|
|
11
|
+
* BUG-01: isExpectedSubagentError guard prevents marking daemon-mode stale
|
|
12
|
+
* workflows as terminal_error (line 122)
|
|
13
|
+
* BUG-02: Gateway fallback cleans up child sessions via agentSession when
|
|
14
|
+
* subagentRuntime unavailable (lines 148-156)
|
|
15
|
+
* BUG-03: Nocturnal workflow snapshot validation detects pain_context_fallback
|
|
16
|
+
* with zero stats (lines 184-198)
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { WorkspaceContext } from '../core/workspace-context.js';
|
|
20
|
+
import type { OpenClawPluginApi, PluginLogger } from '../openclaw-sdk.js';
|
|
21
|
+
import type { WorkflowRow } from './subagent-workflow/types.js';
|
|
22
|
+
import { WorkflowStore } from './subagent-workflow/workflow-store.js';
|
|
23
|
+
import { isExpectedSubagentError } from './subagent-workflow/subagent-error-utils.js';
|
|
24
|
+
import { WORKFLOW_TTL_MS } from '../config/defaults/runtime.js';
|
|
25
|
+
|
|
26
|
+
export interface WatchdogResult {
|
|
27
|
+
anomalies: number;
|
|
28
|
+
details: string[];
|
|
29
|
+
/** Set when the watchdog scan itself failed (e.g., store errors). Undefined means scan succeeded. */
|
|
30
|
+
scanError?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
export async function runWorkflowWatchdog(
|
|
35
|
+
wctx: WorkspaceContext,
|
|
36
|
+
api: OpenClawPluginApi | null,
|
|
37
|
+
logger?: PluginLogger,
|
|
38
|
+
): Promise<WatchdogResult> {
|
|
39
|
+
const details: string[] = [];
|
|
40
|
+
const now = Date.now();
|
|
41
|
+
const subagentRuntime = api?.runtime?.subagent;
|
|
42
|
+
const agentSession = api?.runtime?.agent?.session;
|
|
43
|
+
|
|
44
|
+
try {
|
|
45
|
+
const store = new WorkflowStore({ workspaceDir: wctx.workspaceDir });
|
|
46
|
+
try {
|
|
47
|
+
const allWorkflows: WorkflowRow[] = store.listWorkflows();
|
|
48
|
+
|
|
49
|
+
// Check 1: Stale active workflows (active > 2x TTL)
|
|
50
|
+
const staleThreshold = WORKFLOW_TTL_MS * 2;
|
|
51
|
+
const staleActive = allWorkflows.filter(
|
|
52
|
+
(wf: WorkflowRow) => wf.state === 'active' && (now - wf.created_at) > staleThreshold,
|
|
53
|
+
);
|
|
54
|
+
if (staleActive.length > 0) {
|
|
55
|
+
for (const wf of staleActive) {
|
|
56
|
+
const ageMin = Math.round((now - wf.created_at) / 60000);
|
|
57
|
+
details.push(`stale_active: ${wf.workflow_id} (${wf.workflow_type}, ${ageMin}min old)`);
|
|
58
|
+
|
|
59
|
+
// #257: Check if the last recorded event reason indicates expected subagent unavailability.
|
|
60
|
+
// If so, skip marking as terminal_error — the workflow is stale because the subagent
|
|
61
|
+
// was expectedly unavailable (daemon mode, process isolation), not due to a hard failure.
|
|
62
|
+
const events = store.getEvents(wf.workflow_id);
|
|
63
|
+
const lastEventReason = events.length > 0 ? events[events.length - 1].reason : 'unknown';
|
|
64
|
+
if (isExpectedSubagentError(lastEventReason)) {
|
|
65
|
+
logger?.debug?.(`[PD:Watchdog] Skipping stale active workflow ${wf.workflow_id}: expected subagent error (${lastEventReason})`);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
store.updateWorkflowState(wf.workflow_id, 'terminal_error');
|
|
70
|
+
store.recordEvent(wf.workflow_id, 'watchdog_timeout', 'active', 'terminal_error', `Stale active > ${staleThreshold / 60000}s`, { ageMs: now - wf.created_at });
|
|
71
|
+
|
|
72
|
+
// Cleanup session if possible (#188: gateway-safe fallback)
|
|
73
|
+
if (wf.child_session_key) {
|
|
74
|
+
try {
|
|
75
|
+
if (subagentRuntime) {
|
|
76
|
+
await subagentRuntime.deleteSession({ sessionKey: wf.child_session_key, deleteTranscript: true });
|
|
77
|
+
logger?.info?.(`[PD:Watchdog] Cleaned up stale session: ${wf.child_session_key}`);
|
|
78
|
+
} else if (agentSession) {
|
|
79
|
+
const storePath = agentSession.resolveStorePath();
|
|
80
|
+
const sessionStore = agentSession.loadSessionStore(storePath, { skipCache: true });
|
|
81
|
+
const normalizedKey = wf.child_session_key.toLowerCase();
|
|
82
|
+
if (sessionStore[normalizedKey]) {
|
|
83
|
+
delete sessionStore[normalizedKey];
|
|
84
|
+
await agentSession.saveSessionStore(storePath, sessionStore);
|
|
85
|
+
logger?.info?.(`[PD:Watchdog] Cleaned up stale session via agentSession fallback: ${wf.child_session_key}`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
} catch (cleanupErr) {
|
|
89
|
+
const errMsg = String(cleanupErr);
|
|
90
|
+
if (errMsg.includes('gateway request') && agentSession) {
|
|
91
|
+
const storePath = agentSession.resolveStorePath();
|
|
92
|
+
const sessionStore = agentSession.loadSessionStore(storePath, { skipCache: true });
|
|
93
|
+
const normalizedKey = wf.child_session_key.toLowerCase();
|
|
94
|
+
if (sessionStore[normalizedKey]) {
|
|
95
|
+
delete sessionStore[normalizedKey];
|
|
96
|
+
await agentSession.saveSessionStore(storePath, sessionStore);
|
|
97
|
+
logger?.info?.(`[PD:Watchdog] Cleaned up stale session via agentSession fallback after gateway error: ${wf.child_session_key}`);
|
|
98
|
+
}
|
|
99
|
+
} else {
|
|
100
|
+
logger?.warn?.(`[PD:Watchdog] Failed to cleanup session ${wf.child_session_key}: ${errMsg}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// Check 2: Workflows in terminal_error/expired without cleanup
|
|
108
|
+
const unclearedTerminal = allWorkflows.filter(
|
|
109
|
+
(wf: WorkflowRow) => (wf.state === 'terminal_error' || wf.state === 'expired') && wf.cleanup_state === 'pending',
|
|
110
|
+
);
|
|
111
|
+
if (unclearedTerminal.length > 0) {
|
|
112
|
+
details.push(`uncleared_terminal: ${unclearedTerminal.length} workflows (will be swept next cycle)`);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Check 3: Nocturnal workflow result validation (#181 pattern)
|
|
116
|
+
const nocturnalCompleted = allWorkflows.filter(
|
|
117
|
+
(wf: WorkflowRow) => wf.workflow_type === 'nocturnal' && wf.state === 'completed',
|
|
118
|
+
);
|
|
119
|
+
for (const wf of nocturnalCompleted) {
|
|
120
|
+
// Check if the metadata snapshot has all zeros (invalid data)
|
|
121
|
+
try {
|
|
122
|
+
const meta = JSON.parse(wf.metadata_json) as Record<string, unknown>;
|
|
123
|
+
const snapshot = meta.snapshot as Record<string, unknown> | undefined;
|
|
124
|
+
if (snapshot) {
|
|
125
|
+
// #219: Check for fallback data source (partial stats from pain context)
|
|
126
|
+
const dataSource = snapshot._dataSource as string | undefined;
|
|
127
|
+
if (dataSource === 'pain_context_fallback') {
|
|
128
|
+
details.push(`fallback_snapshot: nocturnal workflow ${wf.workflow_id} uses pain-context fallback (stats may be incomplete)`);
|
|
129
|
+
}
|
|
130
|
+
const stats = snapshot.stats as Record<string, number> | undefined;
|
|
131
|
+
// #246: Stats are now always number (never null). Detect "empty" fallback:
|
|
132
|
+
// fallback + all counts zero means no real data was available.
|
|
133
|
+
// NOTE: totalAssistantTurns may be 0 even for valid sessions because
|
|
134
|
+
// listRecentNocturnalCandidateSessions (used in fallback path) does not
|
|
135
|
+
// populate assistantTurnCount (only getNocturnalSessionSnapshot does).
|
|
136
|
+
// We use totalToolCalls=0 as the primary indicator instead.
|
|
137
|
+
if (stats && dataSource === 'pain_context_fallback' &&
|
|
138
|
+
stats.totalToolCalls === 0 && stats.totalGateBlocks === 0 &&
|
|
139
|
+
stats.failureCount === 0) {
|
|
140
|
+
details.push(`fallback_snapshot_stats: nocturnal workflow ${wf.workflow_id} has empty fallback stats (no trajectory data found)`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
} catch (err) {
|
|
144
|
+
details.push(`malformed_metadata: workflow ${wf.workflow_id} has unparseable metadata: ${String(err).slice(0, 100)}`);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Summary
|
|
149
|
+
const stateCounts: Record<string, number> = {};
|
|
150
|
+
for (const wf of allWorkflows) {
|
|
151
|
+
stateCounts[wf.state] = (stateCounts[wf.state] || 0) + 1;
|
|
152
|
+
}
|
|
153
|
+
const stateSummary = Object.entries(stateCounts).map(([s, c]) => `${s}=${c}`).join(', ');
|
|
154
|
+
if (details.length === 0) {
|
|
155
|
+
logger?.debug?.(`[PD:Watchdog] OK — ${allWorkflows.length} workflows (${stateSummary})`);
|
|
156
|
+
} else {
|
|
157
|
+
logger?.info?.(`[PD:Watchdog] ${details.length} anomalies — ${allWorkflows.length} workflows (${stateSummary})`);
|
|
158
|
+
}
|
|
159
|
+
} finally {
|
|
160
|
+
store.dispose();
|
|
161
|
+
}
|
|
162
|
+
} catch (err) {
|
|
163
|
+
logger?.warn?.(`[PD:Watchdog] Failed to scan workflows: ${String(err)}`);
|
|
164
|
+
return { anomalies: -1, details: [], scanError: String(err) };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
return { anomalies: details.length, details };
|
|
168
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { OpenClawPluginApi } from '../openclaw-sdk.js';
|
|
2
|
+
import type { PluginRuntimeSubagent } from '../service/subagent-workflow/runtime-direct-driver.js';
|
|
2
3
|
import { Type } from '@sinclair/typebox';
|
|
3
4
|
import * as fs from 'fs';
|
|
4
5
|
import { EventLogService } from '../core/event-log.js';
|
|
@@ -23,6 +24,16 @@ interface DeepReflectionConfig {
|
|
|
23
24
|
timeout_ms?: number;
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Type assertion: OpenClaw SDK subagent -> workflow manager subagent type.
|
|
29
|
+
* Both types are structurally identical but come from different import paths.
|
|
30
|
+
*/
|
|
31
|
+
function toWorkflowSubagent(
|
|
32
|
+
subagent: NonNullable<OpenClawPluginApi['runtime']>['subagent']
|
|
33
|
+
): PluginRuntimeSubagent {
|
|
34
|
+
return subagent as unknown as PluginRuntimeSubagent;
|
|
35
|
+
}
|
|
36
|
+
|
|
26
37
|
const DEFAULT_CONFIG: DeepReflectionConfig = {
|
|
27
38
|
enabled: true,
|
|
28
39
|
mode: 'auto',
|
|
@@ -108,7 +119,7 @@ export function createDeepReflectTool(api: OpenClawPluginApi) {
|
|
|
108
119
|
}
|
|
109
120
|
|
|
110
121
|
|
|
111
|
-
|
|
122
|
+
|
|
112
123
|
const effectiveWorkspaceDir = resolveReflectionWorkspace(api);
|
|
113
124
|
|
|
114
125
|
const config = loadConfig(effectiveWorkspaceDir, api);
|
|
@@ -122,11 +133,11 @@ export function createDeepReflectTool(api: OpenClawPluginApi) {
|
|
|
122
133
|
|
|
123
134
|
try {
|
|
124
135
|
|
|
125
|
-
|
|
136
|
+
|
|
126
137
|
return await executeReflectionWorkflow(effectiveWorkspaceDir, config, context, depth, model_id, api);
|
|
127
138
|
} catch (err) {
|
|
128
139
|
|
|
129
|
-
|
|
140
|
+
|
|
130
141
|
return handleReflectionError(err, context, depth, model_id, effectiveWorkspaceDir, api);
|
|
131
142
|
}
|
|
132
143
|
}
|
|
@@ -149,7 +160,7 @@ function resolveReflectionWorkspace(api: OpenClawPluginApi): string {
|
|
|
149
160
|
* Execute the deep reflection workflow: start, poll, collect results.
|
|
150
161
|
*/
|
|
151
162
|
|
|
152
|
-
|
|
163
|
+
|
|
153
164
|
async function executeReflectionWorkflow(
|
|
154
165
|
effectiveWorkspaceDir: string,
|
|
155
166
|
config: DeepReflectionConfig,
|
|
@@ -165,8 +176,8 @@ async function executeReflectionWorkflow(
|
|
|
165
176
|
const manager = new DeepReflectWorkflowManager({
|
|
166
177
|
workspaceDir: effectiveWorkspaceDir,
|
|
167
178
|
logger: api.logger,
|
|
168
|
-
|
|
169
|
-
subagent: api.runtime.subagent
|
|
179
|
+
|
|
180
|
+
subagent: toWorkflowSubagent(api.runtime.subagent),
|
|
170
181
|
agentSession: api.runtime.agent?.session,
|
|
171
182
|
});
|
|
172
183
|
|
|
@@ -181,7 +192,7 @@ async function executeReflectionWorkflow(
|
|
|
181
192
|
const startTime = Date.now();
|
|
182
193
|
const timeoutMs = config.timeout_ms ?? 60000;
|
|
183
194
|
|
|
184
|
-
|
|
195
|
+
|
|
185
196
|
return await pollReflectionCompletion(manager, handle, timeoutMs, startTime, eventLog, effectiveWorkspaceDir, context, model_id, depth);
|
|
186
197
|
} finally {
|
|
187
198
|
manager.dispose();
|
|
@@ -192,7 +203,7 @@ async function executeReflectionWorkflow(
|
|
|
192
203
|
* Poll the reflection workflow until completion, timeout, or error.
|
|
193
204
|
*/
|
|
194
205
|
|
|
195
|
-
|
|
206
|
+
|
|
196
207
|
async function pollReflectionCompletion(
|
|
197
208
|
manager: DeepReflectWorkflowManager,
|
|
198
209
|
handle: { workflowId: string; childSessionKey: string },
|
|
@@ -213,7 +224,7 @@ async function pollReflectionCompletion(
|
|
|
213
224
|
|
|
214
225
|
if (workflowState === 'completed') {
|
|
215
226
|
|
|
216
|
-
|
|
227
|
+
|
|
217
228
|
return formatReflectionSuccess(handle, context, depth, model_id, startTime, eventLog, workspaceDir);
|
|
218
229
|
}
|
|
219
230
|
|
|
@@ -229,7 +240,7 @@ async function pollReflectionCompletion(
|
|
|
229
240
|
* Format the success response from a completed reflection.
|
|
230
241
|
*/
|
|
231
242
|
|
|
232
|
-
|
|
243
|
+
|
|
233
244
|
function formatReflectionSuccess(
|
|
234
245
|
handle: { childSessionKey: string },
|
|
235
246
|
context: string,
|
|
@@ -283,7 +294,7 @@ ${insights || '反思完成,详见 REFLECTION_LOG。'}
|
|
|
283
294
|
* Handle reflection errors and format error response.
|
|
284
295
|
*/
|
|
285
296
|
|
|
286
|
-
|
|
297
|
+
|
|
287
298
|
function handleReflectionError(
|
|
288
299
|
err: unknown,
|
|
289
300
|
context: string,
|
package/src/tools/model-index.ts
CHANGED
|
@@ -62,7 +62,7 @@ export function loadModelIndex(
|
|
|
62
62
|
const customConfig = loadCustomConfig(wctx);
|
|
63
63
|
|
|
64
64
|
|
|
65
|
-
|
|
65
|
+
|
|
66
66
|
let modelsDir: string;
|
|
67
67
|
if (customConfig?.modelsDir) {
|
|
68
68
|
modelsDir = path.isAbsolute(customConfig.modelsDir)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Discriminated union for EventLogEntry — replaces flat data: Record<string, unknown>.
|
|
3
|
+
* Each union member is keyed on the `type` field for type narrowing.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import type {
|
|
7
|
+
ToolCallEventData,
|
|
8
|
+
PainSignalEventData,
|
|
9
|
+
RuleMatchEventData,
|
|
10
|
+
RulePromotionEventData,
|
|
11
|
+
HookExecutionEventData,
|
|
12
|
+
GateBlockEventData,
|
|
13
|
+
GateBypassEventData,
|
|
14
|
+
PlanApprovalEventData,
|
|
15
|
+
EvolutionTaskEventData,
|
|
16
|
+
DeepReflectionEventData,
|
|
17
|
+
EmpathyRollbackEventData,
|
|
18
|
+
EventCategory,
|
|
19
|
+
} from './event-types.js';
|
|
20
|
+
|
|
21
|
+
export type EventLogEntry =
|
|
22
|
+
| { ts: string; date: string; type: 'tool_call'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: ToolCallEventData }
|
|
23
|
+
| { ts: string; date: string; type: 'pain_signal'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: PainSignalEventData }
|
|
24
|
+
| { ts: string; date: string; type: 'rule_match'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: RuleMatchEventData }
|
|
25
|
+
| { ts: string; date: string; type: 'rule_promotion'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: RulePromotionEventData }
|
|
26
|
+
| { ts: string; date: string; type: 'hook_execution'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: HookExecutionEventData }
|
|
27
|
+
| { ts: string; date: string; type: 'gate_block'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: GateBlockEventData }
|
|
28
|
+
| { ts: string; date: string; type: 'gate_bypass'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: GateBypassEventData }
|
|
29
|
+
| { ts: string; date: string; type: 'plan_approval'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: PlanApprovalEventData }
|
|
30
|
+
| { ts: string; date: string; type: 'evolution_task'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: EvolutionTaskEventData }
|
|
31
|
+
| { ts: string; date: string; type: 'deep_reflection'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: DeepReflectionEventData }
|
|
32
|
+
| { ts: string; date: string; type: 'empathy_rollback'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: EmpathyRollbackEventData }
|
|
33
|
+
| { ts: string; date: string; type: 'error'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: Record<string, unknown> }
|
|
34
|
+
| { ts: string; date: string; type: 'warn'; category: EventCategory; sessionId?: string; workspaceDir?: string; data: Record<string, unknown> };
|
|
35
|
+
|
|
36
|
+
// Type predicates for safe narrowing
|
|
37
|
+
|
|
38
|
+
export function isToolCallEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'tool_call' }> {
|
|
39
|
+
return entry.type === 'tool_call';
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function isPainSignalEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'pain_signal' }> {
|
|
43
|
+
return entry.type === 'pain_signal';
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function isRuleMatchEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'rule_match' }> {
|
|
47
|
+
return entry.type === 'rule_match';
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function isRulePromotionEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'rule_promotion' }> {
|
|
51
|
+
return entry.type === 'rule_promotion';
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function isHookExecutionEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'hook_execution' }> {
|
|
55
|
+
return entry.type === 'hook_execution';
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function isGateBlockEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'gate_block' }> {
|
|
59
|
+
return entry.type === 'gate_block';
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function isGateBypassEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'gate_bypass' }> {
|
|
63
|
+
return entry.type === 'gate_bypass';
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function isPlanApprovalEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'plan_approval' }> {
|
|
67
|
+
return entry.type === 'plan_approval';
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function isEvolutionTaskEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'evolution_task' }> {
|
|
71
|
+
return entry.type === 'evolution_task';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function isDeepReflectionEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'deep_reflection' }> {
|
|
75
|
+
return entry.type === 'deep_reflection';
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function isEmpathyRollbackEventEntry(entry: EventLogEntry): entry is Extract<EventLogEntry, { type: 'empathy_rollback' }> {
|
|
79
|
+
return entry.type === 'empathy_rollback';
|
|
80
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Branded types for queue and workflow domain identifiers.
|
|
3
|
+
* These prevent accidental interchange of plain strings with domain-specific IDs.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Brand type constructor using intersection type pattern.
|
|
8
|
+
* @example type UserId = Brand<string, 'UserId'>;
|
|
9
|
+
*/
|
|
10
|
+
export type Brand<T, B> = T & { readonly _brand: B };
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Queue item identifier — not interchangeable with plain string.
|
|
14
|
+
*/
|
|
15
|
+
export type QueueItemId = Brand<string, 'QueueItemId'>;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Workflow identifier — not interchangeable with plain string.
|
|
19
|
+
*/
|
|
20
|
+
export type WorkflowId = Brand<string, 'WorkflowId'>;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Session key — not interchangeable with plain string.
|
|
24
|
+
*/
|
|
25
|
+
export type SessionKey = Brand<string, 'SessionKey'>;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Constructor for QueueItemId.
|
|
29
|
+
* @param id - raw string ID from queue operations
|
|
30
|
+
*/
|
|
31
|
+
export function toQueueItemId(id: string): QueueItemId {
|
|
32
|
+
return id as QueueItemId;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Constructor for WorkflowId.
|
|
37
|
+
* @param id - raw string ID from workflow operations
|
|
38
|
+
*/
|
|
39
|
+
export function toWorkflowId(id: string): WorkflowId {
|
|
40
|
+
return id as WorkflowId;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Constructor for SessionKey.
|
|
45
|
+
* @param key - raw string key from session operations
|
|
46
|
+
*/
|
|
47
|
+
export function toSessionKey(key: string): SessionKey {
|
|
48
|
+
return key as SessionKey;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Type predicate: true if value is a QueueItemId.
|
|
53
|
+
*/
|
|
54
|
+
export function isQueueItemId(value: unknown): value is QueueItemId {
|
|
55
|
+
return typeof value === 'string';
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Type predicate: true if value is a WorkflowId.
|
|
60
|
+
*/
|
|
61
|
+
export function isWorkflowId(value: unknown): value is WorkflowId {
|
|
62
|
+
return typeof value === 'string';
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Type predicate: true if value is a SessionKey.
|
|
67
|
+
*/
|
|
68
|
+
export function isSessionKey(value: unknown): value is SessionKey {
|
|
69
|
+
return typeof value === 'string';
|
|
70
|
+
}
|
package/src/utils/file-lock.ts
CHANGED
|
@@ -322,7 +322,7 @@ export async function withLockAsync<T>(
|
|
|
322
322
|
* 注意:这是一个简化的实现,适用于单进程内的异步并发控制
|
|
323
323
|
* 对于多进程场景,应使用同步版本的 acquireLock
|
|
324
324
|
*/
|
|
325
|
-
const asyncLockQueues = new Map<string, Promise<void>>();
|
|
325
|
+
export const asyncLockQueues = new Map<string, Promise<void>>();
|
|
326
326
|
|
|
327
327
|
export async function withAsyncLock<T>(
|
|
328
328
|
filePath: string,
|
|
@@ -335,7 +335,7 @@ export async function withAsyncLock<T>(
|
|
|
335
335
|
|
|
336
336
|
// 创建新的 Promise 链
|
|
337
337
|
|
|
338
|
-
|
|
338
|
+
|
|
339
339
|
let resolveRelease: () => void;
|
|
340
340
|
const releasePromise = new Promise<void>(resolve => {
|
|
341
341
|
resolveRelease = resolve;
|