principles-disciple 1.7.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/evolution-status.js +114 -118
- package/dist/commands/rollback.js +9 -3
- package/dist/commands/trust.js +64 -81
- package/dist/core/event-log.d.ts +6 -1
- package/dist/core/event-log.js +7 -0
- package/dist/core/session-tracker.d.ts +10 -2
- package/dist/core/session-tracker.js +60 -9
- package/dist/core/trust-engine.d.ts +4 -0
- package/dist/core/trust-engine.js +20 -25
- package/dist/hooks/gate.js +68 -53
- package/dist/hooks/llm.js +5 -2
- package/dist/hooks/subagent.js +42 -27
- package/dist/hooks/trajectory-collector.d.ts +32 -0
- package/dist/hooks/trajectory-collector.js +256 -0
- package/dist/index.js +22 -0
- package/dist/service/empathy-observer-manager.d.ts +2 -0
- package/dist/service/empathy-observer-manager.js +43 -1
- package/dist/service/evolution-worker.d.ts +19 -1
- package/dist/service/evolution-worker.js +116 -31
- package/dist/service/runtime-summary-service.d.ts +79 -0
- package/dist/service/runtime-summary-service.js +319 -0
- package/dist/types/event-types.d.ts +1 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Trajectory Collector - 行为进化引擎 Phase 0 数据收集
|
|
3
|
+
*
|
|
4
|
+
* 收集工具调用和 LLM 输出到 memory/trajectories/ 目录
|
|
5
|
+
* 用于分析工具使用模式、识别原则应用案例、评估行为质量
|
|
6
|
+
*/
|
|
7
|
+
import * as fs from 'fs';
|
|
8
|
+
import * as path from 'path';
|
|
9
|
+
const TRAJECTORY_DIR = 'memory/trajectories/';
|
|
10
|
+
// 敏感字段匹配正则
|
|
11
|
+
const SENSITIVE_KEY_PATTERN = /password|token|authorization|secret|api[_-]?key|credential|cookie|session/i;
|
|
12
|
+
// 最大字符串长度
|
|
13
|
+
const MAX_STRING_LENGTH = 1000;
|
|
14
|
+
const MAX_RESULT_LENGTH = 500;
|
|
15
|
+
/**
|
|
16
|
+
* 递归脱敏处理:遍历对象/数组,移除敏感字段值
|
|
17
|
+
*/
|
|
18
|
+
function scrubSensitive(obj, depth = 0) {
|
|
19
|
+
// 防止无限递归
|
|
20
|
+
if (depth > 10)
|
|
21
|
+
return '[MAX_DEPTH]';
|
|
22
|
+
// 处理 null/undefined
|
|
23
|
+
if (obj == null)
|
|
24
|
+
return obj;
|
|
25
|
+
// 处理基本类型
|
|
26
|
+
if (typeof obj !== 'object') {
|
|
27
|
+
if (typeof obj === 'string' && obj.length > MAX_STRING_LENGTH) {
|
|
28
|
+
return obj.slice(0, MAX_STRING_LENGTH) + '...[truncated]';
|
|
29
|
+
}
|
|
30
|
+
return obj;
|
|
31
|
+
}
|
|
32
|
+
// 处理数组
|
|
33
|
+
if (Array.isArray(obj)) {
|
|
34
|
+
return obj.map(item => scrubSensitive(item, depth + 1));
|
|
35
|
+
}
|
|
36
|
+
// 处理对象
|
|
37
|
+
const result = {};
|
|
38
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
39
|
+
if (SENSITIVE_KEY_PATTERN.test(key)) {
|
|
40
|
+
result[key] = '[REDACTED]';
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
result[key] = scrubSensitive(value, depth + 1);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return result;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* 异步写入队列 - 确保有序、非阻塞写入
|
|
50
|
+
*/
|
|
51
|
+
class AsyncWriteQueue {
|
|
52
|
+
queue = [];
|
|
53
|
+
processing = false;
|
|
54
|
+
async enqueue(task) {
|
|
55
|
+
this.queue.push(task);
|
|
56
|
+
if (!this.processing) {
|
|
57
|
+
this.processNext();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async processNext() {
|
|
61
|
+
if (this.queue.length === 0) {
|
|
62
|
+
this.processing = false;
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
this.processing = true;
|
|
66
|
+
const task = this.queue.shift();
|
|
67
|
+
try {
|
|
68
|
+
await task();
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// Silently fail - trajectory collection should not block main functionality
|
|
72
|
+
}
|
|
73
|
+
// 处理下一个任务
|
|
74
|
+
this.processNext();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
// 全局写入队列实例
|
|
78
|
+
const writeQueue = new AsyncWriteQueue();
|
|
79
|
+
// 目录缓存(避免重复检查)
|
|
80
|
+
const dirCache = new Map();
|
|
81
|
+
/**
|
|
82
|
+
* 确保轨迹目录存在(异步)
|
|
83
|
+
*/
|
|
84
|
+
async function ensureTrajectoryDirAsync(workspaceDir) {
|
|
85
|
+
const dir = path.join(workspaceDir, TRAJECTORY_DIR);
|
|
86
|
+
if (dirCache.get(dir)) {
|
|
87
|
+
return dir;
|
|
88
|
+
}
|
|
89
|
+
try {
|
|
90
|
+
await fs.promises.mkdir(dir, { recursive: true });
|
|
91
|
+
dirCache.set(dir, true);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// 目录可能已存在,忽略错误
|
|
95
|
+
dirCache.set(dir, true);
|
|
96
|
+
}
|
|
97
|
+
return dir;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* 获取今日轨迹文件名
|
|
101
|
+
*/
|
|
102
|
+
function getTodayFilename() {
|
|
103
|
+
const now = new Date();
|
|
104
|
+
const year = now.getUTCFullYear();
|
|
105
|
+
const month = String(now.getUTCMonth() + 1).padStart(2, '0');
|
|
106
|
+
return `${year}-${month}-${String(now.getUTCDate()).padStart(2, '0')}.jsonl`;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* 写入轨迹记录(JSON Lines 格式)- 异步版本
|
|
110
|
+
*/
|
|
111
|
+
function writeTrajectoryRecord(workspaceDir, record) {
|
|
112
|
+
const line = JSON.stringify(record) + '\n';
|
|
113
|
+
writeQueue.enqueue(async () => {
|
|
114
|
+
const dir = await ensureTrajectoryDirAsync(workspaceDir);
|
|
115
|
+
const filepath = path.join(dir, getTodayFilename());
|
|
116
|
+
await fs.promises.appendFile(filepath, line, 'utf8');
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* 工具调用完成后的处理
|
|
121
|
+
* 记录:工具名、参数、结果、错误、执行时间
|
|
122
|
+
*/
|
|
123
|
+
export function handleAfterToolCall(event, ctx) {
|
|
124
|
+
const workspaceDir = ctx.workspaceDir;
|
|
125
|
+
if (!workspaceDir)
|
|
126
|
+
return;
|
|
127
|
+
// 递归脱敏处理所有字段
|
|
128
|
+
const sanitizedParams = scrubSensitive(event.params);
|
|
129
|
+
const sanitizedResult = event.result == null
|
|
130
|
+
? null
|
|
131
|
+
: String(scrubSensitive(event.result)).slice(0, MAX_RESULT_LENGTH);
|
|
132
|
+
const sanitizedError = event.error == null
|
|
133
|
+
? null
|
|
134
|
+
: String(scrubSensitive(event.error));
|
|
135
|
+
writeTrajectoryRecord(workspaceDir, {
|
|
136
|
+
type: 'tool_call',
|
|
137
|
+
timestamp: new Date().toISOString(),
|
|
138
|
+
sessionId: ctx.sessionId || 'unknown',
|
|
139
|
+
toolName: event.toolName,
|
|
140
|
+
params: sanitizedParams,
|
|
141
|
+
result: sanitizedResult,
|
|
142
|
+
error: sanitizedError,
|
|
143
|
+
durationMs: event.durationMs,
|
|
144
|
+
success: !event.error,
|
|
145
|
+
runId: event.runId || null,
|
|
146
|
+
toolCallId: event.toolCallId || null
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* LLM 输出处理
|
|
151
|
+
* 记录:provider、model、输出长度、token 使用量
|
|
152
|
+
*/
|
|
153
|
+
export function handleLlmOutput(event, ctx) {
|
|
154
|
+
const workspaceDir = ctx.workspaceDir;
|
|
155
|
+
if (!workspaceDir)
|
|
156
|
+
return;
|
|
157
|
+
const totalTextLength = event.assistantTexts?.reduce((sum, text) => sum + (text?.length || 0), 0) || 0;
|
|
158
|
+
writeTrajectoryRecord(workspaceDir, {
|
|
159
|
+
type: 'llm_output',
|
|
160
|
+
timestamp: new Date().toISOString(),
|
|
161
|
+
sessionId: ctx.sessionId || 'unknown',
|
|
162
|
+
provider: event.provider,
|
|
163
|
+
model: event.model,
|
|
164
|
+
textLength: totalTextLength,
|
|
165
|
+
outputCount: event.assistantTexts?.length || 0,
|
|
166
|
+
usage: event.usage ? scrubSensitive(event.usage) : null
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* 消息写入前的处理
|
|
171
|
+
* 记录:用户/助手消息内容
|
|
172
|
+
*/
|
|
173
|
+
export function handleBeforeMessageWrite(event, ctx) {
|
|
174
|
+
const workspaceDir = ctx.workspaceDir;
|
|
175
|
+
if (!workspaceDir)
|
|
176
|
+
return;
|
|
177
|
+
const msg = event.message;
|
|
178
|
+
if (!msg || !msg.role)
|
|
179
|
+
return;
|
|
180
|
+
// 只记录 user 和 assistant 消息
|
|
181
|
+
if (msg.role !== 'user' && msg.role !== 'assistant')
|
|
182
|
+
return;
|
|
183
|
+
// 提取文本内容
|
|
184
|
+
let content = '';
|
|
185
|
+
if (typeof msg.content === 'string') {
|
|
186
|
+
content = msg.content;
|
|
187
|
+
}
|
|
188
|
+
else if (Array.isArray(msg.content)) {
|
|
189
|
+
content = msg.content
|
|
190
|
+
.filter((part) => part?.type === 'text')
|
|
191
|
+
.map((part) => part.text)
|
|
192
|
+
.join('\n');
|
|
193
|
+
}
|
|
194
|
+
// 脱敏处理内容预览
|
|
195
|
+
const sanitizedPreview = scrubSensitive(content.slice(0, 200));
|
|
196
|
+
writeTrajectoryRecord(workspaceDir, {
|
|
197
|
+
type: 'message',
|
|
198
|
+
timestamp: new Date().toISOString(),
|
|
199
|
+
sessionId: event.sessionKey || 'unknown',
|
|
200
|
+
role: msg.role,
|
|
201
|
+
contentLength: content.length,
|
|
202
|
+
contentPreview: typeof sanitizedPreview === 'string' ? sanitizedPreview : '[sanitized]',
|
|
203
|
+
agentId: event.agentId || null
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* 脱敏处理:移除敏感参数(保留旧函数签名以兼容)
|
|
208
|
+
* @deprecated 使用 scrubSensitive 替代
|
|
209
|
+
*/
|
|
210
|
+
function sanitizeParams(params) {
|
|
211
|
+
return scrubSensitive(params);
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* 轨迹汇总统计(供 cron 任务调用)
|
|
215
|
+
*/
|
|
216
|
+
export function computeTrajectoryStats(workspaceDir) {
|
|
217
|
+
const dir = path.join(workspaceDir, TRAJECTORY_DIR);
|
|
218
|
+
const todayFile = path.join(dir, getTodayFilename());
|
|
219
|
+
if (!fs.existsSync(todayFile)) {
|
|
220
|
+
return { date: getTodayFilename(), totalRecords: 0, toolCalls: 0, llmOutputs: 0, messages: 0 };
|
|
221
|
+
}
|
|
222
|
+
const content = fs.readFileSync(todayFile, 'utf8');
|
|
223
|
+
const lines = content.split('\n').filter(line => line.trim());
|
|
224
|
+
const toolCalls = lines.filter(line => {
|
|
225
|
+
try {
|
|
226
|
+
return JSON.parse(line).type === 'tool_call';
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
return false;
|
|
230
|
+
}
|
|
231
|
+
}).length;
|
|
232
|
+
const llmOutputs = lines.filter(line => {
|
|
233
|
+
try {
|
|
234
|
+
return JSON.parse(line).type === 'llm_output';
|
|
235
|
+
}
|
|
236
|
+
catch {
|
|
237
|
+
return false;
|
|
238
|
+
}
|
|
239
|
+
}).length;
|
|
240
|
+
const messages = lines.filter(line => {
|
|
241
|
+
try {
|
|
242
|
+
return JSON.parse(line).type === 'message';
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
return false;
|
|
246
|
+
}
|
|
247
|
+
}).length;
|
|
248
|
+
return {
|
|
249
|
+
date: getTodayFilename(),
|
|
250
|
+
totalRecords: lines.length,
|
|
251
|
+
toolCalls,
|
|
252
|
+
llmOutputs,
|
|
253
|
+
messages,
|
|
254
|
+
generatedAt: new Date().toISOString()
|
|
255
|
+
};
|
|
256
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -6,6 +6,7 @@ import { handleBeforeReset, handleBeforeCompaction, handleAfterCompaction } from
|
|
|
6
6
|
import { handleLlmOutput } from './hooks/llm.js';
|
|
7
7
|
import { handleSubagentEnded } from './hooks/subagent.js';
|
|
8
8
|
import { handleBeforeMessageWrite } from './hooks/message-sanitize.js';
|
|
9
|
+
import * as TrajectoryCollector from './hooks/trajectory-collector.js';
|
|
9
10
|
import { handleInitStrategy, handleManageOkr } from './commands/strategy.js';
|
|
10
11
|
import { handleBootstrapTools, handleResearchTools } from './commands/capabilities.js';
|
|
11
12
|
import { handleThinkingOs } from './commands/thinking-os.js';
|
|
@@ -97,6 +98,27 @@ const plugin = {
|
|
|
97
98
|
api.logger.error(`[PD] Error in before_message_write: ${String(err)}`);
|
|
98
99
|
}
|
|
99
100
|
});
|
|
101
|
+
// ── Hook: Trajectory Collection (Behavior Evolution Phase 0) ──
|
|
102
|
+
// Note: after_tool_call and llm_output are safe to collect
|
|
103
|
+
// before_message_write conflicts with message-sanitize, skipping for now
|
|
104
|
+
api.on('after_tool_call', (event, ctx) => {
|
|
105
|
+
try {
|
|
106
|
+
const workspaceDir = ctx.workspaceDir || api.resolvePath('.');
|
|
107
|
+
TrajectoryCollector.handleAfterToolCall(event, { ...ctx, workspaceDir });
|
|
108
|
+
}
|
|
109
|
+
catch (err) {
|
|
110
|
+
// Non-critical: don't log, just skip
|
|
111
|
+
}
|
|
112
|
+
});
|
|
113
|
+
api.on('llm_output', (event, ctx) => {
|
|
114
|
+
try {
|
|
115
|
+
const workspaceDir = ctx.workspaceDir || api.resolvePath('.');
|
|
116
|
+
TrajectoryCollector.handleLlmOutput(event, { ...ctx, workspaceDir });
|
|
117
|
+
}
|
|
118
|
+
catch (err) {
|
|
119
|
+
// Non-critical: don't log, just skip
|
|
120
|
+
}
|
|
121
|
+
});
|
|
100
122
|
// ── Hook: Subagent Loop Closure ──
|
|
101
123
|
api.on('subagent_spawning', (_event, _ctx) => {
|
|
102
124
|
// No-op for now, just to satisfy the interface expected by tests.
|
|
@@ -38,5 +38,7 @@ export declare class EmpathyObserverManager {
|
|
|
38
38
|
private parseJsonPayload;
|
|
39
39
|
private extractAssistantText;
|
|
40
40
|
private scoreFromSeverity;
|
|
41
|
+
private normalizeSeverity;
|
|
42
|
+
private normalizeConfidence;
|
|
41
43
|
}
|
|
42
44
|
export declare const empathyObserverManager: EmpathyObserverManager;
|
|
@@ -71,7 +71,37 @@ export class EmpathyObserverManager {
|
|
|
71
71
|
if (parsed?.damageDetected && sessionId) {
|
|
72
72
|
const wctx = WorkspaceContext.fromHookContext({ workspaceDir });
|
|
73
73
|
const score = this.scoreFromSeverity(parsed.severity, wctx.config);
|
|
74
|
-
trackFriction(sessionId, score, `observer_empathy_${parsed.severity || 'mild'}`, workspaceDir);
|
|
74
|
+
trackFriction(sessionId, score, `observer_empathy_${parsed.severity || 'mild'}`, workspaceDir, { source: 'user_empathy' });
|
|
75
|
+
const eventId = `emp_obs_${Date.now()}_${Math.random().toString(36).substring(2, 8)}`;
|
|
76
|
+
wctx.eventLog.recordPainSignal(sessionId, {
|
|
77
|
+
score,
|
|
78
|
+
source: 'user_empathy',
|
|
79
|
+
reason: parsed.reason || 'Empathy observer detected likely user frustration.',
|
|
80
|
+
isRisky: false,
|
|
81
|
+
origin: 'system_infer',
|
|
82
|
+
severity: this.normalizeSeverity(parsed.severity),
|
|
83
|
+
confidence: this.normalizeConfidence(parsed.confidence),
|
|
84
|
+
detection_mode: 'structured',
|
|
85
|
+
deduped: false,
|
|
86
|
+
trigger_text_excerpt: rawText.substring(0, 120),
|
|
87
|
+
raw_score: score,
|
|
88
|
+
calibrated_score: score,
|
|
89
|
+
eventId,
|
|
90
|
+
});
|
|
91
|
+
try {
|
|
92
|
+
wctx.trajectory?.recordPainEvent?.({
|
|
93
|
+
sessionId,
|
|
94
|
+
source: 'user_empathy',
|
|
95
|
+
score,
|
|
96
|
+
reason: parsed.reason || 'Empathy observer detected likely user frustration.',
|
|
97
|
+
severity: this.normalizeSeverity(parsed.severity),
|
|
98
|
+
origin: 'system_infer',
|
|
99
|
+
confidence: this.normalizeConfidence(parsed.confidence),
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
api.logger.warn(`[PD:EmpathyObserver] Failed to persist observer pain event for ${sessionId}: ${String(error)}`);
|
|
104
|
+
}
|
|
75
105
|
api.logger.info(`[PD:EmpathyObserver] Applied GFI +${score} for ${sessionId}`);
|
|
76
106
|
}
|
|
77
107
|
}
|
|
@@ -143,5 +173,17 @@ export class EmpathyObserverManager {
|
|
|
143
173
|
return Number(config.get('empathy_engine.penalties.moderate') ?? 25);
|
|
144
174
|
return Number(config.get('empathy_engine.penalties.mild') ?? 10);
|
|
145
175
|
}
|
|
176
|
+
normalizeSeverity(severity) {
|
|
177
|
+
if (severity === 'severe')
|
|
178
|
+
return 'severe';
|
|
179
|
+
if (severity === 'moderate')
|
|
180
|
+
return 'moderate';
|
|
181
|
+
return 'mild';
|
|
182
|
+
}
|
|
183
|
+
normalizeConfidence(value) {
|
|
184
|
+
if (!Number.isFinite(value))
|
|
185
|
+
return 1;
|
|
186
|
+
return Math.max(0, Math.min(1, Number(value)));
|
|
187
|
+
}
|
|
146
188
|
}
|
|
147
189
|
export const empathyObserverManager = EmpathyObserverManager.getInstance();
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { OpenClawPluginServiceContext, OpenClawPluginApi } from '../openclaw-sdk.js';
|
|
2
|
+
import { WorkspaceContext } from '../core/workspace-context.js';
|
|
2
3
|
export interface EvolutionQueueItem {
|
|
3
4
|
id: string;
|
|
4
5
|
task?: string;
|
|
@@ -9,7 +10,22 @@ export interface EvolutionQueueItem {
|
|
|
9
10
|
trigger_text_preview?: string;
|
|
10
11
|
status: 'pending' | 'in_progress' | 'completed';
|
|
11
12
|
}
|
|
12
|
-
export declare
|
|
13
|
+
export declare const EVOLUTION_QUEUE_LOCK_SUFFIX = ".lock";
|
|
14
|
+
export declare const PAIN_CANDIDATES_LOCK_SUFFIX = ".candidates.lock";
|
|
15
|
+
export declare const LOCK_MAX_RETRIES = 50;
|
|
16
|
+
export declare const LOCK_RETRY_DELAY_MS = 50;
|
|
17
|
+
export declare const LOCK_STALE_MS = 30000;
|
|
18
|
+
export declare function createEvolutionTaskId(source: string, score: number, preview: string, reason: string, now: number): string;
|
|
19
|
+
export declare function shouldTrackPainCandidate(text: string): boolean;
|
|
20
|
+
export declare function createPainCandidateFingerprint(text: string): string;
|
|
21
|
+
export declare function summarizePainCandidateSample(text: string): string;
|
|
22
|
+
/**
|
|
23
|
+
* Acquire an exclusive file lock for the given resource.
|
|
24
|
+
* Returns a release function. Uses 'wx' flag for atomic exclusive create.
|
|
25
|
+
* Detects stale locks by checking PID and mtime.
|
|
26
|
+
*/
|
|
27
|
+
export declare function acquireQueueLock(lockPath: string, logger: any): (() => void) | null;
|
|
28
|
+
export declare function hasRecentDuplicateTask(queue: EvolutionQueueItem[], source: string, preview: string, now: number, reason?: string): boolean;
|
|
13
29
|
export declare function hasEquivalentPromotedRule(dictionary: {
|
|
14
30
|
getAllRules(): Record<string, {
|
|
15
31
|
type: string;
|
|
@@ -18,6 +34,8 @@ export declare function hasEquivalentPromotedRule(dictionary: {
|
|
|
18
34
|
status: string;
|
|
19
35
|
}>;
|
|
20
36
|
}, phrase: string): boolean;
|
|
37
|
+
export declare function trackPainCandidate(text: string, wctx: WorkspaceContext): void;
|
|
38
|
+
export declare function processPromotion(wctx: WorkspaceContext, logger: any, eventLog: any): void;
|
|
21
39
|
export interface ExtendedEvolutionWorkerService {
|
|
22
40
|
id: string;
|
|
23
41
|
api: OpenClawPluginApi | null;
|
|
@@ -10,16 +10,63 @@ import { initPersistence, flushAllSessions } from '../core/session-tracker.js';
|
|
|
10
10
|
let intervalId = null;
|
|
11
11
|
const PAIN_QUEUE_DEDUP_WINDOW_MS = 30 * 60 * 1000;
|
|
12
12
|
// P0 fix: File lock constants and helper for queue operations (prevents TOCTOU race)
|
|
13
|
-
const EVOLUTION_QUEUE_LOCK_SUFFIX = '.lock';
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
13
|
+
export const EVOLUTION_QUEUE_LOCK_SUFFIX = '.lock';
|
|
14
|
+
export const PAIN_CANDIDATES_LOCK_SUFFIX = '.candidates.lock';
|
|
15
|
+
export const LOCK_MAX_RETRIES = 50;
|
|
16
|
+
export const LOCK_RETRY_DELAY_MS = 50;
|
|
17
|
+
export const LOCK_STALE_MS = 30_000;
|
|
18
|
+
const PAIN_CANDIDATE_MAX_SAMPLES = 5;
|
|
19
|
+
const PAIN_CANDIDATE_SAMPLE_LEN = 1000;
|
|
20
|
+
const PAIN_CANDIDATE_FINGERPRINT_HEAD_LEN = 160;
|
|
21
|
+
const PAIN_CANDIDATE_FINGERPRINT_TAIL_LEN = 80;
|
|
22
|
+
export function createEvolutionTaskId(source, score, preview, reason, now) {
|
|
23
|
+
// Keep ids short for prompt injection, but include enough entropy to avoid
|
|
24
|
+
// collisions between different pain events that share the same source/score/preview.
|
|
25
|
+
return createHash('md5')
|
|
26
|
+
.update(`${source}:${score}:${preview}:${reason}:${now}`)
|
|
27
|
+
.digest('hex')
|
|
28
|
+
.substring(0, 8);
|
|
29
|
+
}
|
|
30
|
+
function normalizePainCandidateText(text) {
|
|
31
|
+
return text.replace(/\s+/g, ' ').trim();
|
|
32
|
+
}
|
|
33
|
+
export function shouldTrackPainCandidate(text) {
|
|
34
|
+
const normalized = normalizePainCandidateText(text);
|
|
35
|
+
if (!normalized)
|
|
36
|
+
return false;
|
|
37
|
+
if (normalized === 'NO_REPLY')
|
|
38
|
+
return false;
|
|
39
|
+
// Skip empathy observer payloads: they are classifier telemetry, not user/system pain patterns.
|
|
40
|
+
if (normalized.startsWith('{')
|
|
41
|
+
&& normalized.endsWith('}')
|
|
42
|
+
&& normalized.includes('"damageDetected"')
|
|
43
|
+
&& normalized.includes('"severity"')
|
|
44
|
+
&& normalized.includes('"confidence"')) {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
return true;
|
|
48
|
+
}
|
|
49
|
+
export function createPainCandidateFingerprint(text) {
|
|
50
|
+
const normalized = normalizePainCandidateText(text);
|
|
51
|
+
const head = normalized.substring(0, PAIN_CANDIDATE_FINGERPRINT_HEAD_LEN);
|
|
52
|
+
const tail = normalized.slice(-PAIN_CANDIDATE_FINGERPRINT_TAIL_LEN);
|
|
53
|
+
return createHash('md5')
|
|
54
|
+
.update(`${normalized.length}:${head}:${tail}`)
|
|
55
|
+
.digest('hex')
|
|
56
|
+
.substring(0, 8);
|
|
57
|
+
}
|
|
58
|
+
export function summarizePainCandidateSample(text) {
|
|
59
|
+
return normalizePainCandidateText(text).substring(0, PAIN_CANDIDATE_SAMPLE_LEN);
|
|
60
|
+
}
|
|
61
|
+
function isPendingPainCandidate(status) {
|
|
62
|
+
return status === undefined || status === 'pending';
|
|
63
|
+
}
|
|
17
64
|
/**
|
|
18
65
|
* Acquire an exclusive file lock for the given resource.
|
|
19
66
|
* Returns a release function. Uses 'wx' flag for atomic exclusive create.
|
|
20
67
|
* Detects stale locks by checking PID and mtime.
|
|
21
68
|
*/
|
|
22
|
-
function acquireQueueLock(lockPath, logger) {
|
|
69
|
+
export function acquireQueueLock(lockPath, logger) {
|
|
23
70
|
let retries = 0;
|
|
24
71
|
while (retries < LOCK_MAX_RETRIES) {
|
|
25
72
|
try {
|
|
@@ -75,18 +122,21 @@ function acquireQueueLock(lockPath, logger) {
|
|
|
75
122
|
logger?.warn?.(`[PD:EvolutionWorker] Failed to acquire lock after ${LOCK_MAX_RETRIES} retries: ${lockPath}`);
|
|
76
123
|
return null;
|
|
77
124
|
}
|
|
78
|
-
function normalizePainDedupKey(source, preview) {
|
|
79
|
-
|
|
125
|
+
function normalizePainDedupKey(source, preview, reason) {
|
|
126
|
+
// Include reason in dedup key to match createEvolutionTaskId() behavior
|
|
127
|
+
// Different reasons for the same source/preview should create different tasks
|
|
128
|
+
const normalizedReason = (reason || '').trim().toLowerCase();
|
|
129
|
+
return `${source.trim().toLowerCase()}::${preview.trim().toLowerCase()}::${normalizedReason}`;
|
|
80
130
|
}
|
|
81
|
-
export function hasRecentDuplicateTask(queue, source, preview, now) {
|
|
82
|
-
const key = normalizePainDedupKey(source, preview);
|
|
131
|
+
export function hasRecentDuplicateTask(queue, source, preview, now, reason) {
|
|
132
|
+
const key = normalizePainDedupKey(source, preview, reason);
|
|
83
133
|
return queue.some((task) => {
|
|
84
134
|
if (task.status === 'completed')
|
|
85
135
|
return false;
|
|
86
136
|
const taskTime = new Date(task.timestamp).getTime();
|
|
87
137
|
if (!Number.isFinite(taskTime) || (now - taskTime) > PAIN_QUEUE_DEDUP_WINDOW_MS)
|
|
88
138
|
return false;
|
|
89
|
-
return normalizePainDedupKey(task.source, task.trigger_text_preview || '') === key;
|
|
139
|
+
return normalizePainDedupKey(task.source, task.trigger_text_preview || '', task.reason) === key;
|
|
90
140
|
});
|
|
91
141
|
}
|
|
92
142
|
export function hasEquivalentPromotedRule(dictionary, phrase) {
|
|
@@ -148,12 +198,12 @@ function checkPainFlag(wctx, logger) {
|
|
|
148
198
|
}
|
|
149
199
|
}
|
|
150
200
|
const now = Date.now();
|
|
151
|
-
if (hasRecentDuplicateTask(queue, source, preview, now)) {
|
|
201
|
+
if (hasRecentDuplicateTask(queue, source, preview, now, reason)) {
|
|
152
202
|
logger?.info?.(`[PD:EvolutionWorker] Duplicate pain task skipped for source=${source} preview=${preview || 'N/A'}`);
|
|
153
203
|
fs.appendFileSync(painFlagPath, `\nstatus: queued\n`, 'utf8');
|
|
154
204
|
return;
|
|
155
205
|
}
|
|
156
|
-
const taskId =
|
|
206
|
+
const taskId = createEvolutionTaskId(source, score, preview, reason, now);
|
|
157
207
|
queue.push({
|
|
158
208
|
id: taskId,
|
|
159
209
|
score,
|
|
@@ -302,46 +352,77 @@ async function processDetectionQueue(wctx, api, eventLog) {
|
|
|
302
352
|
logger.warn(`[PD:EvolutionWorker] Detection queue failed: ${String(err)}`);
|
|
303
353
|
}
|
|
304
354
|
}
|
|
305
|
-
function trackPainCandidate(text, wctx) {
|
|
355
|
+
export function trackPainCandidate(text, wctx) {
|
|
356
|
+
if (!shouldTrackPainCandidate(text))
|
|
357
|
+
return;
|
|
306
358
|
const candidatePath = wctx.resolve('PAIN_CANDIDATES');
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
359
|
+
const lockPath = candidatePath + PAIN_CANDIDATES_LOCK_SUFFIX;
|
|
360
|
+
const releaseLock = acquireQueueLock(lockPath, console);
|
|
361
|
+
if (!releaseLock) {
|
|
362
|
+
console.warn('[PD:EvolutionWorker] Failed to acquire pain candidates lock, skipping track');
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
try {
|
|
366
|
+
let data = { candidates: {} };
|
|
367
|
+
if (fs.existsSync(candidatePath)) {
|
|
368
|
+
try {
|
|
369
|
+
data = JSON.parse(fs.readFileSync(candidatePath, 'utf8'));
|
|
370
|
+
}
|
|
371
|
+
catch (e) {
|
|
372
|
+
// Keep going with empty data if parse fails, but log it
|
|
373
|
+
console.error(`[PD:EvolutionWorker] Failed to parse pain candidates: ${String(e)}`);
|
|
374
|
+
}
|
|
311
375
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
376
|
+
const fingerprint = createPainCandidateFingerprint(text);
|
|
377
|
+
const now = new Date().toISOString();
|
|
378
|
+
if (!data.candidates[fingerprint]) {
|
|
379
|
+
data.candidates[fingerprint] = { count: 0, status: 'pending', firstSeen: now, lastSeen: now, samples: [] };
|
|
315
380
|
}
|
|
381
|
+
const cand = data.candidates[fingerprint];
|
|
382
|
+
cand.status = cand.status || 'pending';
|
|
383
|
+
cand.count++;
|
|
384
|
+
cand.lastSeen = now;
|
|
385
|
+
const sample = summarizePainCandidateSample(text);
|
|
386
|
+
if (cand.samples.length < PAIN_CANDIDATE_MAX_SAMPLES && !cand.samples.includes(sample)) {
|
|
387
|
+
cand.samples.push(sample);
|
|
388
|
+
}
|
|
389
|
+
fs.writeFileSync(candidatePath, JSON.stringify(data, null, 2), 'utf8');
|
|
316
390
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
data.candidates[fingerprint] = { count: 0, firstSeen: new Date().toISOString(), samples: [] };
|
|
391
|
+
finally {
|
|
392
|
+
releaseLock();
|
|
320
393
|
}
|
|
321
|
-
const cand = data.candidates[fingerprint];
|
|
322
|
-
cand.count++;
|
|
323
|
-
if (cand.samples.length < 5)
|
|
324
|
-
cand.samples.push(text.substring(0, 200));
|
|
325
|
-
fs.writeFileSync(candidatePath, JSON.stringify(data, null, 2), 'utf8');
|
|
326
394
|
}
|
|
327
|
-
function processPromotion(wctx, logger, eventLog) {
|
|
395
|
+
export function processPromotion(wctx, logger, eventLog) {
|
|
328
396
|
const candidatePath = wctx.resolve('PAIN_CANDIDATES');
|
|
329
397
|
if (!fs.existsSync(candidatePath))
|
|
330
398
|
return;
|
|
399
|
+
const lockPath = candidatePath + PAIN_CANDIDATES_LOCK_SUFFIX;
|
|
400
|
+
const releaseLock = acquireQueueLock(lockPath, logger);
|
|
401
|
+
if (!releaseLock) {
|
|
402
|
+
logger?.warn?.('[PD:EvolutionWorker] Failed to acquire pain candidates lock, skipping promotion');
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
331
405
|
try {
|
|
332
406
|
const config = wctx.config;
|
|
333
407
|
const dictionary = wctx.dictionary;
|
|
334
408
|
const data = JSON.parse(fs.readFileSync(candidatePath, 'utf8'));
|
|
335
409
|
const countThreshold = config.get('thresholds.promotion_count_threshold') || 3;
|
|
336
410
|
let promotedCount = 0;
|
|
411
|
+
let changed = false;
|
|
337
412
|
for (const [fingerprint, cand] of Object.entries(data.candidates)) {
|
|
338
|
-
if (cand.status
|
|
413
|
+
if (isPendingPainCandidate(cand.status) && cand.count >= countThreshold) {
|
|
414
|
+
// Normalize undefined status to 'pending'
|
|
415
|
+
if (cand.status !== 'pending') {
|
|
416
|
+
cand.status = 'pending';
|
|
417
|
+
changed = true;
|
|
418
|
+
}
|
|
339
419
|
const commonPhrases = extractCommonSubstring(cand.samples);
|
|
340
420
|
if (commonPhrases.length > 0) {
|
|
341
421
|
const phrase = commonPhrases[0];
|
|
342
422
|
const ruleId = `P_PROMOTED_${fingerprint.toUpperCase()}`;
|
|
343
423
|
if (hasEquivalentPromotedRule(dictionary, phrase)) {
|
|
344
424
|
cand.status = 'duplicate';
|
|
425
|
+
changed = true;
|
|
345
426
|
logger?.info?.(`[PD:EvolutionWorker] Skipping duplicate promoted rule for candidate ${fingerprint}: ${phrase}`);
|
|
346
427
|
continue;
|
|
347
428
|
}
|
|
@@ -356,10 +437,11 @@ function processPromotion(wctx, logger, eventLog) {
|
|
|
356
437
|
});
|
|
357
438
|
cand.status = 'promoted';
|
|
358
439
|
promotedCount++;
|
|
440
|
+
changed = true;
|
|
359
441
|
}
|
|
360
442
|
}
|
|
361
443
|
}
|
|
362
|
-
if (
|
|
444
|
+
if (changed) {
|
|
363
445
|
fs.writeFileSync(candidatePath, JSON.stringify(data, null, 2), 'utf8');
|
|
364
446
|
}
|
|
365
447
|
}
|
|
@@ -367,6 +449,9 @@ function processPromotion(wctx, logger, eventLog) {
|
|
|
367
449
|
if (logger)
|
|
368
450
|
logger.warn(`[PD:EvolutionWorker] Error during rule promotion: ${String(err)}`);
|
|
369
451
|
}
|
|
452
|
+
finally {
|
|
453
|
+
releaseLock();
|
|
454
|
+
}
|
|
370
455
|
}
|
|
371
456
|
export const EvolutionWorkerService = {
|
|
372
457
|
id: 'principles-evolution-worker',
|