principles-disciple 1.52.0 → 1.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
- package/src/core/bootstrap-rules.ts +41 -4
- package/src/core/evolution-hook.ts +74 -0
- package/src/core/file-storage-adapter.ts +203 -0
- package/src/core/init.ts +29 -2
- package/src/core/nocturnal-trinity.ts +230 -0
- package/src/core/observability.ts +242 -0
- package/src/core/pain-signal-adapter.ts +42 -0
- package/src/core/pain-signal.ts +136 -0
- package/src/core/principle-injection.ts +208 -0
- package/src/core/principle-injector.ts +84 -0
- package/src/core/storage-adapter.ts +65 -0
- package/src/core/telemetry-event.ts +109 -0
- package/src/hooks/prompt.ts +18 -3
- package/src/service/evolution-worker.ts +52 -2
- package/tests/core/evolution-hook.test.ts +123 -0
- package/tests/core/file-storage-adapter.test.ts +285 -0
- package/tests/core/nocturnal-trinity.test.ts +236 -0
- package/tests/core/observability.test.ts +383 -0
- package/tests/core/pain-signal-adapter.test.ts +116 -0
- package/tests/core/pain-signal.test.ts +190 -0
- package/tests/core/principle-injection.test.ts +223 -0
- package/tests/core/principle-injector.test.ts +90 -0
- package/tests/core/storage-conformance.test.ts +429 -0
- package/tests/core/telemetry-event.test.ts +119 -0
package/openclaw.plugin.json
CHANGED
package/package.json
CHANGED
|
@@ -13,8 +13,10 @@
|
|
|
13
13
|
* npm run bootstrap-rules (production)
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { loadLedger, createRule, updatePrinciple } from './principle-tree-ledger.js';
|
|
16
|
+
import { loadLedger, createRule, updatePrinciple, addPrincipleToLedger } from './principle-tree-ledger.js';
|
|
17
|
+
import type { LedgerPrinciple } from './principle-tree-ledger.js';
|
|
17
18
|
import { loadStore } from './principle-training-state.js';
|
|
19
|
+
import { CORE_THINKING_MODELS } from './init.js';
|
|
18
20
|
|
|
19
21
|
export interface BootstrapResult {
|
|
20
22
|
principleId: string;
|
|
@@ -77,12 +79,47 @@ export function selectPrinciplesForBootstrap(stateDir: string, limit = 3): strin
|
|
|
77
79
|
* @throws Error if no deterministic principles found
|
|
78
80
|
*/
|
|
79
81
|
export function bootstrapRules(stateDir: string, limit = 3): BootstrapResult[] {
|
|
82
|
+
// Migration: if T-01..T-10 exist in Training Store but not in Ledger Tree, backfill.
|
|
83
|
+
// This handles workspaces initialized before Ledger Tree was added.
|
|
84
|
+
const store = loadStore(stateDir);
|
|
85
|
+
const ledger = loadLedger(stateDir);
|
|
86
|
+
const hasTrainingT = Object.keys(store).some((id) => id.startsWith('T-'));
|
|
87
|
+
const hasAnyLedgerT = Object.keys(ledger.tree.principles).some((id) => id.startsWith('T-'));
|
|
88
|
+
if (hasTrainingT && !hasAnyLedgerT) {
|
|
89
|
+
console.warn('[bootstrap] Migrating T-01..T-10 from Training Store to Ledger Tree');
|
|
90
|
+
const now = new Date().toISOString();
|
|
91
|
+
for (const [id, entry] of Object.entries(store)) {
|
|
92
|
+
if (!id.startsWith('T-')) continue;
|
|
93
|
+
const model = CORE_THINKING_MODELS.find((m) => m.id === id);
|
|
94
|
+
if (!model) continue;
|
|
95
|
+
const lp: LedgerPrinciple = {
|
|
96
|
+
id,
|
|
97
|
+
version: 1,
|
|
98
|
+
text: model.description,
|
|
99
|
+
coreAxiomId: id,
|
|
100
|
+
triggerPattern: '',
|
|
101
|
+
action: '',
|
|
102
|
+
status: 'active',
|
|
103
|
+
priority: 'P1',
|
|
104
|
+
scope: 'general',
|
|
105
|
+
evaluability: entry.evaluability,
|
|
106
|
+
valueScore: 0,
|
|
107
|
+
adherenceRate: 0,
|
|
108
|
+
painPreventedCount: 0,
|
|
109
|
+
derivedFromPainIds: [],
|
|
110
|
+
ruleIds: [],
|
|
111
|
+
conflictsWithPrincipleIds: [],
|
|
112
|
+
createdAt: now,
|
|
113
|
+
updatedAt: now,
|
|
114
|
+
suggestedRules: [],
|
|
115
|
+
};
|
|
116
|
+
addPrincipleToLedger(stateDir, lp);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
80
120
|
// Select principles for bootstrap
|
|
81
121
|
const selectedPrincipleIds = selectPrinciplesForBootstrap(stateDir, limit);
|
|
82
122
|
|
|
83
|
-
// Load current ledger state
|
|
84
|
-
const ledger = loadLedger(stateDir);
|
|
85
|
-
|
|
86
123
|
const results: BootstrapResult[] = [];
|
|
87
124
|
|
|
88
125
|
for (const principleId of selectedPrincipleIds) {
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EvolutionHook interface for the Evolution SDK.
|
|
3
|
+
*
|
|
4
|
+
* Provides a callback-based interface for observing evolution lifecycle
|
|
5
|
+
* events: pain detection, principle creation, and principle promotion.
|
|
6
|
+
*
|
|
7
|
+
* Per D-03, this interface contains only the 3 core event methods.
|
|
8
|
+
* Per D-04, consumers implement the interface directly (no EventEmitter).
|
|
9
|
+
* Hooks not needed can use the provided noOpEvolutionHook and override
|
|
10
|
+
* individual methods.
|
|
11
|
+
*/
|
|
12
|
+
import type { PainSignal } from './pain-signal.js';
|
|
13
|
+
|
|
14
|
+
// ---------------------------------------------------------------------------
|
|
15
|
+
// Event Types
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
/** Event payload for principle creation lifecycle events. */
|
|
19
|
+
export interface PrincipleCreatedEvent {
|
|
20
|
+
/** Unique principle identifier */
|
|
21
|
+
id: string;
|
|
22
|
+
/** Principle text ("When X, then Y.") */
|
|
23
|
+
text: string;
|
|
24
|
+
/** What triggered this principle's creation */
|
|
25
|
+
trigger: string;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Event payload for principle promotion lifecycle events. */
|
|
29
|
+
export interface PrinciplePromotedEvent {
|
|
30
|
+
/** Unique principle identifier */
|
|
31
|
+
id: string;
|
|
32
|
+
/** Previous status tier */
|
|
33
|
+
from: string;
|
|
34
|
+
/** New status tier */
|
|
35
|
+
to: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// ---------------------------------------------------------------------------
|
|
39
|
+
// EvolutionHook Interface
|
|
40
|
+
// ---------------------------------------------------------------------------
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Callback interface for observing evolution lifecycle events.
|
|
44
|
+
*
|
|
45
|
+
* Implement all 3 methods, or spread noOpEvolutionHook and override
|
|
46
|
+
* only the methods you need:
|
|
47
|
+
*
|
|
48
|
+
* @example
|
|
49
|
+
* ```ts
|
|
50
|
+
* const myHook: EvolutionHook = {
|
|
51
|
+
* ...noOpEvolutionHook,
|
|
52
|
+
* onPainDetected(signal) { console.log(signal); },
|
|
53
|
+
* };
|
|
54
|
+
* ```
|
|
55
|
+
*/
|
|
56
|
+
export interface EvolutionHook {
|
|
57
|
+
/** Called when a pain signal is detected and recorded. */
|
|
58
|
+
onPainDetected(signal: PainSignal): void;
|
|
59
|
+
/** Called when a new principle candidate is created. */
|
|
60
|
+
onPrincipleCreated(event: PrincipleCreatedEvent): void;
|
|
61
|
+
/** Called when a principle is promoted to a higher tier. */
|
|
62
|
+
onPrinciplePromoted(event: PrinciplePromotedEvent): void;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// No-op Helper
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
/** No-op implementation -- consumers can spread and override individual methods. */
|
|
70
|
+
export const noOpEvolutionHook: EvolutionHook = {
|
|
71
|
+
onPainDetected(_signal: PainSignal): void {},
|
|
72
|
+
onPrincipleCreated(_event: PrincipleCreatedEvent): void {},
|
|
73
|
+
onPrinciplePromoted(_event: PrinciplePromotedEvent): void {},
|
|
74
|
+
};
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FileStorageAdapter — file-backed implementation of StorageAdapter.
|
|
3
|
+
*
|
|
4
|
+
* Wraps principle-tree-ledger functions with the async StorageAdapter
|
|
5
|
+
* contract. Uses withLockAsync for thread-safe mutateLedger with
|
|
6
|
+
* retry with exponential backoff for lock acquisition (5 retries).
|
|
7
|
+
* Write failures are logged via SystemLogger and re-thrown.
|
|
8
|
+
*
|
|
9
|
+
* Guarantees:
|
|
10
|
+
* - Atomic writes via atomicWriteFileSync (temp + rename)
|
|
11
|
+
* - Thread-safe concurrent access via file locks
|
|
12
|
+
* - Consistent read-after-write visibility
|
|
13
|
+
* - Write failures logged to SystemLogger and re-thrown
|
|
14
|
+
*/
|
|
15
|
+
import * as fs from 'fs';
|
|
16
|
+
import * as path from 'path';
|
|
17
|
+
import type { StorageAdapter } from './storage-adapter.js';
|
|
18
|
+
import type { HybridLedgerStore } from './principle-tree-ledger.js';
|
|
19
|
+
import { TREE_NAMESPACE } from './principle-tree-ledger.js';
|
|
20
|
+
import {
|
|
21
|
+
loadLedger as loadLedgerFromFile,
|
|
22
|
+
saveLedgerAsync,
|
|
23
|
+
} from './principle-tree-ledger.js';
|
|
24
|
+
import { withLockAsync, type LockOptions, LockAcquisitionError } from '../utils/file-lock.js';
|
|
25
|
+
import { atomicWriteFileSync } from '../utils/io.js';
|
|
26
|
+
import { SystemLogger } from './system-logger.js';
|
|
27
|
+
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
// Configuration
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
|
|
32
|
+
/** Maximum retries for lock acquisition in mutateLedger. */
|
|
33
|
+
const MUTATE_RETRY_COUNT = 5;
|
|
34
|
+
|
|
35
|
+
/** Base delay in ms for exponential backoff between retries. */
|
|
36
|
+
const MUTATE_BACKOFF_BASE_MS = 50;
|
|
37
|
+
|
|
38
|
+
/** Maximum backoff delay in ms. */
|
|
39
|
+
const MUTATE_BACKOFF_MAX_MS = 500;
|
|
40
|
+
|
|
41
|
+
const PRINCIPLE_TRAINING_FILE = 'principle_training_state.json';
|
|
42
|
+
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
// Internal helpers
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Serialize the hybrid ledger store to JSON.
|
|
49
|
+
* Mirrors the unexported serializeLedger from principle-tree-ledger.ts.
|
|
50
|
+
*/
|
|
51
|
+
function serializeStore(store: HybridLedgerStore): string {
|
|
52
|
+
return JSON.stringify(
|
|
53
|
+
{
|
|
54
|
+
...store.trainingStore,
|
|
55
|
+
[TREE_NAMESPACE]: {
|
|
56
|
+
...store.tree,
|
|
57
|
+
lastUpdated: new Date().toISOString(),
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
null,
|
|
61
|
+
2,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Ensure the parent directory exists before writing. */
|
|
66
|
+
function ensureParentDir(filePath: string): void {
|
|
67
|
+
const dir = path.dirname(filePath);
|
|
68
|
+
if (!fs.existsSync(dir)) {
|
|
69
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// ---------------------------------------------------------------------------
|
|
74
|
+
// FileStorageAdapter
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* File-system backed storage adapter for the principle ledger.
|
|
79
|
+
*
|
|
80
|
+
* Delegates read/write operations to principle-tree-ledger while providing
|
|
81
|
+
* the async StorageAdapter interface. The mutateLedger method uses
|
|
82
|
+
* withLockAsync with exponential backoff retry for robust concurrent access.
|
|
83
|
+
*/
|
|
84
|
+
export class FileStorageAdapter implements StorageAdapter {
|
|
85
|
+
private readonly stateDir: string;
|
|
86
|
+
private readonly workspaceDir: string | undefined;
|
|
87
|
+
|
|
88
|
+
constructor(stateDir: string, workspaceDir?: string) {
|
|
89
|
+
this.stateDir = stateDir;
|
|
90
|
+
this.workspaceDir = workspaceDir;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Resolve the ledger file path for this state directory. */
|
|
94
|
+
private get filePath(): string {
|
|
95
|
+
return path.join(this.stateDir, PRINCIPLE_TRAINING_FILE);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Load the current ledger state from the file system.
|
|
100
|
+
*
|
|
101
|
+
* Returns an empty store if no persisted state exists (first run).
|
|
102
|
+
* Uses the synchronous loadLedger from principle-tree-ledger which
|
|
103
|
+
* handles missing/corrupted files gracefully.
|
|
104
|
+
*/
|
|
105
|
+
async loadLedger(): Promise<HybridLedgerStore> {
|
|
106
|
+
return loadLedgerFromFile(this.stateDir);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Persist the full ledger state atomically.
|
|
111
|
+
*
|
|
112
|
+
* Delegates to principle-tree-ledger's saveLedgerAsync which uses
|
|
113
|
+
* withLockAsync internally. Logs failures via SystemLogger.
|
|
114
|
+
*/
|
|
115
|
+
async saveLedger(store: HybridLedgerStore): Promise<void> {
|
|
116
|
+
try {
|
|
117
|
+
await saveLedgerAsync(this.stateDir, store);
|
|
118
|
+
} catch (err) {
|
|
119
|
+
SystemLogger.log(
|
|
120
|
+
this.workspaceDir,
|
|
121
|
+
'STORAGE_WRITE_FAILED',
|
|
122
|
+
`FileStorageAdapter.saveLedger failed: ${String(err)}`,
|
|
123
|
+
);
|
|
124
|
+
throw err;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Perform a read-modify-write cycle with automatic locking and retry.
|
|
130
|
+
*
|
|
131
|
+
* Uses withLockAsync to acquire a file lock, reads the current store,
|
|
132
|
+
* applies the mutate function, then writes the modified store atomically.
|
|
133
|
+
* On lock acquisition failure, retries up to MUTATE_RETRY_COUNT (5) times
|
|
134
|
+
* with exponential backoff + jitter to reduce contention.
|
|
135
|
+
*
|
|
136
|
+
* Write failures are logged to SystemLogger and re-thrown so callers
|
|
137
|
+
* can decide how to handle persistence errors.
|
|
138
|
+
*/
|
|
139
|
+
async mutateLedger<T>(mutate: (store: HybridLedgerStore) => T | Promise<T>): Promise<T> {
|
|
140
|
+
let lastError: Error | undefined;
|
|
141
|
+
|
|
142
|
+
for (let attempt = 0; attempt < MUTATE_RETRY_COUNT; attempt++) {
|
|
143
|
+
try {
|
|
144
|
+
const lockOptions: LockOptions = {
|
|
145
|
+
maxRetries: 3,
|
|
146
|
+
baseRetryDelayMs: 10,
|
|
147
|
+
maxRetryDelayMs: 200,
|
|
148
|
+
lockStaleMs: 10_000,
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
const ledgerPath = this.filePath;
|
|
152
|
+
|
|
153
|
+
return await withLockAsync(ledgerPath, async () => {
|
|
154
|
+
const store = loadLedgerFromFile(this.stateDir);
|
|
155
|
+
const result = await mutate(store);
|
|
156
|
+
|
|
157
|
+
// Write directly — we already hold the lock, so we must NOT
|
|
158
|
+
// call saveLedger/saveLedgerAsync (they try to acquire the same lock).
|
|
159
|
+
try {
|
|
160
|
+
ensureParentDir(ledgerPath);
|
|
161
|
+
atomicWriteFileSync(ledgerPath, serializeStore(store));
|
|
162
|
+
} catch (writeErr) {
|
|
163
|
+
SystemLogger.log(
|
|
164
|
+
this.workspaceDir,
|
|
165
|
+
'STORAGE_WRITE_FAILED',
|
|
166
|
+
`FileStorageAdapter.mutateLedger write failed: ${String(writeErr)}`,
|
|
167
|
+
);
|
|
168
|
+
throw writeErr;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
return result;
|
|
172
|
+
}, lockOptions);
|
|
173
|
+
} catch (err) {
|
|
174
|
+
lastError = err as Error;
|
|
175
|
+
|
|
176
|
+
// Only retry on lock acquisition errors
|
|
177
|
+
if (err instanceof LockAcquisitionError && attempt < MUTATE_RETRY_COUNT - 1) {
|
|
178
|
+
const delay = Math.min(
|
|
179
|
+
MUTATE_BACKOFF_BASE_MS * Math.pow(2, attempt),
|
|
180
|
+
MUTATE_BACKOFF_MAX_MS,
|
|
181
|
+
);
|
|
182
|
+
// Add jitter (0-20%) to avoid thundering herd
|
|
183
|
+
const jitter = delay * 0.2 * Math.random();
|
|
184
|
+
const totalDelay = Math.floor(delay + jitter);
|
|
185
|
+
|
|
186
|
+
await new Promise((resolve) => setTimeout(resolve, totalDelay));
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// Non-retryable error or exhausted retries
|
|
191
|
+
SystemLogger.log(
|
|
192
|
+
this.workspaceDir,
|
|
193
|
+
'STORAGE_MUTATE_FAILED',
|
|
194
|
+
`FileStorageAdapter.mutateLedger failed after ${attempt + 1} attempts: ${String(err)}`,
|
|
195
|
+
);
|
|
196
|
+
throw err;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Should not reach here, but satisfy the type checker
|
|
201
|
+
throw lastError ?? new Error('FileStorageAdapter.mutateLedger: unexpected state');
|
|
202
|
+
}
|
|
203
|
+
}
|
package/src/core/init.ts
CHANGED
|
@@ -5,6 +5,8 @@ import type { OpenClawPluginApi, PluginLogger } from '../openclaw-sdk.js';
|
|
|
5
5
|
import { PD_DIRS } from './paths.js';
|
|
6
6
|
import { defaultContextConfig } from '../types.js';
|
|
7
7
|
import { loadStore, setPrincipleState, type PrincipleTrainingState } from './principle-training-state.js';
|
|
8
|
+
import { addPrincipleToLedger } from './principle-tree-ledger.js';
|
|
9
|
+
import type { LedgerPrinciple } from './principle-tree-ledger.js';
|
|
8
10
|
import { atomicWriteFileSync } from '../utils/io.js';
|
|
9
11
|
import { createDefaultKeywordStore, saveKeywordStore } from './empathy-keyword-matcher.js';
|
|
10
12
|
|
|
@@ -150,7 +152,7 @@ function copyRecursiveSync(srcDir: string, destDir: string, api: OpenClawPluginA
|
|
|
150
152
|
* Core thinking model definitions (T-01 through T-10).
|
|
151
153
|
* These are the built-in cognitive patterns that every workspace should have.
|
|
152
154
|
*/
|
|
153
|
-
const CORE_THINKING_MODELS: Array<{
|
|
155
|
+
export const CORE_THINKING_MODELS: Array<{
|
|
154
156
|
id: string;
|
|
155
157
|
name: string;
|
|
156
158
|
description: string;
|
|
@@ -190,7 +192,7 @@ export function ensureCorePrinciples(stateDir: string, logger: PluginLogger): bo
|
|
|
190
192
|
for (const model of CORE_THINKING_MODELS) {
|
|
191
193
|
const state: PrincipleTrainingState = {
|
|
192
194
|
principleId: model.id,
|
|
193
|
-
evaluability: '
|
|
195
|
+
evaluability: 'manual_only',
|
|
194
196
|
applicableOpportunityCount: 0,
|
|
195
197
|
observedViolationCount: 0,
|
|
196
198
|
complianceRate: 0,
|
|
@@ -202,6 +204,31 @@ export function ensureCorePrinciples(stateDir: string, logger: PluginLogger): bo
|
|
|
202
204
|
internalizationStatus: 'needs_training',
|
|
203
205
|
};
|
|
204
206
|
setPrincipleState(stateDir, state);
|
|
207
|
+
|
|
208
|
+
// Also write to Ledger Tree so bootstrapRules() can find them
|
|
209
|
+
const now = new Date().toISOString();
|
|
210
|
+
const ledgerPrinciple: LedgerPrinciple = {
|
|
211
|
+
id: model.id,
|
|
212
|
+
version: 1,
|
|
213
|
+
text: model.description,
|
|
214
|
+
coreAxiomId: model.id,
|
|
215
|
+
triggerPattern: '',
|
|
216
|
+
action: '',
|
|
217
|
+
status: 'active',
|
|
218
|
+
priority: 'P1',
|
|
219
|
+
scope: 'general',
|
|
220
|
+
evaluability: 'manual_only',
|
|
221
|
+
valueScore: 0,
|
|
222
|
+
adherenceRate: 0,
|
|
223
|
+
painPreventedCount: 0,
|
|
224
|
+
derivedFromPainIds: [],
|
|
225
|
+
ruleIds: [],
|
|
226
|
+
conflictsWithPrincipleIds: [],
|
|
227
|
+
createdAt: now,
|
|
228
|
+
updatedAt: now,
|
|
229
|
+
suggestedRules: [],
|
|
230
|
+
};
|
|
231
|
+
addPrincipleToLedger(stateDir, ledgerPrinciple);
|
|
205
232
|
}
|
|
206
233
|
|
|
207
234
|
logger.info(`[PD] Initialized ${CORE_THINKING_MODELS.length} core thinking models: T-01 through T-10`);
|
|
@@ -2211,6 +2211,20 @@ export async function runTrinityAsync(options: RunTrinityOptions): Promise<Trini
|
|
|
2211
2211
|
telemetry.eligibleCandidateCount = draftArtifact.telemetry.eligibleCandidateCount;
|
|
2212
2212
|
}
|
|
2213
2213
|
|
|
2214
|
+
// Hallucination detection (SDK-QUAL-02): validate extraction against snapshot
|
|
2215
|
+
const hallucinationResult = validateExtraction(draftArtifact, snapshot);
|
|
2216
|
+
if (!hallucinationResult.isGrounded) {
|
|
2217
|
+
const reason = hallucinationResult.reason ?? 'Extraction not grounded in session evidence';
|
|
2218
|
+
console.warn(`[Trinity] HALLUCINATION_DETECTED: ${reason}`);
|
|
2219
|
+
telemetry.stageFailures.push(`Hallucination: ${reason}`);
|
|
2220
|
+
return {
|
|
2221
|
+
success: false,
|
|
2222
|
+
telemetry,
|
|
2223
|
+
failures: [{ stage: 'scribe', reason }],
|
|
2224
|
+
fallbackOccurred: false,
|
|
2225
|
+
};
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2214
2228
|
return {
|
|
2215
2229
|
success: true,
|
|
2216
2230
|
artifact: draftArtifact,
|
|
@@ -2339,6 +2353,20 @@ function runTrinityWithStubs(
|
|
|
2339
2353
|
telemetry.eligibleCandidateCount = draftArtifact.telemetry.eligibleCandidateCount;
|
|
2340
2354
|
}
|
|
2341
2355
|
|
|
2356
|
+
// Hallucination detection (SDK-QUAL-02): validate extraction against snapshot
|
|
2357
|
+
const hallucinationResult = validateExtraction(draftArtifact, snapshot);
|
|
2358
|
+
if (!hallucinationResult.isGrounded) {
|
|
2359
|
+
const reason = hallucinationResult.reason ?? 'Extraction not grounded in session evidence';
|
|
2360
|
+
console.warn(`[Trinity] HALLUCINATION_DETECTED: ${reason}`);
|
|
2361
|
+
telemetry.stageFailures.push(`Hallucination: ${reason}`);
|
|
2362
|
+
return {
|
|
2363
|
+
success: false,
|
|
2364
|
+
telemetry,
|
|
2365
|
+
failures: [{ stage: 'scribe', reason }],
|
|
2366
|
+
fallbackOccurred: false,
|
|
2367
|
+
};
|
|
2368
|
+
}
|
|
2369
|
+
|
|
2342
2370
|
return {
|
|
2343
2371
|
success: true,
|
|
2344
2372
|
artifact: draftArtifact,
|
|
@@ -2405,6 +2433,208 @@ export function validateDraftArtifact(draft: TrinityDraftArtifact): DraftValidat
|
|
|
2405
2433
|
};
|
|
2406
2434
|
}
|
|
2407
2435
|
|
|
2436
|
+
// ---------------------------------------------------------------------------
|
|
2437
|
+
// Hallucination Detection (SDK-QUAL-02)
|
|
2438
|
+
// ---------------------------------------------------------------------------
|
|
2439
|
+
|
|
2440
|
+
/**
|
|
2441
|
+
* Result of hallucination validation against session snapshot evidence.
|
|
2442
|
+
*/
|
|
2443
|
+
export interface HallucinationDetectionResult {
|
|
2444
|
+
/** Whether the extraction is grounded in real session evidence */
|
|
2445
|
+
isGrounded: boolean;
|
|
2446
|
+
/** List of evidence types found in the snapshot supporting the extraction */
|
|
2447
|
+
evidenceTypes: string[];
|
|
2448
|
+
/** Detailed reason if hallucination is detected */
|
|
2449
|
+
reason?: string;
|
|
2450
|
+
/** Matching evidence items for telemetry (truncated for safety) */
|
|
2451
|
+
evidencePreview: string[];
|
|
2452
|
+
}
|
|
2453
|
+
|
|
2454
|
+
/**
|
|
2455
|
+
* Validate that an extracted badDecision corresponds to actual events in the
|
|
2456
|
+
* NocturnalSessionSnapshot. This catches hallucinated extractions where the
|
|
2457
|
+
* Trinity chain produces a badDecision that has no grounding in real failures,
|
|
2458
|
+
* pain events, or gate blocks.
|
|
2459
|
+
*
|
|
2460
|
+
* Evidence sources checked:
|
|
2461
|
+
* 1. Failed tool calls (snapshot.toolCalls with outcome='failure')
|
|
2462
|
+
* 2. Pain events (snapshot.painEvents with score >= 50)
|
|
2463
|
+
* 3. Gate blocks (snapshot.gateBlocks)
|
|
2464
|
+
* 4. User corrections (snapshot.userTurns with correctionDetected=true)
|
|
2465
|
+
*
|
|
2466
|
+
* The function uses keyword overlap heuristics: it extracts tool names, file
|
|
2467
|
+
* paths, error messages, and pain reasons from the snapshot and checks if the
|
|
2468
|
+
* badDecision text overlaps meaningfully with any of them.
|
|
2469
|
+
*
|
|
2470
|
+
* @param artifact The draft artifact produced by the Scribe stage
|
|
2471
|
+
* @param snapshot The session snapshot used to generate the extraction
|
|
2472
|
+
* @returns HallucinationDetectionResult indicating whether the extraction is grounded
|
|
2473
|
+
*/
|
|
2474
|
+
export function validateExtraction(
|
|
2475
|
+
artifact: TrinityDraftArtifact,
|
|
2476
|
+
snapshot: NocturnalSessionSnapshot
|
|
2477
|
+
): HallucinationDetectionResult {
|
|
2478
|
+
const evidenceTypes: string[] = [];
|
|
2479
|
+
const evidencePreview: string[] = [];
|
|
2480
|
+
|
|
2481
|
+
// Shared token normalizer: lowercase + strip punctuation, same as badDecisionTokens
|
|
2482
|
+
const normalizeEvidenceToken = (value: string): string =>
|
|
2483
|
+
value.toLowerCase().replace(/[^a-z0-9]/g, '');
|
|
2484
|
+
|
|
2485
|
+
// Build a set of evidence tokens from the snapshot
|
|
2486
|
+
const evidenceTokens = new Set<string>();
|
|
2487
|
+
const badDecisionLower = artifact.badDecision.toLowerCase();
|
|
2488
|
+
|
|
2489
|
+
// 1. Failed tool calls
|
|
2490
|
+
const failedToolCalls = (snapshot.toolCalls ?? []).filter(tc => tc.outcome === 'failure');
|
|
2491
|
+
if (failedToolCalls.length > 0) {
|
|
2492
|
+
evidenceTypes.push('tool_failures');
|
|
2493
|
+
for (const tc of failedToolCalls) {
|
|
2494
|
+
// Extract tool name tokens
|
|
2495
|
+
evidenceTokens.add(tc.toolName.toLowerCase());
|
|
2496
|
+
if (tc.filePath) {
|
|
2497
|
+
// Extract all path segments and normalize each for matching
|
|
2498
|
+
const rawPathParts = [tc.filePath, ...tc.filePath.split(/[\\/]/)];
|
|
2499
|
+
for (const part of rawPathParts) {
|
|
2500
|
+
const normalized = normalizeEvidenceToken(part);
|
|
2501
|
+
if (normalized.length > 0) evidenceTokens.add(normalized);
|
|
2502
|
+
}
|
|
2503
|
+
}
|
|
2504
|
+
if (tc.errorMessage) {
|
|
2505
|
+
// Extract key words from error messages (filter stop words)
|
|
2506
|
+
const errorWords = tc.errorMessage.toLowerCase().split(/\s+/)
|
|
2507
|
+
.filter(w => w.length > 3 && !['with', 'from', 'that', 'this', 'which', 'been', 'have', 'were', 'they', 'their'].includes(w));
|
|
2508
|
+
for (const w of errorWords) {
|
|
2509
|
+
const normalized = normalizeEvidenceToken(w);
|
|
2510
|
+
if (normalized.length > 0) evidenceTokens.add(normalized);
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
2513
|
+
if (tc.errorType) evidenceTokens.add(tc.errorType.toLowerCase());
|
|
2514
|
+
evidencePreview.push(`tool:${tc.toolName}${tc.filePath ? `@${tc.filePath}` : ''} -> ${tc.errorMessage ?? 'unknown'}`.slice(0, 100));
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
// 2. Pain events (score >= 50 indicates meaningful pain)
|
|
2519
|
+
const significantPainEvents = (snapshot.painEvents ?? []).filter(pe => pe.score >= 50);
|
|
2520
|
+
if (significantPainEvents.length > 0) {
|
|
2521
|
+
evidenceTypes.push('pain_events');
|
|
2522
|
+
for (const pe of significantPainEvents) {
|
|
2523
|
+
evidenceTokens.add(pe.source.toLowerCase());
|
|
2524
|
+
if (pe.reason) {
|
|
2525
|
+
const painWords = pe.reason.toLowerCase().split(/\s+/)
|
|
2526
|
+
.filter(w => w.length > 3 && !['with', 'from', 'that', 'this', 'which', 'been', 'have', 'were', 'they', 'their'].includes(w));
|
|
2527
|
+
for (const w of painWords) {
|
|
2528
|
+
const normalized = normalizeEvidenceToken(w);
|
|
2529
|
+
if (normalized.length > 0) evidenceTokens.add(normalized);
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
evidencePreview.push(`pain:${pe.score} [${pe.source}] ${pe.reason ?? ''}`.slice(0, 100));
|
|
2533
|
+
}
|
|
2534
|
+
}
|
|
2535
|
+
|
|
2536
|
+
// 3. Gate blocks
|
|
2537
|
+
if ((snapshot.gateBlocks ?? []).length > 0) {
|
|
2538
|
+
evidenceTypes.push('gate_blocks');
|
|
2539
|
+
for (const gb of snapshot.gateBlocks) {
|
|
2540
|
+
evidenceTokens.add(gb.toolName.toLowerCase());
|
|
2541
|
+
evidenceTokens.add('gate');
|
|
2542
|
+
evidenceTokens.add('blocked');
|
|
2543
|
+
if (gb.reason) {
|
|
2544
|
+
const blockWords = gb.reason.toLowerCase().split(/\s+/)
|
|
2545
|
+
.filter(w => w.length > 3);
|
|
2546
|
+
for (const w of blockWords) {
|
|
2547
|
+
const normalized = normalizeEvidenceToken(w);
|
|
2548
|
+
if (normalized.length > 0) evidenceTokens.add(normalized);
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2551
|
+
evidencePreview.push(`gate:${gb.toolName} -> ${gb.reason}`.slice(0, 100));
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
|
|
2555
|
+
// 4. User corrections
|
|
2556
|
+
const userCorrections = (snapshot.userTurns ?? []).filter(ut => ut.correctionDetected);
|
|
2557
|
+
if (userCorrections.length > 0) {
|
|
2558
|
+
evidenceTypes.push('user_corrections');
|
|
2559
|
+
evidenceTokens.add('correction');
|
|
2560
|
+
evidenceTokens.add('wrong');
|
|
2561
|
+
evidenceTokens.add('incorrect');
|
|
2562
|
+
evidencePreview.push(`corrections:${userCorrections.length}`);
|
|
2563
|
+
}
|
|
2564
|
+
|
|
2565
|
+
// If no evidence exists at all in the snapshot, we cannot validate.
|
|
2566
|
+
// Allow the extraction through — the pipeline already has guardrails for
|
|
2567
|
+
// empty snapshots (Dreamer returns valid:false).
|
|
2568
|
+
if (evidenceTypes.length === 0) {
|
|
2569
|
+
return {
|
|
2570
|
+
isGrounded: true,
|
|
2571
|
+
evidenceTypes: [],
|
|
2572
|
+
reason: undefined,
|
|
2573
|
+
evidencePreview: [],
|
|
2574
|
+
};
|
|
2575
|
+
}
|
|
2576
|
+
|
|
2577
|
+
// Check for overlap between badDecision text and evidence tokens
|
|
2578
|
+
// We look for meaningful keyword matches (tokens of length > 4)
|
|
2579
|
+
const badDecisionTokens = badDecisionLower.split(/\s+/)
|
|
2580
|
+
.map(t => t.replace(/[^a-z0-9]/g, ''))
|
|
2581
|
+
.filter(t => t.length > 4);
|
|
2582
|
+
|
|
2583
|
+
let matchCount = 0;
|
|
2584
|
+
const matchedTokens: string[] = [];
|
|
2585
|
+
for (const token of badDecisionTokens) {
|
|
2586
|
+
// Direct match
|
|
2587
|
+
if (evidenceTokens.has(token)) {
|
|
2588
|
+
matchCount++;
|
|
2589
|
+
matchedTokens.push(token);
|
|
2590
|
+
continue;
|
|
2591
|
+
}
|
|
2592
|
+
// Partial match: check if any evidence token contains this token or vice versa
|
|
2593
|
+
for (const evToken of evidenceTokens) {
|
|
2594
|
+
if (evToken.length > 4 && (evToken.includes(token) || token.includes(evToken))) {
|
|
2595
|
+
matchCount++;
|
|
2596
|
+
matchedTokens.push(token);
|
|
2597
|
+
break;
|
|
2598
|
+
}
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
|
|
2602
|
+
// Heuristic: if at least 2 meaningful tokens overlap, consider grounded
|
|
2603
|
+
// Single overlap is acceptable if the token is highly specific (length > 8)
|
|
2604
|
+
const minOverlap = badDecisionTokens.length > 0
|
|
2605
|
+
? Math.max(1, Math.ceil(badDecisionTokens.length * 0.15))
|
|
2606
|
+
: 0;
|
|
2607
|
+
|
|
2608
|
+
if (matchCount >= Math.max(2, minOverlap)) {
|
|
2609
|
+
return {
|
|
2610
|
+
isGrounded: true,
|
|
2611
|
+
evidenceTypes,
|
|
2612
|
+
evidencePreview: evidencePreview.slice(0, 5),
|
|
2613
|
+
};
|
|
2614
|
+
}
|
|
2615
|
+
|
|
2616
|
+
// Also check for at least one highly-specific match (length > 8)
|
|
2617
|
+
const hasHighlySpecificMatch = matchedTokens.some(t => t.length > 8);
|
|
2618
|
+
if (hasHighlySpecificMatch) {
|
|
2619
|
+
return {
|
|
2620
|
+
isGrounded: true,
|
|
2621
|
+
evidenceTypes,
|
|
2622
|
+
evidencePreview: evidencePreview.slice(0, 5),
|
|
2623
|
+
};
|
|
2624
|
+
}
|
|
2625
|
+
|
|
2626
|
+
// Hallucination detected — badDecision has no grounding in snapshot evidence
|
|
2627
|
+
const reason = `Hallucinated extraction: badDecision "${artifact.badDecision.slice(0, 80)}" has insufficient overlap with session evidence. ` +
|
|
2628
|
+
`Evidence types available: [${evidenceTypes.join(', ')}]. Matched tokens: [${matchedTokens.join(', ')}] (needed >= ${Math.max(2, minOverlap)}).`;
|
|
2629
|
+
|
|
2630
|
+
return {
|
|
2631
|
+
isGrounded: false,
|
|
2632
|
+
evidenceTypes,
|
|
2633
|
+
reason,
|
|
2634
|
+
evidencePreview: evidencePreview.slice(0, 5),
|
|
2635
|
+
};
|
|
2636
|
+
}
|
|
2637
|
+
|
|
2408
2638
|
/**
|
|
2409
2639
|
* Convert a TrinityDraftArtifact to a NocturnalArtifact-compatible structure.
|
|
2410
2640
|
*/
|