principles-disciple 1.10.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +66 -0
- package/esbuild.config.js +1 -1
- package/openclaw.plugin.json +4 -4
- package/package.json +2 -3
- package/run-nocturnal.mjs +30 -0
- package/scripts/db-migrate.mjs +170 -0
- package/scripts/sync-plugin.mjs +94 -5
- package/src/commands/archive-impl.ts +136 -0
- package/src/commands/capabilities.ts +4 -2
- package/src/commands/context.ts +5 -1
- package/src/commands/disable-impl.ts +151 -0
- package/src/commands/evolution-status.ts +64 -19
- package/src/commands/export.ts +8 -6
- package/src/commands/focus.ts +8 -20
- package/src/commands/nocturnal-review.ts +5 -7
- package/src/commands/nocturnal-rollout.ts +1 -12
- package/src/commands/nocturnal-train.ts +17 -42
- package/src/commands/pain.ts +10 -5
- package/src/commands/principle-rollback.ts +4 -2
- package/src/commands/promote-impl.ts +274 -0
- package/src/commands/rollback-impl.ts +234 -0
- package/src/commands/rollback.ts +6 -3
- package/src/commands/samples.ts +2 -0
- package/src/commands/thinking-os.ts +3 -4
- package/src/commands/workflow-debug.ts +2 -1
- package/src/config/errors.ts +1 -0
- package/src/core/AGENTS.md +34 -0
- package/src/core/adaptive-thresholds.ts +4 -3
- package/src/core/code-implementation-storage.ts +241 -0
- package/src/core/config.ts +5 -2
- package/src/core/control-ui-db.ts +29 -10
- package/src/core/detection-funnel.ts +12 -7
- package/src/core/diagnostician-task-store.ts +156 -0
- package/src/core/dictionary.ts +4 -4
- package/src/core/empathy-keyword-matcher.ts +7 -3
- package/src/core/empathy-types.ts +13 -2
- package/src/core/event-log.ts +14 -6
- package/src/core/evolution-engine.ts +27 -31
- package/src/core/evolution-logger.ts +3 -2
- package/src/core/evolution-reducer.ts +83 -31
- package/src/core/evolution-types.ts +10 -0
- package/src/core/external-training-contract.ts +1 -0
- package/src/core/focus-history.ts +38 -24
- package/src/core/hygiene/tracker.ts +10 -6
- package/src/core/init.ts +5 -2
- package/src/core/migration.ts +3 -3
- package/src/core/model-deployment-registry.ts +6 -4
- package/src/core/model-training-registry.ts +5 -3
- package/src/core/nocturnal-arbiter.ts +13 -14
- package/src/core/nocturnal-artifact-lineage.ts +117 -0
- package/src/core/nocturnal-artificer.ts +257 -0
- package/src/core/nocturnal-candidate-scoring.ts +4 -2
- package/src/core/nocturnal-compliance.ts +67 -19
- package/src/core/nocturnal-dataset.ts +95 -2
- package/src/core/nocturnal-executability.ts +2 -3
- package/src/core/nocturnal-export.ts +6 -3
- package/src/core/nocturnal-rule-implementation-validator.ts +245 -0
- package/src/core/nocturnal-trajectory-extractor.ts +10 -3
- package/src/core/nocturnal-trinity.ts +300 -57
- package/src/core/pain-context-extractor.ts +29 -15
- package/src/core/pain.ts +7 -5
- package/src/core/path-resolver.ts +16 -15
- package/src/core/paths.ts +2 -1
- package/src/core/pd-task-reconciler.ts +463 -0
- package/src/core/pd-task-service.ts +42 -0
- package/src/core/pd-task-store.ts +77 -0
- package/src/core/pd-task-types.ts +128 -0
- package/src/core/principle-internalization/deprecated-readiness.ts +91 -0
- package/src/core/principle-internalization/internalization-routing-policy.ts +208 -0
- package/src/core/principle-internalization/lifecycle-metrics.ts +149 -0
- package/src/core/principle-internalization/lifecycle-read-model.ts +243 -0
- package/src/core/principle-internalization/lifecycle-refresh.ts +11 -0
- package/src/core/principle-internalization/principle-lifecycle-service.ts +167 -0
- package/src/core/principle-training-state.ts +95 -370
- package/src/core/principle-tree-ledger.ts +713 -0
- package/src/core/profile.ts +3 -1
- package/src/core/promotion-gate.ts +14 -18
- package/src/core/replay-engine.ts +562 -0
- package/src/core/risk-calculator.ts +6 -4
- package/src/core/rule-host-helpers.ts +39 -0
- package/src/core/rule-host-types.ts +82 -0
- package/src/core/rule-host.ts +245 -0
- package/src/core/rule-implementation-runtime.ts +38 -0
- package/src/core/schema/db-types.ts +16 -0
- package/src/core/schema/index.ts +26 -0
- package/src/core/schema/migration-runner.ts +207 -0
- package/src/core/schema/migrations/001-init-trajectory.ts +211 -0
- package/src/core/schema/migrations/002-init-central.ts +122 -0
- package/src/core/schema/migrations/003-init-workflow.ts +55 -0
- package/src/core/schema/migrations/004-add-thinking-and-gfi.ts +74 -0
- package/src/core/schema/migrations/index.ts +31 -0
- package/src/core/schema/schema-definitions.ts +650 -0
- package/src/core/session-tracker.ts +6 -4
- package/src/core/shadow-observation-registry.ts +6 -3
- package/src/core/system-logger.ts +2 -2
- package/src/core/thinking-models.ts +182 -46
- package/src/core/thinking-os-parser.ts +164 -0
- package/src/core/training-program.ts +7 -7
- package/src/core/trajectory.ts +42 -36
- package/src/core/workspace-context.ts +77 -11
- package/src/core/workspace-dir-validation.ts +152 -0
- package/src/hooks/AGENTS.md +31 -0
- package/src/hooks/bash-risk.ts +3 -1
- package/src/hooks/edit-verification.ts +9 -5
- package/src/hooks/gate-block-helper.ts +5 -1
- package/src/hooks/gate.ts +152 -5
- package/src/hooks/gfi-gate.ts +9 -2
- package/src/hooks/lifecycle-routing.ts +124 -0
- package/src/hooks/lifecycle.ts +12 -12
- package/src/hooks/llm.ts +17 -109
- package/src/hooks/message-sanitize.ts +5 -3
- package/src/hooks/pain.ts +19 -15
- package/src/hooks/progressive-trust-gate.ts +7 -1
- package/src/hooks/prompt.ts +169 -60
- package/src/hooks/subagent.ts +5 -4
- package/src/hooks/thinking-checkpoint.ts +2 -0
- package/src/hooks/trajectory-collector.ts +15 -12
- package/src/http/principles-console-route.ts +31 -68
- package/src/i18n/commands.ts +2 -2
- package/src/index.ts +126 -40
- package/src/service/central-database.ts +131 -43
- package/src/service/central-health-service.ts +47 -0
- package/src/service/central-overview-service.ts +135 -0
- package/src/service/central-sync-service.ts +87 -0
- package/src/service/control-ui-query-service.ts +46 -36
- package/src/service/event-log-auditor.ts +261 -0
- package/src/service/evolution-query-service.ts +23 -22
- package/src/service/evolution-worker.ts +565 -261
- package/src/service/health-query-service.ts +213 -36
- package/src/service/nocturnal-runtime.ts +8 -4
- package/src/service/nocturnal-service.ts +499 -59
- package/src/service/nocturnal-target-selector.ts +5 -7
- package/src/service/runtime-summary-service.ts +2 -1
- package/src/service/subagent-workflow/deep-reflect-workflow-manager.ts +25 -336
- package/src/service/subagent-workflow/dynamic-timeout.ts +30 -0
- package/src/service/subagent-workflow/empathy-observer-workflow-manager.ts +48 -386
- package/src/service/subagent-workflow/index.ts +2 -0
- package/src/service/subagent-workflow/nocturnal-workflow-manager.ts +155 -285
- package/src/service/subagent-workflow/runtime-direct-driver.ts +114 -16
- package/src/service/subagent-workflow/subagent-error-utils.ts +25 -0
- package/src/service/subagent-workflow/types.ts +9 -4
- package/src/service/subagent-workflow/workflow-manager-base.ts +573 -0
- package/src/service/subagent-workflow/workflow-store.ts +71 -11
- package/src/service/trajectory-service.ts +2 -1
- package/src/tools/critique-prompt.ts +1 -1
- package/src/tools/deep-reflect.ts +175 -209
- package/src/tools/model-index.ts +2 -1
- package/src/types/event-types.ts +2 -2
- package/src/types/principle-tree-schema.ts +29 -23
- package/src/utils/file-lock.ts +5 -3
- package/src/utils/io.ts +5 -2
- package/src/utils/nlp.ts +5 -46
- package/src/utils/node-vm-polyfill.ts +11 -0
- package/src/utils/plugin-logger.ts +2 -0
- package/src/utils/retry.ts +572 -0
- package/src/utils/subagent-probe.ts +1 -1
- package/templates/langs/en/core/AGENTS.md +0 -13
- package/templates/langs/en/core/SOUL.md +1 -31
- package/templates/langs/en/core/TOOLS.md +0 -4
- package/templates/langs/en/principles/THINKING_OS.md +64 -0
- package/templates/langs/en/skills/admin/SKILL.md +0 -1
- package/templates/langs/en/skills/evolution-framework-update/SKILL.md +1 -1
- package/templates/langs/en/skills/pd-diagnostician/SKILL.md +18 -5
- package/templates/langs/zh/core/AGENTS.md +0 -22
- package/templates/langs/zh/core/SOUL.md +1 -31
- package/templates/langs/zh/core/TOOLS.md +0 -4
- package/templates/langs/zh/principles/THINKING_OS.md +64 -0
- package/templates/langs/zh/skills/admin/SKILL.md +0 -1
- package/templates/langs/zh/skills/evolution-framework-update/SKILL.md +1 -1
- package/templates/langs/zh/skills/pd-diagnostician/SKILL.md +25 -4
- package/tests/commands/evolution-status.test.ts +119 -0
- package/tests/commands/implementation-lifecycle.test.ts +362 -0
- package/tests/core/code-implementation-storage.test.ts +398 -0
- package/tests/core/evolution-reducer.detector-metadata.test.ts +28 -28
- package/tests/core/nocturnal-artifact-lineage.test.ts +53 -0
- package/tests/core/nocturnal-artificer.test.ts +241 -0
- package/tests/core/nocturnal-compliance-p-principles.test.ts +133 -0
- package/tests/core/nocturnal-rule-implementation-validator.test.ts +127 -0
- package/tests/core/pd-task-store.test.ts +126 -0
- package/tests/core/principle-internalization/deprecated-readiness.test.ts +193 -0
- package/tests/core/principle-internalization/internalization-routing-policy.test.ts +212 -0
- package/tests/core/principle-internalization/lifecycle-metrics.test.ts +350 -0
- package/tests/core/principle-internalization/principle-lifecycle-service.test.ts +211 -0
- package/tests/core/principle-training-state.test.ts +228 -1
- package/tests/core/principle-tree-ledger.test.ts +423 -0
- package/tests/core/regression-v1-9-1.test.ts +265 -0
- package/tests/core/replay-engine.test.ts +234 -0
- package/tests/core/rule-host-helpers.test.ts +120 -0
- package/tests/core/rule-host.test.ts +389 -0
- package/tests/core/rule-implementation-runtime.test.ts +64 -0
- package/tests/core/workspace-context.test.ts +53 -0
- package/tests/core/workspace-dir-validation.test.ts +272 -0
- package/tests/hooks/gate-rule-host-pipeline.test.ts +385 -0
- package/tests/hooks/pain.test.ts +74 -10
- package/tests/hooks/prompt.test.ts +63 -1
- package/tests/integration/principle-lifecycle.e2e.test.ts +197 -0
- package/tests/integration/tool-hooks-workspace-dir.e2e.test.ts +211 -0
- package/tests/service/data-endpoints-regression.test.ts +834 -0
- package/tests/service/evolution-worker.test.ts +0 -123
- package/tests/service/nocturnal-service-code-candidate.test.ts +330 -0
- package/tests/utils/nlp.test.ts +1 -19
- package/tests/utils/retry.test.ts +327 -0
- package/ui/src/App.tsx +1 -1
- package/ui/src/api.ts +4 -0
- package/ui/src/charts.tsx +366 -0
- package/ui/src/components/WorkspaceConfig.tsx +107 -75
- package/ui/src/i18n/ui.ts +55 -22
- package/ui/src/pages/OverviewPage.tsx +441 -81
- package/ui/src/styles.css +43 -0
- package/ui/src/types.ts +17 -1
- package/src/agents/nocturnal-dreamer.md +0 -152
- package/src/agents/nocturnal-philosopher.md +0 -138
- package/src/agents/nocturnal-reflector.md +0 -126
- package/src/agents/nocturnal-scribe.md +0 -164
- package/templates/workspace/.principles/00-kernel.md +0 -51
- package/templates/workspace/.principles/DECISION_POLICY.json +0 -44
- package/templates/workspace/.principles/PRINCIPLES.md +0 -20
- package/templates/workspace/.principles/PROFILE.json +0 -54
- package/templates/workspace/.principles/PROFILE.schema.json +0 -56
- package/templates/workspace/.principles/THINKING_OS.md +0 -64
- package/templates/workspace/.principles/THINKING_OS_ARCHIVE.md +0 -7
- package/templates/workspace/.principles/THINKING_OS_CANDIDATES.md +0 -9
- package/templates/workspace/.principles/models/_INDEX.md +0 -27
- package/templates/workspace/.principles/models/first_principles.md +0 -62
- package/templates/workspace/.principles/models/marketing_4p.md +0 -52
- package/templates/workspace/.principles/models/porter_five.md +0 -63
- package/templates/workspace/.principles/models/swot.md +0 -60
- package/templates/workspace/.principles/models/user_story_map.md +0 -63
- package/templates/workspace/.state/WORKBOARD.json +0 -4
package/src/core/profile.ts
CHANGED
|
@@ -54,9 +54,11 @@ export const PROFILE_DEFAULTS = {
|
|
|
54
54
|
window_ms: 5 * 60 * 1000, // 5 minute window
|
|
55
55
|
high_risk_tools: ['run_shell_command', 'delete_file', 'move_file'],
|
|
56
56
|
},
|
|
57
|
-
custom_guards: [] as
|
|
57
|
+
custom_guards: [] as { pattern: string; message: string; severity: string }[],
|
|
58
58
|
};
|
|
59
59
|
|
|
60
|
+
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
61
|
+
// Reason: normalizeProfile handles arbitrary JSON profile shapes where static typing cannot capture runtime field existence
|
|
60
62
|
export function normalizeProfile(rawProfile: any): any {
|
|
61
63
|
const defaults = JSON.parse(JSON.stringify(PROFILE_DEFAULTS));
|
|
62
64
|
const warnings: string[] = [];
|
|
@@ -44,25 +44,12 @@ import * as fs from 'fs';
|
|
|
44
44
|
import * as path from 'path';
|
|
45
45
|
import * as crypto from 'crypto';
|
|
46
46
|
import { withLock } from '../utils/file-lock.js';
|
|
47
|
-
import type { WorkerProfile } from './model-deployment-registry.js';
|
|
48
47
|
import {
|
|
49
48
|
getCheckpoint,
|
|
50
49
|
getEvalSummary,
|
|
51
|
-
listEvalSummaries,
|
|
52
|
-
getCheckpointLineage,
|
|
53
50
|
} from './model-training-registry.js';
|
|
54
|
-
import {
|
|
55
|
-
|
|
56
|
-
getActiveCheckpointForProfile,
|
|
57
|
-
rollbackDeployment,
|
|
58
|
-
} from './model-deployment-registry.js';
|
|
59
|
-
import {
|
|
60
|
-
type TrainableWorkerProfile,
|
|
61
|
-
} from './external-training-contract.js';
|
|
62
|
-
import {
|
|
63
|
-
computeShadowStats,
|
|
64
|
-
type ShadowStats,
|
|
65
|
-
} from './shadow-observation-registry.js';
|
|
51
|
+
import { type TrainableWorkerProfile } from './external-training-contract.js';
|
|
52
|
+
import { computeShadowStats } from './shadow-observation-registry.js';
|
|
66
53
|
|
|
67
54
|
// ---------------------------------------------------------------------------
|
|
68
55
|
// Constants
|
|
@@ -83,6 +70,7 @@ export const DEFAULT_ALLOWED_MARGIN = 0.05;
|
|
|
83
70
|
* Allowed worker profiles for Phase 7 shadow rollout.
|
|
84
71
|
* Only bounded local workers eligible. local-reader first, local-editor deferred.
|
|
85
72
|
*/
|
|
73
|
+
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -- Reason: reserved for Phase 7 shadow rollout profile validation
|
|
86
74
|
const ALLOWED_ROLLOUT_PROFILES: readonly TrainableWorkerProfile[] = ['local-reader'];
|
|
87
75
|
|
|
88
76
|
/**
|
|
@@ -245,7 +233,8 @@ function writeRegistry(stateDir: string, registry: PromotionRegistry): void {
|
|
|
245
233
|
*/
|
|
246
234
|
function withPromotionRegistryLock<T>(
|
|
247
235
|
stateDir: string,
|
|
248
|
-
|
|
236
|
+
// eslint-disable-next-line no-unused-vars -- Reason: callback parameter name is type documentation, actual value passed at call site
|
|
237
|
+
fn: (_registry: PromotionRegistry) => T
|
|
249
238
|
): T {
|
|
250
239
|
const registryPath = getRegistryPath(stateDir);
|
|
251
240
|
return withLock(registryPath, () => {
|
|
@@ -329,7 +318,8 @@ export function evaluatePromotionGate(
|
|
|
329
318
|
): PromotionGateResult {
|
|
330
319
|
const {
|
|
331
320
|
checkpointId,
|
|
332
|
-
|
|
321
|
+
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars -- Reason: reserved for Phase 7 profile-based targeting
|
|
322
|
+
targetProfile: _targetProfile,
|
|
333
323
|
baselineMetrics,
|
|
334
324
|
minDelta = DEFAULT_MIN_DELTA,
|
|
335
325
|
allowedMargin = DEFAULT_ALLOWED_MARGIN,
|
|
@@ -380,7 +370,7 @@ export function evaluatePromotionGate(
|
|
|
380
370
|
}
|
|
381
371
|
|
|
382
372
|
// --- Check 4: Delta must be positive and above threshold ---
|
|
383
|
-
const delta = evalSummary
|
|
373
|
+
const {delta} = evalSummary;
|
|
384
374
|
const deltaCheck = {
|
|
385
375
|
actual: delta,
|
|
386
376
|
threshold: minDelta,
|
|
@@ -398,7 +388,9 @@ export function evaluatePromotionGate(
|
|
|
398
388
|
// PREFER real shadow evidence over eval verdict proxy
|
|
399
389
|
// Shadow evidence comes from actual runtime routing decisions
|
|
400
390
|
const shadowStats = computeShadowStats(stateDir, { checkpointId });
|
|
391
|
+
// eslint-disable-next-line @typescript-eslint/init-declarations -- assigned in both if/else branches
|
|
401
392
|
let arbiterRejectRate: number;
|
|
393
|
+
// eslint-disable-next-line @typescript-eslint/init-declarations -- assigned in both if/else branches
|
|
402
394
|
let arbiterRejectSource: 'shadow' | 'eval-proxy';
|
|
403
395
|
|
|
404
396
|
if (shadowStats && shadowStats.isStatisticallySignificant) {
|
|
@@ -432,7 +424,9 @@ export function evaluatePromotionGate(
|
|
|
432
424
|
|
|
433
425
|
// --- Check 6: Executability reject rate constraint ---
|
|
434
426
|
// PREFER real shadow evidence: escalation rate + profile rejection rate
|
|
427
|
+
// eslint-disable-next-line @typescript-eslint/init-declarations -- assigned in both if/else branches
|
|
435
428
|
let executabilityRejectRate: number;
|
|
429
|
+
// eslint-disable-next-line @typescript-eslint/init-declarations -- assigned in both if/else branches
|
|
436
430
|
let executabilityRejectSource: 'shadow' | 'eval-proxy';
|
|
437
431
|
|
|
438
432
|
if (shadowStats && shadowStats.isStatisticallySignificant) {
|
|
@@ -490,6 +484,7 @@ export function evaluatePromotionGate(
|
|
|
490
484
|
qualityCheck.passed;
|
|
491
485
|
|
|
492
486
|
// --- Suggest state based on checks ---
|
|
487
|
+
// eslint-disable-next-line @typescript-eslint/init-declarations -- assigned in all branches
|
|
493
488
|
let suggestedState: PromotionState | undefined;
|
|
494
489
|
if (allPassed) {
|
|
495
490
|
suggestedState = 'candidate_only';
|
|
@@ -590,6 +585,7 @@ export function advancePromotion(
|
|
|
590
585
|
// - rejected → candidate_only/shadow_ready: allowed via re-evaluation
|
|
591
586
|
// (new eval data may reverse a previous rejection)
|
|
592
587
|
//
|
|
588
|
+
// eslint-disable-next-line @typescript-eslint/init-declarations -- assigned in all branches
|
|
593
589
|
let targetState: PromotionState;
|
|
594
590
|
if (!gateResult.passes) {
|
|
595
591
|
targetState = 'rejected';
|
|
@@ -0,0 +1,562 @@
|
|
|
1
|
+
import * as fs from 'fs';
|
|
2
|
+
import * as path from 'path';
|
|
3
|
+
import { withLock } from '../utils/file-lock.js';
|
|
4
|
+
import { normalizePath, isRisky, planStatus as getPlanStatus } from '../utils/io.js';
|
|
5
|
+
import {
|
|
6
|
+
listSamplesByClassification,
|
|
7
|
+
loadSampleContent,
|
|
8
|
+
} from './nocturnal-dataset.js';
|
|
9
|
+
import {
|
|
10
|
+
getImplementationAssetRoot,
|
|
11
|
+
loadEntrySource,
|
|
12
|
+
} from './code-implementation-storage.js';
|
|
13
|
+
import type {
|
|
14
|
+
NocturnalDatasetRecord,
|
|
15
|
+
SampleClassification,
|
|
16
|
+
} from './nocturnal-dataset.js';
|
|
17
|
+
import { loadLedger } from './principle-tree-ledger.js';
|
|
18
|
+
import type { Implementation } from '../types/principle-tree-schema.js';
|
|
19
|
+
import type { RuleHostHelpers } from './rule-host-helpers.js';
|
|
20
|
+
import { createRuleHostHelpers } from './rule-host-helpers.js';
|
|
21
|
+
import type { RuleHostInput, RuleHostResult } from './rule-host-types.js';
|
|
22
|
+
import { loadRuleImplementationModule } from './rule-implementation-runtime.js';
|
|
23
|
+
import {
|
|
24
|
+
getNocturnalSessionSnapshot,
|
|
25
|
+
type NocturnalGateBlock,
|
|
26
|
+
type NocturnalSessionSnapshot,
|
|
27
|
+
type NocturnalToolCall,
|
|
28
|
+
} from './nocturnal-trajectory-extractor.js';
|
|
29
|
+
import { TrajectoryRegistry } from './trajectory.js';
|
|
30
|
+
|
|
31
|
+
export interface ReplaySample {
|
|
32
|
+
fingerprint: string;
|
|
33
|
+
classification: SampleClassification;
|
|
34
|
+
content: unknown;
|
|
35
|
+
expectedOutcome: {
|
|
36
|
+
shouldBlock?: boolean;
|
|
37
|
+
shouldPass?: boolean;
|
|
38
|
+
expectedPrinciple?: string;
|
|
39
|
+
};
|
|
40
|
+
record: NocturnalDatasetRecord;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface ReplayResult {
|
|
44
|
+
sampleFingerprint: string;
|
|
45
|
+
classification: SampleClassification;
|
|
46
|
+
passed: boolean;
|
|
47
|
+
reason?: string;
|
|
48
|
+
decision: string;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface ClassificationSummary {
|
|
52
|
+
total: number;
|
|
53
|
+
passed: number;
|
|
54
|
+
failed: number;
|
|
55
|
+
details: ReplayResult[];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface ReplayReport {
|
|
59
|
+
overallDecision: 'pass' | 'fail' | 'needs-review';
|
|
60
|
+
replayResults: {
|
|
61
|
+
painNegative: ClassificationSummary;
|
|
62
|
+
successPositive: ClassificationSummary;
|
|
63
|
+
principleAnchor: ClassificationSummary;
|
|
64
|
+
};
|
|
65
|
+
blockers: string[];
|
|
66
|
+
generatedAt: string;
|
|
67
|
+
implementationId: string;
|
|
68
|
+
sampleFingerprints: string[];
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface CandidateEvaluator {
|
|
72
|
+
/* eslint-disable no-unused-vars -- Reason: interface method params are type signatures */
|
|
73
|
+
evaluate(sample: unknown): { passed: boolean; reason?: string; decision: string };
|
|
74
|
+
/* eslint-enable no-unused-vars */
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export class ReplayEngine {
|
|
78
|
+
private readonly workspaceDir: string;
|
|
79
|
+
private readonly stateDir: string;
|
|
80
|
+
|
|
81
|
+
constructor(workspaceDir: string, stateDir: string) {
|
|
82
|
+
this.workspaceDir = workspaceDir;
|
|
83
|
+
this.stateDir = stateDir;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
loadSamples(classifications: SampleClassification[]): ReplaySample[] {
|
|
87
|
+
const samples: ReplaySample[] = [];
|
|
88
|
+
|
|
89
|
+
for (const classification of classifications) {
|
|
90
|
+
const records = listSamplesByClassification(this.workspaceDir, classification);
|
|
91
|
+
|
|
92
|
+
for (const record of records) {
|
|
93
|
+
try {
|
|
94
|
+
const content = loadSampleContent(this.workspaceDir, record);
|
|
95
|
+
const expectedOutcome = this._deriveExpectedOutcome(record);
|
|
96
|
+
|
|
97
|
+
samples.push({
|
|
98
|
+
fingerprint: record.sampleFingerprint,
|
|
99
|
+
classification,
|
|
100
|
+
content,
|
|
101
|
+
expectedOutcome,
|
|
102
|
+
record,
|
|
103
|
+
});
|
|
104
|
+
} catch (err) {
|
|
105
|
+
console.warn(
|
|
106
|
+
`[ReplayEngine] Skipping sample ${record.sampleFingerprint}: ${String(err)}`
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
return samples;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Public API method that delegates to evaluator, no instance state needed
|
|
116
|
+
runSingleSample(sample: ReplaySample, evaluator: CandidateEvaluator): ReplayResult {
|
|
117
|
+
const evaluation = evaluator.evaluate(sample);
|
|
118
|
+
return {
|
|
119
|
+
sampleFingerprint: sample.fingerprint,
|
|
120
|
+
classification: sample.classification,
|
|
121
|
+
passed: evaluation.passed,
|
|
122
|
+
reason: evaluation.passed ? undefined : evaluation.reason,
|
|
123
|
+
decision: evaluation.decision,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
runReplay(
|
|
128
|
+
candidateImplId: string,
|
|
129
|
+
evaluator: CandidateEvaluator,
|
|
130
|
+
classifications?: SampleClassification[]
|
|
131
|
+
): ReplayReport {
|
|
132
|
+
const selectedClassifications: SampleClassification[] = classifications ?? [
|
|
133
|
+
'pain-negative',
|
|
134
|
+
'success-positive',
|
|
135
|
+
'principle-anchor',
|
|
136
|
+
];
|
|
137
|
+
|
|
138
|
+
const samples = this.loadSamples(selectedClassifications);
|
|
139
|
+
const allResults = samples.map((sample) => this.runSingleSample(sample, evaluator));
|
|
140
|
+
const report = this._buildReport(candidateImplId, allResults);
|
|
141
|
+
this._persistReport(report);
|
|
142
|
+
return report;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
runReplayForImplementation(
|
|
146
|
+
implementationId: string,
|
|
147
|
+
classifications?: SampleClassification[],
|
|
148
|
+
): ReplayReport {
|
|
149
|
+
const implementation = this._getImplementationById(implementationId);
|
|
150
|
+
if (!implementation) {
|
|
151
|
+
throw new Error(`Implementation not found: ${implementationId}`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const evaluator = this._createEvaluatorForImplementation(implementation);
|
|
155
|
+
return this.runReplay(implementationId, evaluator, classifications);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
listReports(implementationId: string): ReplayReport[] {
|
|
159
|
+
const reportDir = path.join(
|
|
160
|
+
getImplementationAssetRoot(this.stateDir, implementationId),
|
|
161
|
+
'replays'
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
if (!fs.existsSync(reportDir)) return [];
|
|
165
|
+
|
|
166
|
+
try {
|
|
167
|
+
const files = fs.readdirSync(reportDir).filter((file) => file.endsWith('.json'));
|
|
168
|
+
return files
|
|
169
|
+
.sort()
|
|
170
|
+
.reverse()
|
|
171
|
+
.map((file) => {
|
|
172
|
+
const content = fs.readFileSync(path.join(reportDir, file), 'utf-8');
|
|
173
|
+
return JSON.parse(content) as ReplayReport;
|
|
174
|
+
});
|
|
175
|
+
} catch {
|
|
176
|
+
return [];
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
getLatestReport(implementationId: string): ReplayReport | null {
|
|
181
|
+
const reports = this.listReports(implementationId);
|
|
182
|
+
return reports.length > 0 ? reports[0] : null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
hasPassingReport(implementationId: string): boolean {
|
|
186
|
+
return this.listReports(implementationId).some((report) => report.overallDecision === 'pass');
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
private _getImplementationById(implementationId: string): Implementation | null {
|
|
190
|
+
const ledger = loadLedger(this.stateDir);
|
|
191
|
+
return ledger.tree.implementations[implementationId] ?? null;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
private _createEvaluatorForImplementation(implementation: Implementation): CandidateEvaluator {
|
|
195
|
+
const sourceCode = loadEntrySource(this.stateDir, implementation.id);
|
|
196
|
+
if (!sourceCode) {
|
|
197
|
+
throw new Error(`Implementation asset entry source missing: ${implementation.id}`);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const moduleExports = loadRuleImplementationModule(sourceCode, implementation.id);
|
|
201
|
+
if (typeof moduleExports.evaluate !== 'function') {
|
|
202
|
+
throw new Error(`Implementation ${implementation.id} does not export evaluate().`);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/* eslint-disable no-unused-vars -- Reason: type-only parameters in type cast, not used at runtime */
|
|
206
|
+
const evaluate = moduleExports.evaluate as (
|
|
207
|
+
_input: RuleHostInput,
|
|
208
|
+
_helpers: RuleHostHelpers,
|
|
209
|
+
) => RuleHostResult;
|
|
210
|
+
/* eslint-enable no-unused-vars */
|
|
211
|
+
|
|
212
|
+
return {
|
|
213
|
+
evaluate: (sample: unknown) => {
|
|
214
|
+
const replaySample = sample as ReplaySample;
|
|
215
|
+
const input = this._buildRuleHostInput(replaySample);
|
|
216
|
+
if (!input) {
|
|
217
|
+
return {
|
|
218
|
+
passed: false,
|
|
219
|
+
reason: `Could not build replay input for sample ${replaySample.fingerprint}.`,
|
|
220
|
+
decision: 'replay-input-missing',
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const result = evaluate(input, createRuleHostHelpers(input));
|
|
225
|
+
return this._scoreEvaluation(replaySample, result);
|
|
226
|
+
},
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
private _buildRuleHostInput(sample: ReplaySample): RuleHostInput | null {
|
|
231
|
+
const snapshot = getNocturnalSessionSnapshot(
|
|
232
|
+
TrajectoryRegistry.get(this.workspaceDir),
|
|
233
|
+
sample.record.sessionId,
|
|
234
|
+
);
|
|
235
|
+
if (!snapshot) {
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const toolCall = this._selectToolCall(snapshot, sample.classification);
|
|
240
|
+
if (!toolCall) {
|
|
241
|
+
return null;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const normalizedPath =
|
|
245
|
+
typeof toolCall.filePath === 'string' && toolCall.filePath.length > 0
|
|
246
|
+
? normalizePath(toolCall.filePath, this.workspaceDir)
|
|
247
|
+
: null;
|
|
248
|
+
const matchedGateBlock = this._matchGateBlock(snapshot.gateBlocks, toolCall);
|
|
249
|
+
|
|
250
|
+
return {
|
|
251
|
+
action: {
|
|
252
|
+
toolName: toolCall.toolName,
|
|
253
|
+
normalizedPath,
|
|
254
|
+
paramsSummary: {
|
|
255
|
+
artifactId: sample.record.artifactId,
|
|
256
|
+
sourceSnapshotRef: sample.record.sourceSnapshotRef,
|
|
257
|
+
classification: sample.classification,
|
|
258
|
+
},
|
|
259
|
+
},
|
|
260
|
+
workspace: {
|
|
261
|
+
isRiskPath:
|
|
262
|
+
Boolean(matchedGateBlock) ||
|
|
263
|
+
(normalizedPath !== null && this._isRiskPath(normalizedPath)),
|
|
264
|
+
planStatus:
|
|
265
|
+
matchedGateBlock?.planStatus === 'READY' ||
|
|
266
|
+
matchedGateBlock?.planStatus === 'DRAFT' ||
|
|
267
|
+
matchedGateBlock?.planStatus === 'NONE'
|
|
268
|
+
? matchedGateBlock.planStatus
|
|
269
|
+
: this._safePlanStatus(),
|
|
270
|
+
hasPlanFile: fs.existsSync(path.join(this.workspaceDir, 'PLAN.md')),
|
|
271
|
+
},
|
|
272
|
+
session: {
|
|
273
|
+
sessionId: sample.record.sessionId,
|
|
274
|
+
currentGfi: 0,
|
|
275
|
+
recentThinking: false,
|
|
276
|
+
},
|
|
277
|
+
evolution: {
|
|
278
|
+
epTier: 0,
|
|
279
|
+
},
|
|
280
|
+
derived: {
|
|
281
|
+
estimatedLineChanges: this._estimateLineChanges(toolCall),
|
|
282
|
+
bashRisk: this._inferBashRisk(toolCall),
|
|
283
|
+
},
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
288
|
+
private _selectToolCall(
|
|
289
|
+
snapshot: NocturnalSessionSnapshot,
|
|
290
|
+
classification: SampleClassification,
|
|
291
|
+
): NocturnalToolCall | null {
|
|
292
|
+
const byNewest = [...snapshot.toolCalls].sort(
|
|
293
|
+
(left, right) => new Date(right.createdAt).getTime() - new Date(left.createdAt).getTime(),
|
|
294
|
+
);
|
|
295
|
+
|
|
296
|
+
if (classification === 'pain-negative') {
|
|
297
|
+
return (
|
|
298
|
+
byNewest.find((toolCall) => toolCall.outcome === 'blocked') ??
|
|
299
|
+
byNewest.find((toolCall) => toolCall.outcome === 'failure') ??
|
|
300
|
+
byNewest[0] ??
|
|
301
|
+
null
|
|
302
|
+
);
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
if (classification === 'success-positive' || classification === 'principle-anchor') {
|
|
306
|
+
return (
|
|
307
|
+
byNewest.find((toolCall) => toolCall.outcome === 'success') ??
|
|
308
|
+
byNewest.find((toolCall) => toolCall.outcome === 'failure') ??
|
|
309
|
+
byNewest[0] ??
|
|
310
|
+
null
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
return byNewest[0] ?? null;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
318
|
+
private _matchGateBlock(
|
|
319
|
+
gateBlocks: NocturnalGateBlock[],
|
|
320
|
+
toolCall: NocturnalToolCall,
|
|
321
|
+
): NocturnalGateBlock | null {
|
|
322
|
+
return (
|
|
323
|
+
[...gateBlocks]
|
|
324
|
+
.sort((left, right) => new Date(right.createdAt).getTime() - new Date(left.createdAt).getTime())
|
|
325
|
+
.find((gateBlock) => gateBlock.toolName === toolCall.toolName) ?? null
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
private _isRiskPath(normalizedPath: string): boolean {
|
|
330
|
+
try {
|
|
331
|
+
const profilePath = path.join(this.workspaceDir, 'PROFILE.json');
|
|
332
|
+
const riskPaths =
|
|
333
|
+
fs.existsSync(profilePath)
|
|
334
|
+
? (((JSON.parse(fs.readFileSync(profilePath, 'utf-8')) as { risk_paths?: unknown }).risk_paths as string[] | undefined) ?? [])
|
|
335
|
+
: [];
|
|
336
|
+
return isRisky(normalizedPath, riskPaths);
|
|
337
|
+
} catch {
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
private _safePlanStatus(): 'NONE' | 'DRAFT' | 'READY' | 'UNKNOWN' {
|
|
343
|
+
try {
|
|
344
|
+
const status = getPlanStatus(this.workspaceDir);
|
|
345
|
+
if (status === 'READY') return 'READY';
|
|
346
|
+
if (status === 'DRAFT') return 'DRAFT';
|
|
347
|
+
if (status === '') return 'NONE';
|
|
348
|
+
return 'UNKNOWN';
|
|
349
|
+
} catch {
|
|
350
|
+
return 'UNKNOWN';
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
355
|
+
private _estimateLineChanges(toolCall: NocturnalToolCall): number {
|
|
356
|
+
if (toolCall.toolName === 'edit' || toolCall.toolName === 'write') {
|
|
357
|
+
return 20;
|
|
358
|
+
}
|
|
359
|
+
return 0;
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
363
|
+
private _inferBashRisk(toolCall: NocturnalToolCall): 'safe' | 'normal' | 'dangerous' | 'unknown' {
|
|
364
|
+
if (toolCall.toolName !== 'bash' && toolCall.toolName !== 'run_shell_command') {
|
|
365
|
+
return 'unknown';
|
|
366
|
+
}
|
|
367
|
+
const errorText = `${toolCall.errorType ?? ''} ${toolCall.errorMessage ?? ''}`;
|
|
368
|
+
if (/\brm\s+-rf\b|\bchmod\b|\bchown\b|>\s*\/dev\//.test(errorText)) {
|
|
369
|
+
return 'dangerous';
|
|
370
|
+
}
|
|
371
|
+
return toolCall.outcome === 'success' ? 'safe' : 'normal';
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
375
|
+
private _scoreEvaluation(
|
|
376
|
+
sample: ReplaySample,
|
|
377
|
+
result: RuleHostResult,
|
|
378
|
+
): { passed: boolean; reason?: string; decision: string } {
|
|
379
|
+
switch (sample.classification) {
|
|
380
|
+
case 'pain-negative':
|
|
381
|
+
return {
|
|
382
|
+
passed: result.decision === 'block' || result.decision === 'requireApproval',
|
|
383
|
+
reason:
|
|
384
|
+
result.decision === 'block' || result.decision === 'requireApproval'
|
|
385
|
+
? undefined
|
|
386
|
+
: `Expected block/requireApproval but received ${result.decision}.`,
|
|
387
|
+
decision: result.decision,
|
|
388
|
+
};
|
|
389
|
+
case 'success-positive':
|
|
390
|
+
return {
|
|
391
|
+
passed: result.decision === 'allow' || !result.matched,
|
|
392
|
+
reason:
|
|
393
|
+
result.decision === 'allow' || !result.matched
|
|
394
|
+
? undefined
|
|
395
|
+
: `Expected allow/no-match but received ${result.decision}.`,
|
|
396
|
+
decision: result.decision,
|
|
397
|
+
};
|
|
398
|
+
case 'principle-anchor':
|
|
399
|
+
return {
|
|
400
|
+
passed: result.decision !== 'block',
|
|
401
|
+
reason:
|
|
402
|
+
result.decision !== 'block'
|
|
403
|
+
? undefined
|
|
404
|
+
: 'Principle-anchor sample should not regress to a hard block.',
|
|
405
|
+
decision: result.decision,
|
|
406
|
+
};
|
|
407
|
+
default:
|
|
408
|
+
return {
|
|
409
|
+
passed: false,
|
|
410
|
+
reason: 'Unknown replay classification.',
|
|
411
|
+
decision: result.decision,
|
|
412
|
+
};
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
private _buildReport(
|
|
417
|
+
implementationId: string,
|
|
418
|
+
results: ReplayResult[]
|
|
419
|
+
): ReplayReport {
|
|
420
|
+
const painNegative = results.filter((result) => result.classification === 'pain-negative');
|
|
421
|
+
const successPositive = results.filter((result) => result.classification === 'success-positive');
|
|
422
|
+
const principleAnchor = results.filter((result) => result.classification === 'principle-anchor');
|
|
423
|
+
|
|
424
|
+
const toSummary = (details: ReplayResult[]): ClassificationSummary => ({
|
|
425
|
+
total: details.length,
|
|
426
|
+
passed: details.filter((result) => result.passed).length,
|
|
427
|
+
failed: details.filter((result) => !result.passed).length,
|
|
428
|
+
details,
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
const painSummary = toSummary(painNegative);
|
|
432
|
+
const successSummary = toSummary(successPositive);
|
|
433
|
+
const anchorSummary = toSummary(principleAnchor);
|
|
434
|
+
const blockers: string[] = [];
|
|
435
|
+
|
|
436
|
+
for (const leak of painSummary.details.filter((result) => !result.passed)) {
|
|
437
|
+
blockers.push(
|
|
438
|
+
`PAIN-NEGATIVE LEAK: Sample ${leak.sampleFingerprint} was not blocked. ${leak.reason ?? ''}`
|
|
439
|
+
);
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
for (const violation of anchorSummary.details.filter((result) => !result.passed)) {
|
|
443
|
+
blockers.push(
|
|
444
|
+
`PRINCIPLE-ANCHOR VIOLATION: Sample ${violation.sampleFingerprint} did not adhere. ${violation.reason ?? ''}`
|
|
445
|
+
);
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
for (const falsePositive of successSummary.details.filter((result) => !result.passed)) {
|
|
449
|
+
blockers.push(
|
|
450
|
+
`FALSE POSITIVE: Sample ${falsePositive.sampleFingerprint} was incorrectly blocked. ${falsePositive.reason ?? ''}`
|
|
451
|
+
);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
return {
|
|
455
|
+
overallDecision: this._determineDecision(painSummary, successSummary, anchorSummary),
|
|
456
|
+
replayResults: {
|
|
457
|
+
painNegative: painSummary,
|
|
458
|
+
successPositive: successSummary,
|
|
459
|
+
principleAnchor: anchorSummary,
|
|
460
|
+
},
|
|
461
|
+
blockers,
|
|
462
|
+
generatedAt: new Date().toISOString(),
|
|
463
|
+
implementationId,
|
|
464
|
+
sampleFingerprints: results.map((result) => result.sampleFingerprint),
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
469
|
+
private _determineDecision(
|
|
470
|
+
pain: ClassificationSummary,
|
|
471
|
+
success: ClassificationSummary,
|
|
472
|
+
anchor: ClassificationSummary
|
|
473
|
+
): 'pass' | 'fail' | 'needs-review' {
|
|
474
|
+
if (pain.failed > 0) return 'fail';
|
|
475
|
+
if (anchor.failed > 0) return 'fail';
|
|
476
|
+
if (success.failed > 0) return 'needs-review';
|
|
477
|
+
return 'pass';
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
private _persistReport(report: ReplayReport): void {
|
|
481
|
+
const reportDir = path.join(
|
|
482
|
+
getImplementationAssetRoot(this.stateDir, report.implementationId),
|
|
483
|
+
'replays'
|
|
484
|
+
);
|
|
485
|
+
|
|
486
|
+
if (!fs.existsSync(reportDir)) {
|
|
487
|
+
fs.mkdirSync(reportDir, { recursive: true });
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|
491
|
+
const reportPath = path.join(reportDir, `${timestamp}.json`);
|
|
492
|
+
|
|
493
|
+
withLock(reportPath, () => {
|
|
494
|
+
fs.writeFileSync(reportPath, JSON.stringify(report, null, 2), 'utf-8');
|
|
495
|
+
});
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// eslint-disable-next-line @typescript-eslint/class-methods-use-this -- Reason: Private helper doesn't use instance state
|
|
499
|
+
private _deriveExpectedOutcome(
|
|
500
|
+
record: NocturnalDatasetRecord,
|
|
501
|
+
): ReplaySample['expectedOutcome'] {
|
|
502
|
+
switch (record.classification) {
|
|
503
|
+
case 'pain-negative':
|
|
504
|
+
return { shouldBlock: true };
|
|
505
|
+
case 'success-positive':
|
|
506
|
+
return { shouldPass: true };
|
|
507
|
+
case 'principle-anchor':
|
|
508
|
+
return { expectedPrinciple: record.principleId };
|
|
509
|
+
default:
|
|
510
|
+
return {};
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
export function formatReplayReport(report: ReplayReport): string {
|
|
516
|
+
const decisionEmoji =
|
|
517
|
+
report.overallDecision === 'pass'
|
|
518
|
+
? 'PASS'
|
|
519
|
+
: report.overallDecision === 'fail'
|
|
520
|
+
? 'FAIL'
|
|
521
|
+
: 'NEEDS-REVIEW';
|
|
522
|
+
|
|
523
|
+
let output = '';
|
|
524
|
+
output += '\nReplay Evaluation Report\n';
|
|
525
|
+
output += `${'='.repeat(50)}\n`;
|
|
526
|
+
output += `Implementation: ${report.implementationId}\n`;
|
|
527
|
+
output += `Generated At: ${report.generatedAt}\n`;
|
|
528
|
+
output += `Overall Decision: [${decisionEmoji}]\n\n`;
|
|
529
|
+
|
|
530
|
+
const formatSection = (
|
|
531
|
+
label: string,
|
|
532
|
+
summary: ClassificationSummary
|
|
533
|
+
) => {
|
|
534
|
+
const rate = summary.total > 0
|
|
535
|
+
? ((summary.passed / summary.total) * 100).toFixed(1)
|
|
536
|
+
: 'N/A';
|
|
537
|
+
let section = ` ${label}:\n`;
|
|
538
|
+
section += ` Total: ${summary.total} | Passed: ${summary.passed} | Failed: ${summary.failed}\n`;
|
|
539
|
+
section += ` Pass Rate: ${rate}%\n`;
|
|
540
|
+
if (summary.failed > 0) {
|
|
541
|
+
section += ' Failures:\n';
|
|
542
|
+
for (const detail of summary.details.filter((item) => !item.passed)) {
|
|
543
|
+
section += ` - ${detail.sampleFingerprint}: ${detail.reason ?? detail.decision}\n`;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
return section;
|
|
547
|
+
};
|
|
548
|
+
|
|
549
|
+
output += formatSection('Pain-Negative Samples', report.replayResults.painNegative);
|
|
550
|
+
output += formatSection('Success-Positive Samples', report.replayResults.successPositive);
|
|
551
|
+
output += formatSection('Principle-Anchor Samples', report.replayResults.principleAnchor);
|
|
552
|
+
|
|
553
|
+
if (report.blockers.length > 0) {
|
|
554
|
+
output += '\nBlockers:\n';
|
|
555
|
+
for (const blocker of report.blockers) {
|
|
556
|
+
output += ` - ${blocker}\n`;
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
output += `${'='.repeat(50)}\n`;
|
|
561
|
+
return output;
|
|
562
|
+
}
|