recallnest 2.3.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +187 -0
- package/README.md +49 -38
- package/README_CN.md +46 -31
- package/assets/ui/app.js +1031 -0
- package/assets/ui/index.html +224 -0
- package/assets/ui/styles.css +781 -0
- package/config.json.example +4 -1
- package/docs/api-reference.md +1291 -0
- package/docs/architecture.md +168 -0
- package/docs/connector-spec.md +126 -0
- package/eval/cases-canary.json +58 -0
- package/eval/cases-scope-robustness.json +278 -0
- package/eval/cases.json +117 -20
- package/package.json +25 -4
- package/src/__tests__/access-tracker-novelty.test.ts +304 -0
- package/src/__tests__/activity-counter.test.ts +125 -0
- package/src/__tests__/adaptive-fidelity.test.ts +102 -0
- package/src/__tests__/admission-control.test.ts +206 -0
- package/src/__tests__/anchor-generator.test.ts +67 -0
- package/src/__tests__/api-server-security.test.ts +102 -0
- package/src/__tests__/auto-consolidation.test.ts +168 -0
- package/src/__tests__/auto-recall.test.ts +131 -0
- package/src/__tests__/babel-integration.test.ts +144 -0
- package/src/__tests__/batch-dedup.test.ts +65 -0
- package/src/__tests__/belief-history.test.ts +220 -0
- package/src/__tests__/canary-eval.test.ts +114 -0
- package/src/__tests__/capture-engine.test.ts +1328 -0
- package/src/__tests__/capture-heuristic.test.ts +171 -0
- package/src/__tests__/cascade-forget.test.ts +210 -0
- package/src/__tests__/category-min-scores.test.ts +116 -0
- package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
- package/src/__tests__/circuit-breaker.test.ts +99 -0
- package/src/__tests__/cluster-consolidation.test.ts +945 -0
- package/src/__tests__/confidence-metadata.test.ts +125 -0
- package/src/__tests__/confidence-tracker.test.ts +261 -0
- package/src/__tests__/conflict-advisor.test.ts +206 -0
- package/src/__tests__/conflict-escalation.test.ts +149 -0
- package/src/__tests__/conflict-lifecycle.test.ts +83 -0
- package/src/__tests__/conflict-output.test.ts +85 -0
- package/src/__tests__/conflict-store.test.ts +418 -0
- package/src/__tests__/connector-v1.test.ts +181 -0
- package/src/__tests__/consolidation-engine.test.ts +430 -0
- package/src/__tests__/constructive-retrieval.test.ts +509 -0
- package/src/__tests__/context-collapse-renderer.test.ts +214 -0
- package/src/__tests__/context-composer-pins.test.ts +64 -0
- package/src/__tests__/context-composer-task-selection.test.ts +24 -0
- package/src/__tests__/context-reconstructor.test.ts +65 -0
- package/src/__tests__/context-renderer.test.ts +91 -0
- package/src/__tests__/continuity-eval.test.ts +275 -0
- package/src/__tests__/continuity-schema.test.ts +97 -0
- package/src/__tests__/contracts.test.ts +262 -0
- package/src/__tests__/conversation-importer.test.ts +692 -0
- package/src/__tests__/core-recall-eval.test.ts +235 -0
- package/src/__tests__/core-summary.test.ts +109 -0
- package/src/__tests__/data-checkup.test.ts +276 -0
- package/src/__tests__/decay-exemption.test.ts +129 -0
- package/src/__tests__/dedup-actions.test.ts +58 -0
- package/src/__tests__/dedup-guard.test.ts +39 -0
- package/src/__tests__/deterministic-id.test.ts +39 -0
- package/src/__tests__/distill-lock.test.ts +305 -0
- package/src/__tests__/distill-session-argshape.test.ts +79 -0
- package/src/__tests__/doctor.test.ts +319 -0
- package/src/__tests__/dream-pipeline.test.ts +780 -0
- package/src/__tests__/drill-down.test.ts +108 -0
- package/src/__tests__/embedder.test.ts +122 -0
- package/src/__tests__/emotion-decay.test.ts +46 -0
- package/src/__tests__/emotion-detector.test.ts +56 -0
- package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
- package/src/__tests__/emotion-retrieval.test.ts +56 -0
- package/src/__tests__/entity-resolver.test.ts +58 -0
- package/src/__tests__/env-config.test.ts +151 -0
- package/src/__tests__/error-signature-boost.test.ts +87 -0
- package/src/__tests__/error-signature.test.ts +95 -0
- package/src/__tests__/error-taxonomy.test.ts +129 -0
- package/src/__tests__/ethics-layer.test.ts +476 -0
- package/src/__tests__/event-segmenter.test.ts +153 -0
- package/src/__tests__/feature-flags-integration.test.ts +54 -0
- package/src/__tests__/frequency-tracker.test.ts +113 -0
- package/src/__tests__/freshness.test.ts +186 -0
- package/src/__tests__/graph-export.test.ts +489 -0
- package/src/__tests__/hotness-blend.test.ts +283 -0
- package/src/__tests__/ingest-chunking.test.ts +44 -0
- package/src/__tests__/ingest.test.ts +360 -0
- package/src/__tests__/interference-detector.test.ts +203 -0
- package/src/__tests__/kg-extractor.test.ts +192 -0
- package/src/__tests__/kg-store.test.ts +300 -0
- package/src/__tests__/llm-client-pivot.test.ts +628 -0
- package/src/__tests__/llm-consolidation.test.ts +226 -0
- package/src/__tests__/maintenance-locks.test.ts +189 -0
- package/src/__tests__/mcp-efficiency.test.ts +288 -0
- package/src/__tests__/mcp-related-scopes.test.ts +151 -0
- package/src/__tests__/memory-boundaries.test.ts +282 -0
- package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
- package/src/__tests__/memory-evolution.test.ts +271 -0
- package/src/__tests__/memory-health-rebalance.test.ts +144 -0
- package/src/__tests__/memory-lint.test.ts +421 -0
- package/src/__tests__/memory-output.test.ts +215 -0
- package/src/__tests__/memory-promotion.test.ts +315 -0
- package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
- package/src/__tests__/multi-vector.test.ts +242 -0
- package/src/__tests__/narrative-architecture.test.ts +456 -0
- package/src/__tests__/noise-filter.test.ts +205 -0
- package/src/__tests__/noise-prototype-bank.test.ts +280 -0
- package/src/__tests__/obsidian-connector.test.ts +363 -0
- package/src/__tests__/omitted-reporting.test.ts +98 -0
- package/src/__tests__/pending-review.test.ts +77 -0
- package/src/__tests__/pii-redact.test.ts +333 -0
- package/src/__tests__/pivot-apply.test.ts +373 -0
- package/src/__tests__/pivot-distill-supervisor.test.ts +457 -0
- package/src/__tests__/pivot-distill.test.ts +1178 -0
- package/src/__tests__/post-retrieval-filter.test.ts +178 -0
- package/src/__tests__/ppr-traversal.test.ts +256 -0
- package/src/__tests__/prediction-engine.test.ts +349 -0
- package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
- package/src/__tests__/preference-guard-smoke.test.ts +211 -0
- package/src/__tests__/preference-matcher.test.ts +282 -0
- package/src/__tests__/preference-slots.test.ts +205 -0
- package/src/__tests__/prospective-memory.test.ts +283 -0
- package/src/__tests__/pull-from-macbook-sync-only.test.ts +679 -0
- package/src/__tests__/query-entity-detector.test.ts +171 -0
- package/src/__tests__/query-expander-alias.test.ts +146 -0
- package/src/__tests__/query-expander.test.ts +34 -0
- package/src/__tests__/recall-gate.test.ts +266 -0
- package/src/__tests__/recall-governor.test.ts +227 -0
- package/src/__tests__/recall-mode.test.ts +64 -0
- package/src/__tests__/resolve-db-path.test.ts +130 -0
- package/src/__tests__/resolve-tier.test.ts +64 -0
- package/src/__tests__/result-synthesizer.test.ts +198 -0
- package/src/__tests__/resume-context.test.ts +4898 -0
- package/src/__tests__/retrieval-eval.test.ts +163 -0
- package/src/__tests__/retrieval-trace.test.ts +213 -0
- package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
- package/src/__tests__/retriever-lazy-init-dispatch.test.ts +123 -0
- package/src/__tests__/retroactive-boost.test.ts +223 -0
- package/src/__tests__/salience-scoring.test.ts +106 -0
- package/src/__tests__/scope-inventory.test.ts +200 -0
- package/src/__tests__/scope-policy.test.ts +69 -0
- package/src/__tests__/scope-suggester.test.ts +80 -0
- package/src/__tests__/session-checkpoint.test.ts +291 -0
- package/src/__tests__/session-distiller.test.ts +640 -0
- package/src/__tests__/session-gc.test.ts +215 -0
- package/src/__tests__/skill-engine.test.ts +552 -0
- package/src/__tests__/skill-promotion.test.ts +654 -0
- package/src/__tests__/skill-verifier.test.ts +122 -0
- package/src/__tests__/source-heartbeat.test.ts +250 -0
- package/src/__tests__/stable-context-length-contract.test.ts +62 -0
- package/src/__tests__/store-patch-metadata.test.ts +205 -0
- package/src/__tests__/store-read-consistency.test.ts +122 -0
- package/src/__tests__/store-schema-migration.test.ts +87 -0
- package/src/__tests__/store-scope-audit.test.ts +81 -0
- package/src/__tests__/store-upsert.test.ts +201 -0
- package/src/__tests__/supersede-chain.test.ts +154 -0
- package/src/__tests__/temporal-parser.test.ts +241 -0
- package/src/__tests__/temporal-validity.test.ts +160 -0
- package/src/__tests__/term-registry-terminal-roster.test.ts +127 -0
- package/src/__tests__/term-registry.test.ts +121 -0
- package/src/__tests__/tool-error-surfacing.test.ts +98 -0
- package/src/__tests__/tool-output-compressor.test.ts +160 -0
- package/src/__tests__/topic-tag.test.ts +113 -0
- package/src/__tests__/trusted-memory.test.ts +839 -0
- package/src/__tests__/ui-server-security.test.ts +93 -0
- package/src/__tests__/usage-tracker.test.ts +287 -0
- package/src/__tests__/version-manager.test.ts +213 -0
- package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
- package/src/__tests__/workflow-observation.test.ts +433 -0
- package/src/__tests__/write-verifier.test.ts +122 -0
- package/src/access-tracker.ts +90 -34
- package/src/activity-counter.ts +46 -18
- package/src/adaptive-retrieval.ts +11 -0
- package/src/aliases.ts +133 -0
- package/src/api-server.ts +36 -9
- package/src/audit-log.ts +4 -2
- package/src/auto-consolidation.ts +8 -3
- package/src/auto-gc.ts +85 -21
- package/src/auto-recall.ts +61 -1
- package/src/belief-history.ts +164 -0
- package/src/capture-engine.ts +142 -32
- package/src/cli.ts +268 -41
- package/src/confidence-tracker.ts +36 -18
- package/src/conflict-engine.ts +24 -3
- package/src/consolidation-engine.ts +255 -23
- package/src/context-composer-stable-selection.ts +7 -1
- package/src/context-composer.ts +8 -1
- package/src/data-checkup.ts +96 -7
- package/src/decay-engine.ts +59 -7
- package/src/distill-lock.ts +194 -15
- package/src/doctor.ts +13 -4
- package/src/dream-pipeline.ts +473 -41
- package/src/env-config.ts +118 -0
- package/src/error-signature.ts +153 -0
- package/src/error-taxonomy.ts +115 -0
- package/src/eval.ts +402 -14
- package/src/freshness.ts +192 -0
- package/src/http-request-policy.ts +148 -0
- package/src/idempotency.ts +8 -0
- package/src/ingest.ts +308 -12
- package/src/kg-extractor.ts +13 -3
- package/src/kg-store.ts +236 -44
- package/src/llm-client.ts +106 -11
- package/src/llm-consolidation.ts +2 -1
- package/src/mcp-server.ts +87 -1800
- package/src/mcp-tool-deps.ts +48 -0
- package/src/mcp-tools-advanced.ts +954 -0
- package/src/mcp-tools-core.ts +637 -0
- package/src/mcp-tools-governance.ts +475 -0
- package/src/memory-boundaries.ts +26 -3
- package/src/memory-evolution.ts +73 -1
- package/src/memory-health-rebalance.ts +4 -0
- package/src/memory-lint.ts +183 -10
- package/src/memory-output.ts +129 -4
- package/src/memory-promotion.ts +246 -0
- package/src/memory-schema.ts +25 -2
- package/src/memory-store-port.ts +46 -0
- package/src/multi-vector.ts +2 -1
- package/src/narrative-schema.ts +2 -1
- package/src/noise-filter.ts +57 -6
- package/src/noise-prototype-bank.ts +408 -0
- package/src/pii-detector.ts +207 -5
- package/src/pivot-apply.ts +399 -0
- package/src/preference-matcher.ts +37 -3
- package/src/query-expander.ts +90 -9
- package/src/recall-gate.ts +305 -0
- package/src/result-synthesizer.ts +2 -1
- package/src/retention-policy.ts +2 -1
- package/src/retrieval-profiles.ts +46 -4
- package/src/retriever.ts +354 -157
- package/src/retroactive-boost.ts +6 -2
- package/src/runtime-config.ts +78 -6
- package/src/scope-suggester.ts +113 -0
- package/src/session-distiller.ts +23 -2
- package/src/session-engine.ts +2 -1
- package/src/session-schema.ts +1 -0
- package/src/session-store.ts +5 -0
- package/src/skill-engine.ts +78 -0
- package/src/skill-promotion.ts +304 -48
- package/src/skill-schema.ts +16 -3
- package/src/skill-verifier.ts +152 -0
- package/src/store.ts +321 -57
- package/src/term-registry.ts +35 -0
- package/src/tool-output-compressor.ts +3 -1
- package/src/ui-server.ts +9 -1
- package/src/usage-tracker.ts +242 -0
- package/src/version-manager.ts +3 -2
- package/src/version.ts +19 -0
- package/src/workflow-observation-engine.ts +2 -1
- package/src/workflow-observation-schema.ts +2 -1
- package/src/workflow-observation-store.ts +6 -2
- package/.env +0 -4
- package/CLAUDE.md +0 -46
- package/config.json +0 -55
- package/daily-recall.sh +0 -110
- package/logs/daily-recall.error.log +0 -0
- package/logs/daily-recall.log +0 -60
- package/logs/ingest-2026-04-11.log +0 -110
- package/logs/ingest-2026-04-12.log +0 -90
- package/logs/ingest-2026-04-13.log +0 -87
- package/logs/launchagent-stderr.log +0 -0
- package/logs/launchagent-stdout.log +0 -0
|
@@ -17,12 +17,15 @@
|
|
|
17
17
|
* LLM-free — deterministic clustering and merge only.
|
|
18
18
|
*/
|
|
19
19
|
|
|
20
|
-
import type { MemoryEntry, MemorySearchResult
|
|
20
|
+
import type { MemoryEntry, MemorySearchResult } from "./store.js";
|
|
21
|
+
import { deterministicId } from "./store.js";
|
|
22
|
+
import type { MemoryStorePort } from "./memory-store-port.js";
|
|
21
23
|
import { createVersionGroup } from "./version-manager.js";
|
|
22
24
|
import { isActiveMemory, parseEvolution, buildSupersedeMetadata, buildConsolidatedMetadata, patchEvolution } from "./memory-evolution.js";
|
|
23
25
|
import { cosineSimilarity } from "./multi-vector.js";
|
|
24
26
|
import type { LLMClient } from "./llm-client.js";
|
|
25
27
|
import type { Embedder } from "./embedder.js";
|
|
28
|
+
import { logWarn } from "./stderr-log.js";
|
|
26
29
|
|
|
27
30
|
// ---------------------------------------------------------------------------
|
|
28
31
|
// Config & Types
|
|
@@ -35,12 +38,22 @@ export interface ConsolidationConfig {
|
|
|
35
38
|
mergeThreshold: number;
|
|
36
39
|
/** Maximum entries to scan per consolidation run (default 500) */
|
|
37
40
|
maxEntriesPerRun: number;
|
|
41
|
+
/**
|
|
42
|
+
* KG evidence: triple-set Jaccard at/above this merges a grey-zone pair
|
|
43
|
+
* (vector sim between clusterThreshold and mergeThreshold). Default 0.5 —
|
|
44
|
+
* provisional until calibrated on production data (plan Q3).
|
|
45
|
+
*/
|
|
46
|
+
tripleJaccardThreshold?: number;
|
|
47
|
+
/** KG evidence: both sides need at least this many triples to qualify (default 2) */
|
|
48
|
+
minTriplesForEvidence?: number;
|
|
38
49
|
}
|
|
39
50
|
|
|
40
51
|
export const DEFAULT_CONSOLIDATION_CONFIG: ConsolidationConfig = {
|
|
41
52
|
clusterThreshold: 0.82,
|
|
42
53
|
mergeThreshold: 0.92,
|
|
43
54
|
maxEntriesPerRun: 500,
|
|
55
|
+
tripleJaccardThreshold: 0.5,
|
|
56
|
+
minTriplesForEvidence: 2,
|
|
44
57
|
};
|
|
45
58
|
|
|
46
59
|
export interface ConflictEvent {
|
|
@@ -54,10 +67,52 @@ export interface ConsolidationResult {
|
|
|
54
67
|
clustersFound: number;
|
|
55
68
|
mergedCount: number;
|
|
56
69
|
relationsAdded: number;
|
|
70
|
+
/** Of mergedCount, how many were below mergeThreshold and merged on KG triple evidence */
|
|
71
|
+
tripleEvidenceMerges: number;
|
|
57
72
|
conflictsDetected: ConflictEvent[];
|
|
58
73
|
scope: string;
|
|
59
74
|
}
|
|
60
75
|
|
|
76
|
+
// ---------------------------------------------------------------------------
|
|
77
|
+
// KG evidence source (duck-typed subset of KGStore)
|
|
78
|
+
// ---------------------------------------------------------------------------
|
|
79
|
+
|
|
80
|
+
/** The two triple fields consolidation needs — structurally satisfied by KGTriple. */
|
|
81
|
+
export interface ConsolidationTripleEvidence {
|
|
82
|
+
id: string;
|
|
83
|
+
mention_count: number;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface ConsolidationKGSource {
|
|
87
|
+
getTriplesBySourceMemories(memoryIds: string[]): Promise<Map<string, ConsolidationTripleEvidence[]>>;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Jaccard overlap of two triple-id sets. Below `minSize` on either side the
|
|
92
|
+
* evidence is considered too weak and 0 is returned (a single shared triple
|
|
93
|
+
* must not merge two memories on its own).
|
|
94
|
+
*/
|
|
95
|
+
export function tripleJaccard(
|
|
96
|
+
a: ReadonlySet<string> | undefined,
|
|
97
|
+
b: ReadonlySet<string> | undefined,
|
|
98
|
+
minSize = 2,
|
|
99
|
+
): number {
|
|
100
|
+
if (!a || !b || a.size < minSize || b.size < minSize) return 0;
|
|
101
|
+
let inter = 0;
|
|
102
|
+
for (const x of a) if (b.has(x)) inter++;
|
|
103
|
+
return inter / (a.size + b.size - inter);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Mention-frequency boost in canonical selection: tie-breaker ONLY, never an
|
|
108
|
+
* override of a clear canonicalScore gap (an entry with one hot triple must
|
|
109
|
+
* not demote a substantially more important memory to consolidated status).
|
|
110
|
+
* α=0.06 keeps the boost differentiating (m=1 → 1.042, m=9 → 1.138) while the
|
|
111
|
+
* 1.1 cap bounds the maximum flip at ~10% of the base score — effectively ties.
|
|
112
|
+
*/
|
|
113
|
+
const MENTION_ALPHA = 0.06;
|
|
114
|
+
const MENTION_BOOST_CAP = 1.1;
|
|
115
|
+
|
|
61
116
|
// ---------------------------------------------------------------------------
|
|
62
117
|
// Helpers
|
|
63
118
|
// ---------------------------------------------------------------------------
|
|
@@ -108,7 +163,7 @@ export function detectHeuristicContradiction(textA: string, textB: string): bool
|
|
|
108
163
|
// Store interface (duck-typed to avoid hard dependency)
|
|
109
164
|
// ---------------------------------------------------------------------------
|
|
110
165
|
|
|
111
|
-
type ConsolidationStore = Pick<
|
|
166
|
+
type ConsolidationStore = Pick<MemoryStorePort, "list" | "getById" | "vectorSearch" | "update">;
|
|
112
167
|
|
|
113
168
|
// ---------------------------------------------------------------------------
|
|
114
169
|
// Engine
|
|
@@ -118,17 +173,25 @@ export class ConsolidationEngine {
|
|
|
118
173
|
constructor(
|
|
119
174
|
private store: ConsolidationStore,
|
|
120
175
|
private config: ConsolidationConfig = DEFAULT_CONSOLIDATION_CONFIG,
|
|
176
|
+
/** Optional KG evidence source — absent = pure vector behavior, unchanged */
|
|
177
|
+
private kgSource: ConsolidationKGSource | null = null,
|
|
121
178
|
) {}
|
|
122
179
|
|
|
123
180
|
async run(scope: string): Promise<ConsolidationResult> {
|
|
124
|
-
const {
|
|
181
|
+
const {
|
|
182
|
+
clusterThreshold,
|
|
183
|
+
mergeThreshold,
|
|
184
|
+
maxEntriesPerRun,
|
|
185
|
+
tripleJaccardThreshold = 0.5,
|
|
186
|
+
minTriplesForEvidence = 2,
|
|
187
|
+
} = this.config;
|
|
125
188
|
|
|
126
189
|
// 1. Fetch entries in scope
|
|
127
190
|
const entries = await this.store.list([scope], undefined, maxEntriesPerRun, 0);
|
|
128
191
|
const active = entries.filter(isActive);
|
|
129
192
|
|
|
130
193
|
if (active.length === 0) {
|
|
131
|
-
return { originalCount: 0, clustersFound: 0, mergedCount: 0, relationsAdded: 0, conflictsDetected: [], scope };
|
|
194
|
+
return { originalCount: 0, clustersFound: 0, mergedCount: 0, relationsAdded: 0, tripleEvidenceMerges: 0, conflictsDetected: [], scope };
|
|
132
195
|
}
|
|
133
196
|
|
|
134
197
|
// 2. Group by category
|
|
@@ -142,6 +205,7 @@ export class ConsolidationEngine {
|
|
|
142
205
|
let clustersFound = 0;
|
|
143
206
|
let mergedCount = 0;
|
|
144
207
|
let relationsAdded = 0;
|
|
208
|
+
let tripleEvidenceMerges = 0;
|
|
145
209
|
const conflictsDetected: ConflictEvent[] = [];
|
|
146
210
|
|
|
147
211
|
// 3. Cluster within each category
|
|
@@ -161,8 +225,18 @@ export class ConsolidationEngine {
|
|
|
161
225
|
full.vector, 10, clusterThreshold, [scope],
|
|
162
226
|
);
|
|
163
227
|
|
|
228
|
+
// Seeds are active-only (above), but vectorSearch filters by scope alone, so
|
|
229
|
+
// superseded belief-history rows surface here too. A rephrased belief sits well
|
|
230
|
+
// above mergeThreshold from its own archived version, and canonicalScore has no
|
|
231
|
+
// recency term — importance × access count are both inherited by the history row,
|
|
232
|
+
// so the two tie and the "weaker" one is picked arbitrarily. Lose that coin flip
|
|
233
|
+
// and the live belief gets marked consolidated, dropping out of default retrieval
|
|
234
|
+
// while the abandoned version stands in as canonical.
|
|
164
235
|
const members = similar.filter(
|
|
165
|
-
s => s.entry.id !== entry.id
|
|
236
|
+
s => s.entry.id !== entry.id
|
|
237
|
+
&& !clustered.has(s.entry.id)
|
|
238
|
+
&& s.entry.category === entry.category
|
|
239
|
+
&& isActive(s.entry),
|
|
166
240
|
);
|
|
167
241
|
|
|
168
242
|
if (members.length === 0) continue;
|
|
@@ -183,8 +257,27 @@ export class ConsolidationEngine {
|
|
|
183
257
|
}
|
|
184
258
|
if (memberEntries.length < 2) continue;
|
|
185
259
|
|
|
186
|
-
//
|
|
187
|
-
|
|
260
|
+
// KG triple evidence for this cluster (single batch query; best-effort —
|
|
261
|
+
// on failure consolidation proceeds vector-only)
|
|
262
|
+
const tripleIdsByMember = new Map<string, Set<string>>();
|
|
263
|
+
const maxMentionByMember = new Map<string, number>();
|
|
264
|
+
if (this.kgSource) {
|
|
265
|
+
try {
|
|
266
|
+
const triplesByMember = await this.kgSource.getTriplesBySourceMemories(memberEntries.map(e => e.id));
|
|
267
|
+
for (const [mid, triples] of triplesByMember) {
|
|
268
|
+
tripleIdsByMember.set(mid, new Set(triples.map(t => t.id)));
|
|
269
|
+
maxMentionByMember.set(mid, triples.reduce((m, t) => Math.max(m, t.mention_count), 0));
|
|
270
|
+
}
|
|
271
|
+
} catch { /* KG evidence is best-effort */ }
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// Determine canonical: highest canonicalScore, frequency-boosted when
|
|
275
|
+
// KG evidence exists (memories carrying often-mentioned facts win ties)
|
|
276
|
+
const mentionBoost = (e: MemoryEntry) =>
|
|
277
|
+
Math.min(MENTION_BOOST_CAP, 1 + MENTION_ALPHA * Math.log((maxMentionByMember.get(e.id) ?? 0) + 1));
|
|
278
|
+
const sorted = [...memberEntries].sort(
|
|
279
|
+
(a, b) => canonicalScore(b) * mentionBoost(b) - canonicalScore(a) * mentionBoost(a),
|
|
280
|
+
);
|
|
188
281
|
const canonical = sorted[0];
|
|
189
282
|
|
|
190
283
|
if (!canonical.vector?.length) continue;
|
|
@@ -199,7 +292,16 @@ export class ConsolidationEngine {
|
|
|
199
292
|
for (const member of sorted.slice(1)) {
|
|
200
293
|
const sim = scoreMap.get(member.id) ?? clusterThreshold;
|
|
201
294
|
|
|
202
|
-
|
|
295
|
+
// Second evidence source: two memories whose extracted fact sets
|
|
296
|
+
// overlap heavily are duplicates even when vector sim sits in the
|
|
297
|
+
// grey zone (the "pairwise near-duplicates" the 0.92 bar misses).
|
|
298
|
+
const jaccard = tripleJaccard(
|
|
299
|
+
tripleIdsByMember.get(canonical.id),
|
|
300
|
+
tripleIdsByMember.get(member.id),
|
|
301
|
+
minTriplesForEvidence,
|
|
302
|
+
);
|
|
303
|
+
|
|
304
|
+
if (sim >= mergeThreshold || jaccard >= tripleJaccardThreshold) {
|
|
203
305
|
// Tier 3.3: Version coexistence — both stay but grouped.
|
|
204
306
|
await createVersionGroup(this.store, canonical, member, scope);
|
|
205
307
|
// C-1: Also mark the weaker entry as consolidated via evolution metadata
|
|
@@ -214,6 +316,7 @@ export class ConsolidationEngine {
|
|
|
214
316
|
await this.store.update(canonical.id, { metadata: updatedCanon }, [scope]);
|
|
215
317
|
}
|
|
216
318
|
mergedCount++;
|
|
319
|
+
if (sim < mergeThreshold) tripleEvidenceMerges++;
|
|
217
320
|
} else {
|
|
218
321
|
// Link: add clustered_with relation
|
|
219
322
|
const memberMeta = parseMetadata(member);
|
|
@@ -245,7 +348,7 @@ export class ConsolidationEngine {
|
|
|
245
348
|
}
|
|
246
349
|
}
|
|
247
350
|
|
|
248
|
-
return { originalCount: active.length, clustersFound, mergedCount, relationsAdded, conflictsDetected, scope };
|
|
351
|
+
return { originalCount: active.length, clustersFound, mergedCount, relationsAdded, tripleEvidenceMerges, conflictsDetected, scope };
|
|
249
352
|
}
|
|
250
353
|
}
|
|
251
354
|
|
|
@@ -262,6 +365,14 @@ export interface ClusterConsolidationResult {
|
|
|
262
365
|
entriesLinked: number;
|
|
263
366
|
/** CC-9: Set when consecutive low-yield rounds trigger early termination */
|
|
264
367
|
earlyStop?: "diminishing_returns";
|
|
368
|
+
/**
|
|
369
|
+
* 本轮被吸收的 LLM 调用失败数(超时 / 网络 / 熔断抛出)。
|
|
370
|
+
*
|
|
371
|
+
* 计出来是为了让「吸收」不等于「静默」:吸收前,单个 cluster 撞 15s 超时会把整个
|
|
372
|
+
* scope 的 dream 炸掉(2026-08-09 memory 轮次连挂两次的根因);吸收后若不计数,
|
|
373
|
+
* 「LLM 全程不可用」和「今天没料可炼」会长得一模一样。
|
|
374
|
+
*/
|
|
375
|
+
llmFailures: number;
|
|
265
376
|
}
|
|
266
377
|
|
|
267
378
|
/**
|
|
@@ -278,7 +389,7 @@ export async function clusterAndConsolidate(params: {
|
|
|
278
389
|
entries: MemoryEntry[];
|
|
279
390
|
embedder: Pick<Embedder, "embedPassage">;
|
|
280
391
|
llm: LLMClient;
|
|
281
|
-
store: Pick<
|
|
392
|
+
store: Pick<MemoryStorePort, "store" | "update">;
|
|
282
393
|
scope: string;
|
|
283
394
|
/** Minimum cluster size to trigger consolidation (default: 3) */
|
|
284
395
|
minClusterSize?: number;
|
|
@@ -306,6 +417,7 @@ export async function clusterAndConsolidate(params: {
|
|
|
306
417
|
insightsGenerated: 0,
|
|
307
418
|
patternsExtracted: 0,
|
|
308
419
|
entriesLinked: 0,
|
|
420
|
+
llmFailures: 0,
|
|
309
421
|
};
|
|
310
422
|
|
|
311
423
|
// Filter to active entries with vectors
|
|
@@ -370,8 +482,26 @@ export async function clusterAndConsolidate(params: {
|
|
|
370
482
|
}
|
|
371
483
|
|
|
372
484
|
// Generate insight via LLM from cluster member texts
|
|
485
|
+
//
|
|
486
|
+
// 单个 cluster 的 LLM 失败不该炸掉整个 scope。llm-client 的 chat() 在 catch 里是
|
|
487
|
+
// `throw err`(记完熔断计数后原样抛),所以 15s AbortController 超时会一路冒到
|
|
488
|
+
// runDream 之外 —— 2026-08-09 memory 轮次连挂两次正是这条路径:memory 是最大
|
|
489
|
+
// scope、cluster 最多,撞上一次超时的概率接近 1,且两次挂的位置不同(不是某对
|
|
490
|
+
// 内容有毒,是这一阶段本来就没有防护)。隔壁 llm-consolidation.ts:68 的同类调用
|
|
491
|
+
// 一直有 try/catch,本处缺的是同一层保护。
|
|
492
|
+
//
|
|
493
|
+
// 吸收成 null 而不是新开一条分支:generateL0 返回 null(熔断打开 / 空响应)本来
|
|
494
|
+
// 就是合法路径,下面 `if (!insight)` 已经按「零产出轮次」处理。失败另计 llmFailures,
|
|
495
|
+
// 避免吸收变静默。
|
|
373
496
|
const combinedText = cluster.map(m => m.text).join("\n---\n");
|
|
374
|
-
|
|
497
|
+
let insight: string | null;
|
|
498
|
+
try {
|
|
499
|
+
insight = await llm.generateL0(combinedText);
|
|
500
|
+
} catch (err) {
|
|
501
|
+
result.llmFailures++;
|
|
502
|
+
logWarn(`[WARN] cluster insight LLM 调用失败,本簇跳过: ${err instanceof Error ? err.message : String(err)}`);
|
|
503
|
+
insight = null;
|
|
504
|
+
}
|
|
375
505
|
|
|
376
506
|
if (!insight) {
|
|
377
507
|
// CC-9: No insight produced — this is a zero-yield round
|
|
@@ -400,8 +530,13 @@ export async function clusterAndConsolidate(params: {
|
|
|
400
530
|
// Embed the insight text
|
|
401
531
|
const insightVector = await params.embedder.embedPassage(insight);
|
|
402
532
|
|
|
403
|
-
// Store the insight as a new memory
|
|
533
|
+
// Store the insight as a new memory. P0-2/P1-2: derive a deterministic id from the
|
|
534
|
+
// (sorted) source member ids so re-running dream/consolidate on the same cluster
|
|
535
|
+
// upserts the same insight row instead of appending a near-duplicate each run
|
|
536
|
+
// (the highest-frequency dup-id source). Cross-run stable because sourceIds are stable.
|
|
537
|
+
const sortedSourceIds = cluster.map(m => m.id).slice().sort();
|
|
404
538
|
const insightEntry = await store.store({
|
|
539
|
+
id: deterministicId(scope, `cluster-insight:${sortedSourceIds.join(",")}`),
|
|
405
540
|
text: insight,
|
|
406
541
|
vector: insightVector,
|
|
407
542
|
category: bestCat,
|
|
@@ -439,11 +574,27 @@ export async function clusterAndConsolidate(params: {
|
|
|
439
574
|
|
|
440
575
|
// HP-5: Cross-memory pattern extraction
|
|
441
576
|
if (extractPatterns && cluster.length >= 3) {
|
|
442
|
-
|
|
577
|
+
// 同上:pattern 抽取失败也只丢这一簇的 pattern,不连坐已经写好的 insight。
|
|
578
|
+
let patternText: string | null;
|
|
579
|
+
try {
|
|
580
|
+
patternText = await llm.extractPattern(cluster.map(m => m.text));
|
|
581
|
+
} catch (err) {
|
|
582
|
+
result.llmFailures++;
|
|
583
|
+
logWarn(`[WARN] pattern 抽取 LLM 调用失败,本簇跳过: ${err instanceof Error ? err.message : String(err)}`);
|
|
584
|
+
patternText = null;
|
|
585
|
+
}
|
|
443
586
|
if (patternText) {
|
|
444
|
-
|
|
587
|
+
// 与同函数上方的 cluster insight 取同一个值:派生物继承源里最高的 importance,
|
|
588
|
+
// 不额外加成。原本这里是 maxImportance + 0.1,会让 LLM 抽出的 pattern 比它的
|
|
589
|
+
// 任何一条源记忆都重要——而 importance >= 0.95 在本仓库里是「人工 pin」的专用
|
|
590
|
+
// 语义(cli.ts pin 操作把值抬到 0.95;auto-gc 视其为永不归档、decay-engine 视其
|
|
591
|
+
// 为永不衰减)。于是任何一条源 >= 0.85 的 cluster,其派生 pattern 都会自动跨进
|
|
592
|
+
// 人类锚点频段。加成想表达的「pattern 比源更有价值」若要保留,应走独立通道
|
|
593
|
+
// (检索 boost / 独立 tier / 显式 flag),不该占用这个频段。
|
|
594
|
+
const patternImportance = maxImportance;
|
|
445
595
|
const patternVector = await params.embedder.embedPassage(patternText);
|
|
446
596
|
const patternEntry = await store.store({
|
|
597
|
+
id: deterministicId(scope, `cluster-pattern:${sortedSourceIds.join(",")}`),
|
|
447
598
|
text: patternText,
|
|
448
599
|
vector: patternVector,
|
|
449
600
|
category: "patterns",
|
|
@@ -492,7 +643,7 @@ export function formatConsolidationResult(result: ConsolidationResult): string {
|
|
|
492
643
|
`Consolidation complete for scope: ${result.scope}`,
|
|
493
644
|
`Scanned: ${result.originalCount} active entries`,
|
|
494
645
|
`Clusters found: ${result.clustersFound}`,
|
|
495
|
-
`Merged (versioned): ${result.mergedCount}`,
|
|
646
|
+
`Merged (versioned): ${result.mergedCount}${result.tripleEvidenceMerges > 0 ? ` (${result.tripleEvidenceMerges} on triple evidence)` : ""}`,
|
|
496
647
|
`Relations added: ${result.relationsAdded}`,
|
|
497
648
|
`Conflicts detected: ${result.conflictsDetected.length}`,
|
|
498
649
|
];
|
|
@@ -509,6 +660,89 @@ export function formatConsolidationResult(result: ConsolidationResult): string {
|
|
|
509
660
|
// LC-P2: Cluster-aware deduplication for retrieval results
|
|
510
661
|
// ---------------------------------------------------------------------------
|
|
511
662
|
|
|
663
|
+
/**
|
|
664
|
+
* Whether an entry is a consolidation derivative — a cluster insight or a
|
|
665
|
+
* cross-memory pattern the LLM wrote from other entries, rather than something
|
|
666
|
+
* captured from the outside world.
|
|
667
|
+
*
|
|
668
|
+
* Single source of truth for the two metadata flags: retrieval uses it to
|
|
669
|
+
* collapse a derivative with its sources, and the dream gather uses it to keep
|
|
670
|
+
* derivatives from being re-consolidated as if they were raw material.
|
|
671
|
+
*/
|
|
672
|
+
export function isDerivedInsight(metadata: string | undefined): boolean {
|
|
673
|
+
if (!metadata) return false;
|
|
674
|
+
try {
|
|
675
|
+
const meta = JSON.parse(metadata) as Record<string, unknown>;
|
|
676
|
+
return meta.cluster_insight === true || meta.cross_memory_pattern === true;
|
|
677
|
+
} catch {
|
|
678
|
+
return false;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
// ---------------------------------------------------------------------------
|
|
683
|
+
// Synthesis uptake (Artel archivist_metrics.synthesis_uptake_rate 的对应物)
|
|
684
|
+
// ---------------------------------------------------------------------------
|
|
685
|
+
|
|
686
|
+
export interface SynthesisUptakeStats {
|
|
687
|
+
/** Entries scanned across the store (may be capped). */
|
|
688
|
+
scanned: number;
|
|
689
|
+
/** Derived insights found (cluster_insight / cross_memory_pattern). */
|
|
690
|
+
derivedTotal: number;
|
|
691
|
+
/** Derived insights with accessCount > 0 — actually read after synthesis. */
|
|
692
|
+
derivedRead: number;
|
|
693
|
+
/** derivedRead / derivedTotal; null when no derived insights exist. */
|
|
694
|
+
uptakeRate: number | null;
|
|
695
|
+
/** True when the scan hit the cap before exhausting the store. */
|
|
696
|
+
truncated: boolean;
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/**
|
|
700
|
+
* Measure how many consolidation/dream products were ever read back.
|
|
701
|
+
* 把"升华产物没人用"从体感变成能报警的数字——uptake 长期为 0 说明
|
|
702
|
+
* 升华管线在产出无人消费的内容(或读打点断链)。
|
|
703
|
+
*/
|
|
704
|
+
export async function computeSynthesisUptake(
|
|
705
|
+
store: { listPage(opts: { limit?: number; offset?: number; includeVector?: boolean }): Promise<MemoryEntry[]> },
|
|
706
|
+
// 50K 覆盖当前 ~34K 生产库全量(2026-07-23 实测 cap 20K 只扫 57%,truncated 结论不完整);
|
|
707
|
+
// metadata-only 分页扫描秒级,memory_stats 是低频人工工具,扫全比抽样值钱。
|
|
708
|
+
scanCap = 50_000,
|
|
709
|
+
pageSize = 1_000,
|
|
710
|
+
): Promise<SynthesisUptakeStats> {
|
|
711
|
+
let scanned = 0;
|
|
712
|
+
let derivedTotal = 0;
|
|
713
|
+
let derivedRead = 0;
|
|
714
|
+
let offset = 0;
|
|
715
|
+
let truncated = false;
|
|
716
|
+
|
|
717
|
+
for (;;) {
|
|
718
|
+
const page = await store.listPage({ limit: pageSize, offset, includeVector: false });
|
|
719
|
+
if (page.length === 0) break;
|
|
720
|
+
for (const e of page) {
|
|
721
|
+
scanned++;
|
|
722
|
+
if (!isDerivedInsight(e.metadata)) continue;
|
|
723
|
+
derivedTotal++;
|
|
724
|
+
try {
|
|
725
|
+
const meta = JSON.parse(e.metadata || "{}") as Record<string, unknown>;
|
|
726
|
+
if (typeof meta.accessCount === "number" && meta.accessCount > 0) derivedRead++;
|
|
727
|
+
} catch { /* broken metadata → counts as unread */ }
|
|
728
|
+
}
|
|
729
|
+
offset += page.length;
|
|
730
|
+
if (page.length < pageSize) break;
|
|
731
|
+
if (offset >= scanCap) {
|
|
732
|
+
truncated = true;
|
|
733
|
+
break;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
return {
|
|
738
|
+
scanned,
|
|
739
|
+
derivedTotal,
|
|
740
|
+
derivedRead,
|
|
741
|
+
uptakeRate: derivedTotal > 0 ? derivedRead / derivedTotal : null,
|
|
742
|
+
truncated,
|
|
743
|
+
};
|
|
744
|
+
}
|
|
745
|
+
|
|
512
746
|
/**
|
|
513
747
|
* LC-P2: When a cluster insight and its source memories both appear in
|
|
514
748
|
* retrieval results, keep only the cluster insight (it subsumes the
|
|
@@ -523,15 +757,13 @@ export function deduplicateByClusterInsight<T extends { entry: MemoryEntry; scor
|
|
|
523
757
|
const coveredByInsight = new Set<string>();
|
|
524
758
|
|
|
525
759
|
for (const r of results) {
|
|
526
|
-
if (!r.entry.metadata) continue;
|
|
760
|
+
if (!isDerivedInsight(r.entry.metadata)) continue;
|
|
527
761
|
try {
|
|
528
|
-
const meta = JSON.parse(r.entry.metadata);
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
if (typeof id === "string") coveredByInsight.add(id);
|
|
534
|
-
}
|
|
762
|
+
const meta = JSON.parse(r.entry.metadata as string);
|
|
763
|
+
const sources: unknown[] = meta.evolution?.sourceMemories;
|
|
764
|
+
if (Array.isArray(sources)) {
|
|
765
|
+
for (const id of sources) {
|
|
766
|
+
if (typeof id === "string") coveredByInsight.add(id);
|
|
535
767
|
}
|
|
536
768
|
}
|
|
537
769
|
} catch { /* skip unparseable */ }
|
|
@@ -162,7 +162,13 @@ function formatStableResult(category: StableCategory, result: RetrievalResult):
|
|
|
162
162
|
// F1: Tag low-confidence memories in resume_context output
|
|
163
163
|
const conf = getConfidence(result.entry);
|
|
164
164
|
const tag = conf < 0.5 ? " [低置信]" : "";
|
|
165
|
-
|
|
165
|
+
// 220 是 stableContext 的 schema 上限(session-schema.ts:61),不是随手写的数:
|
|
166
|
+
// 这里曾是 230,比上限大 10,而 interleaveUnique(context-composer-stable.ts:199-206)
|
|
167
|
+
// 不再截一次 —— 于是任何 221~230 字符的行都会让 composeResumeContext 末尾的
|
|
168
|
+
// ResumeContextResponseSchema.parse(context-composer.ts:541)抛
|
|
169
|
+
// "stableContext must be at most 220 characters",整个 resume_context 调用失败。
|
|
170
|
+
// 改这个数之前先看 schema,两边必须一起动。
|
|
171
|
+
return cleanText(`${STABLE_CATEGORY_LABELS[category]}: ${text}${tag}`, 220);
|
|
166
172
|
}
|
|
167
173
|
|
|
168
174
|
export function selectStableResults(
|
package/src/context-composer.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { listPinAssets, type PinAsset } from "./memory-assets.js";
|
|
2
|
+
import * as envConfig from "./env-config.js";
|
|
2
3
|
import { buildStableContextSections } from "./context-composer-stable.js";
|
|
3
4
|
import {
|
|
4
5
|
buildAssociativeNestEntityFallbackQuery,
|
|
@@ -32,6 +33,8 @@ type ResumeCategory = "profile" | "preferences" | "entities" | "patterns" | "cas
|
|
|
32
33
|
|
|
33
34
|
interface ResumeRetriever {
|
|
34
35
|
retrieve(context: RetrievalContext): Promise<RetrievalResult[]>;
|
|
36
|
+
/** Optional: 影子记录 resume reconstruction 引用的记忆(injection-vs-use)。未实现则 no-op。 */
|
|
37
|
+
recordReconstructionUsage?(citedIds: string[]): Promise<void>;
|
|
35
38
|
}
|
|
36
39
|
|
|
37
40
|
interface CheckpointLookup {
|
|
@@ -467,7 +470,7 @@ export async function composeResumeContext(
|
|
|
467
470
|
let reconstructedContext: string | undefined;
|
|
468
471
|
let reconstructionConfidence: number | undefined;
|
|
469
472
|
let reconstructionContradictions: Array<{ memoryIds: [string, string]; description: string }> | undefined;
|
|
470
|
-
const constructiveFlag =
|
|
473
|
+
const constructiveFlag = envConfig.constructiveRetrieval();
|
|
471
474
|
if (constructiveFlag && deps.llm?.isAvailable?.()) {
|
|
472
475
|
const allReconResults = [...profileResults, ...preferenceResults, ...entityResults, ...filteredPatterns, ...filteredCases];
|
|
473
476
|
if (allReconResults.length >= 3) {
|
|
@@ -489,6 +492,10 @@ export async function composeResumeContext(
|
|
|
489
492
|
if (recon.contradictions.length > 0) {
|
|
490
493
|
reconstructionContradictions = recon.contradictions;
|
|
491
494
|
}
|
|
495
|
+
// Shadow usage:resume reconstruction 真正引用的记忆记 use 计数(不动 ranking/forget)。
|
|
496
|
+
if (recon.sources.length > 0) {
|
|
497
|
+
await deps.retriever.recordReconstructionUsage?.(recon.sources.map(s => s.id));
|
|
498
|
+
}
|
|
492
499
|
} catch { /* silent fallback */ }
|
|
493
500
|
}
|
|
494
501
|
}
|
package/src/data-checkup.ts
CHANGED
|
@@ -16,6 +16,7 @@ import type { MemoryEntry, MemoryStore } from "./store.js";
|
|
|
16
16
|
import { resolveTier, type MemoryTier } from "./decay-engine.js";
|
|
17
17
|
import { measureInterferenceDensity } from "./interference-detector.js";
|
|
18
18
|
import { readHeartbeats, checkSourceStaleness, formatAge } from "./source-heartbeat.js";
|
|
19
|
+
import { deriveUsageStatus, isUsageSignalActive, type UsageStatus } from "./usage-tracker.js";
|
|
19
20
|
|
|
20
21
|
// ---------------------------------------------------------------------------
|
|
21
22
|
// Types
|
|
@@ -30,6 +31,8 @@ export interface CheckResult {
|
|
|
30
31
|
export interface CheckupReport {
|
|
31
32
|
checks: CheckResult[];
|
|
32
33
|
totalEntries: number;
|
|
34
|
+
/** Total entries in store; when > totalEntries, only the most recent were scanned. */
|
|
35
|
+
totalAvailable: number;
|
|
33
36
|
timestamp: string;
|
|
34
37
|
}
|
|
35
38
|
|
|
@@ -39,15 +42,33 @@ export interface CheckupReport {
|
|
|
39
42
|
|
|
40
43
|
/** Check 1: All vectors should have the same dimension. */
|
|
41
44
|
function checkVectorDimensions(entries: MemoryEntry[]): CheckResult {
|
|
45
|
+
// 只对真正取到向量的条目判维度一致性;取不到向量的(dim=0)单独计数后跳过——
|
|
46
|
+
// 否则会把"未补到向量"误报成"维度不一致",或把"全 0 维空向量"假报成健康。
|
|
42
47
|
const dims = new Map<number, number>(); // dim → count
|
|
48
|
+
let missingVectors = 0;
|
|
43
49
|
for (const e of entries) {
|
|
44
50
|
const d = e.vector?.length ?? 0;
|
|
51
|
+
if (d === 0) { missingVectors++; continue; }
|
|
45
52
|
dims.set(d, (dims.get(d) ?? 0) + 1);
|
|
46
53
|
}
|
|
47
54
|
|
|
48
|
-
|
|
55
|
+
const withVectors = entries.length - missingVectors;
|
|
56
|
+
if (entries.length === 0) {
|
|
57
|
+
return { name: "vector_dimensions", status: "ok", detail: "No entries to check" };
|
|
58
|
+
}
|
|
59
|
+
if (withVectors === 0) {
|
|
60
|
+
return {
|
|
61
|
+
name: "vector_dimensions",
|
|
62
|
+
status: "warning",
|
|
63
|
+
detail: `No retrievable vectors among ${entries.length} entries — dimension consistency not verified`,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const suffix = missingVectors > 0 ? ` (${missingVectors} without retrievable vector, skipped)` : "";
|
|
68
|
+
|
|
69
|
+
if (dims.size === 1) {
|
|
49
70
|
const dim = dims.keys().next().value ?? 0;
|
|
50
|
-
return { name: "vector_dimensions", status: "ok", detail: `All ${
|
|
71
|
+
return { name: "vector_dimensions", status: "ok", detail: `All ${withVectors} vectors have dimension ${dim}${suffix}` };
|
|
51
72
|
}
|
|
52
73
|
|
|
53
74
|
const sorted = [...dims.entries()].sort((a, b) => b[1] - a[1]);
|
|
@@ -56,7 +77,7 @@ function checkVectorDimensions(entries: MemoryEntry[]): CheckResult {
|
|
|
56
77
|
return {
|
|
57
78
|
name: "vector_dimensions",
|
|
58
79
|
status: "error",
|
|
59
|
-
detail: `${mismatched} entries have wrong dimension (expected ${expected}): ${sorted.map(([d, c]) => `dim=${d}: ${c}`).join(", ")}`,
|
|
80
|
+
detail: `${mismatched} entries have wrong dimension (expected ${expected}): ${sorted.map(([d, c]) => `dim=${d}: ${c}`).join(", ")}${suffix}`,
|
|
60
81
|
};
|
|
61
82
|
}
|
|
62
83
|
|
|
@@ -77,7 +98,7 @@ function checkOrphanMemories(entries: MemoryEntry[]): CheckResult {
|
|
|
77
98
|
function checkTierDistribution(entries: MemoryEntry[]): CheckResult {
|
|
78
99
|
const tierCounts: Record<MemoryTier, number> = { core: 0, working: 0, peripheral: 0 };
|
|
79
100
|
for (const e of entries) {
|
|
80
|
-
const tier = resolveTier(e.metadata);
|
|
101
|
+
const tier = resolveTier(e.metadata, e.importance);
|
|
81
102
|
tierCounts[tier]++;
|
|
82
103
|
}
|
|
83
104
|
|
|
@@ -216,15 +237,78 @@ function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
|
|
|
216
237
|
// Main
|
|
217
238
|
// ---------------------------------------------------------------------------
|
|
218
239
|
|
|
240
|
+
/** Check 8 (P0 B-1 观察): usage 四档分布 — injection-vs-use 影子统计的离线观察口。
|
|
241
|
+
* 纯报告,不接任何决策;cold = 被反复 surface(accessCount≥6)却从未被
|
|
242
|
+
* reconstruction 引用,是第二阶段 decay 联动前要先观察校准的核心对象。 */
|
|
243
|
+
function checkUsageDistribution(entries: MemoryEntry[]): CheckResult {
|
|
244
|
+
// use 信号未采集时(reconstruction flag 关),useCount 恒零 → "cold" 全是假阳性。
|
|
245
|
+
// 如实报告信号缺失,不输出不可解释的判定。
|
|
246
|
+
if (!isUsageSignalActive()) {
|
|
247
|
+
return {
|
|
248
|
+
name: "usage_distribution",
|
|
249
|
+
status: "ok",
|
|
250
|
+
detail: "use signal inactive (RECALLNEST_CONSTRUCTIVE_RETRIEVAL off) — useCount 无写入路径,cold/warm/hot 不可判,统计跳过。该 flag 因合成召回缺失被 Alice 主动关闭(2026-06-11),usage 线搁置中。",
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const counts: Record<UsageStatus, number> = { unused: 0, cold: 0, warm: 0, hot: 0 };
|
|
255
|
+
const coldSamples: Array<{ id: string; injection: number; ageDays: number; scope: string }> = [];
|
|
256
|
+
const now = Date.now();
|
|
257
|
+
|
|
258
|
+
for (const e of entries) {
|
|
259
|
+
const status = deriveUsageStatus(e);
|
|
260
|
+
counts[status]++;
|
|
261
|
+
if (status === "cold") {
|
|
262
|
+
let injection = 0;
|
|
263
|
+
try {
|
|
264
|
+
const meta = JSON.parse(e.metadata || "{}") as Record<string, unknown>;
|
|
265
|
+
injection = typeof meta.accessCount === "number" ? meta.accessCount : 0;
|
|
266
|
+
} catch { /* keep 0 */ }
|
|
267
|
+
coldSamples.push({
|
|
268
|
+
id: e.id.slice(0, 8),
|
|
269
|
+
injection,
|
|
270
|
+
ageDays: Math.floor((now - e.timestamp) / 86_400_000),
|
|
271
|
+
scope: e.scope || "(none)",
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const total = entries.length;
|
|
277
|
+
const coldPct = total > 0 ? (counts.cold / total) * 100 : 0;
|
|
278
|
+
const top = coldSamples
|
|
279
|
+
.sort((a, b) => b.injection - a.injection)
|
|
280
|
+
.slice(0, 20)
|
|
281
|
+
.map(s => `${s.id}(inj=${s.injection}, ${s.ageDays}d, ${s.scope})`)
|
|
282
|
+
.join(", ");
|
|
283
|
+
|
|
284
|
+
return {
|
|
285
|
+
name: "usage_distribution",
|
|
286
|
+
status: coldPct > 10 ? "warning" : "ok",
|
|
287
|
+
detail:
|
|
288
|
+
`unused=${counts.unused} cold=${counts.cold} (${coldPct.toFixed(1)}%) ` +
|
|
289
|
+
`warm=${counts.warm} hot=${counts.hot}` +
|
|
290
|
+
(coldSamples.length > 0 ? `; cold top-${Math.min(20, coldSamples.length)}: ${top}` : ""),
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
|
|
219
294
|
export interface CheckupDeps {
|
|
220
|
-
store: Pick<MemoryStore, "list" | "stats">;
|
|
295
|
+
store: Pick<MemoryStore, "list" | "stats" | "getVectors">;
|
|
221
296
|
openConflictCount: number;
|
|
222
297
|
/** Optional override path for source-heartbeat.json (used in tests). */
|
|
223
298
|
heartbeatPath?: string;
|
|
224
299
|
}
|
|
225
300
|
|
|
226
301
|
export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport> {
|
|
227
|
-
const
|
|
302
|
+
const SCAN_LIMIT = 10000;
|
|
303
|
+
const listed = await deps.store.list(undefined, undefined, SCAN_LIMIT, 0);
|
|
304
|
+
|
|
305
|
+
// list() 为性能不返回向量(vector: []);维度/干扰检查靠向量,否则把"全 0 维"
|
|
306
|
+
// 误判成健康、干扰分析永远"样本不足"。补回真实向量再检查。
|
|
307
|
+
const vectorMap = await deps.store.getVectors(listed.map(e => e.id));
|
|
308
|
+
const entries = listed.map(e => ({ ...e, vector: vectorMap.get(e.id) ?? [] }));
|
|
309
|
+
|
|
310
|
+
// 截断披露:库总量 > 已扫,说明只体检了最近 SCAN_LIMIT 条。
|
|
311
|
+
const totalAvailable = (await deps.store.stats()).totalCount;
|
|
228
312
|
|
|
229
313
|
return {
|
|
230
314
|
checks: [
|
|
@@ -235,16 +319,21 @@ export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport>
|
|
|
235
319
|
checkVersionGroups(entries),
|
|
236
320
|
checkSourceHealth(deps.heartbeatPath),
|
|
237
321
|
checkInterferenceDensity(entries),
|
|
322
|
+
checkUsageDistribution(entries),
|
|
238
323
|
],
|
|
239
324
|
totalEntries: entries.length,
|
|
325
|
+
totalAvailable,
|
|
240
326
|
timestamp: new Date().toISOString(),
|
|
241
327
|
};
|
|
242
328
|
}
|
|
243
329
|
|
|
244
330
|
export function formatCheckupReport(report: CheckupReport): string {
|
|
331
|
+
const truncationNote = report.totalAvailable > report.totalEntries
|
|
332
|
+
? ` (of ${report.totalAvailable} total — only the most recent ${report.totalEntries} analyzed)`
|
|
333
|
+
: "";
|
|
245
334
|
const lines = [
|
|
246
335
|
`Data Checkup Report (${report.timestamp})`,
|
|
247
|
-
`Total entries scanned: ${report.totalEntries}`,
|
|
336
|
+
`Total entries scanned: ${report.totalEntries}${truncationNote}`,
|
|
248
337
|
"",
|
|
249
338
|
];
|
|
250
339
|
|