recallnest 2.3.0 → 2.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/CHANGELOG.md +150 -0
  2. package/README.md +17 -31
  3. package/README_CN.md +12 -22
  4. package/assets/ui/app.js +1031 -0
  5. package/assets/ui/index.html +224 -0
  6. package/assets/ui/styles.css +781 -0
  7. package/config.json.example +4 -1
  8. package/docs/api-reference.md +1291 -0
  9. package/docs/architecture.md +168 -0
  10. package/docs/connector-spec.md +126 -0
  11. package/eval/cases-canary.json +58 -0
  12. package/eval/cases-scope-robustness.json +278 -0
  13. package/eval/cases.json +117 -20
  14. package/package.json +22 -1
  15. package/src/__tests__/access-tracker-novelty.test.ts +238 -0
  16. package/src/__tests__/activity-counter.test.ts +125 -0
  17. package/src/__tests__/adaptive-fidelity.test.ts +102 -0
  18. package/src/__tests__/admission-control.test.ts +206 -0
  19. package/src/__tests__/anchor-generator.test.ts +67 -0
  20. package/src/__tests__/auto-consolidation.test.ts +168 -0
  21. package/src/__tests__/auto-recall.test.ts +131 -0
  22. package/src/__tests__/babel-integration.test.ts +144 -0
  23. package/src/__tests__/batch-dedup.test.ts +65 -0
  24. package/src/__tests__/canary-eval.test.ts +114 -0
  25. package/src/__tests__/capture-engine.test.ts +1238 -0
  26. package/src/__tests__/capture-heuristic.test.ts +171 -0
  27. package/src/__tests__/cascade-forget.test.ts +210 -0
  28. package/src/__tests__/category-min-scores.test.ts +33 -0
  29. package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
  30. package/src/__tests__/circuit-breaker.test.ts +99 -0
  31. package/src/__tests__/cluster-consolidation.test.ts +782 -0
  32. package/src/__tests__/confidence-metadata.test.ts +125 -0
  33. package/src/__tests__/confidence-tracker.test.ts +261 -0
  34. package/src/__tests__/conflict-advisor.test.ts +206 -0
  35. package/src/__tests__/conflict-escalation.test.ts +149 -0
  36. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  37. package/src/__tests__/conflict-output.test.ts +85 -0
  38. package/src/__tests__/conflict-store.test.ts +374 -0
  39. package/src/__tests__/connector-v1.test.ts +181 -0
  40. package/src/__tests__/consolidation-engine.test.ts +333 -0
  41. package/src/__tests__/constructive-retrieval.test.ts +509 -0
  42. package/src/__tests__/context-collapse-renderer.test.ts +214 -0
  43. package/src/__tests__/context-composer-pins.test.ts +64 -0
  44. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  45. package/src/__tests__/context-reconstructor.test.ts +65 -0
  46. package/src/__tests__/context-renderer.test.ts +91 -0
  47. package/src/__tests__/continuity-eval.test.ts +275 -0
  48. package/src/__tests__/continuity-schema.test.ts +97 -0
  49. package/src/__tests__/contracts.test.ts +216 -0
  50. package/src/__tests__/conversation-importer.test.ts +692 -0
  51. package/src/__tests__/core-recall-eval.test.ts +235 -0
  52. package/src/__tests__/core-summary.test.ts +109 -0
  53. package/src/__tests__/data-checkup.test.ts +276 -0
  54. package/src/__tests__/decay-exemption.test.ts +84 -0
  55. package/src/__tests__/dedup-actions.test.ts +58 -0
  56. package/src/__tests__/dedup-guard.test.ts +39 -0
  57. package/src/__tests__/deterministic-id.test.ts +39 -0
  58. package/src/__tests__/distill-lock.test.ts +305 -0
  59. package/src/__tests__/distill-session-argshape.test.ts +79 -0
  60. package/src/__tests__/doctor.test.ts +319 -0
  61. package/src/__tests__/dream-pipeline.test.ts +321 -0
  62. package/src/__tests__/drill-down.test.ts +108 -0
  63. package/src/__tests__/embedder.test.ts +122 -0
  64. package/src/__tests__/emotion-decay.test.ts +46 -0
  65. package/src/__tests__/emotion-detector.test.ts +56 -0
  66. package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
  67. package/src/__tests__/emotion-retrieval.test.ts +56 -0
  68. package/src/__tests__/entity-resolver.test.ts +58 -0
  69. package/src/__tests__/env-config.test.ts +151 -0
  70. package/src/__tests__/error-signature-boost.test.ts +87 -0
  71. package/src/__tests__/error-signature.test.ts +95 -0
  72. package/src/__tests__/error-taxonomy.test.ts +129 -0
  73. package/src/__tests__/ethics-layer.test.ts +476 -0
  74. package/src/__tests__/event-segmenter.test.ts +153 -0
  75. package/src/__tests__/feature-flags-integration.test.ts +54 -0
  76. package/src/__tests__/frequency-tracker.test.ts +113 -0
  77. package/src/__tests__/graph-export.test.ts +489 -0
  78. package/src/__tests__/hotness-blend.test.ts +76 -0
  79. package/src/__tests__/ingest-chunking.test.ts +44 -0
  80. package/src/__tests__/ingest.test.ts +360 -0
  81. package/src/__tests__/interference-detector.test.ts +203 -0
  82. package/src/__tests__/kg-extractor.test.ts +192 -0
  83. package/src/__tests__/kg-store.test.ts +300 -0
  84. package/src/__tests__/llm-consolidation.test.ts +226 -0
  85. package/src/__tests__/maintenance-locks.test.ts +174 -0
  86. package/src/__tests__/mcp-efficiency.test.ts +288 -0
  87. package/src/__tests__/mcp-related-scopes.test.ts +151 -0
  88. package/src/__tests__/memory-boundaries.test.ts +250 -0
  89. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
  90. package/src/__tests__/memory-evolution.test.ts +271 -0
  91. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  92. package/src/__tests__/memory-lint.test.ts +421 -0
  93. package/src/__tests__/memory-output.test.ts +215 -0
  94. package/src/__tests__/memory-promotion.test.ts +315 -0
  95. package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
  96. package/src/__tests__/multi-vector.test.ts +242 -0
  97. package/src/__tests__/narrative-architecture.test.ts +456 -0
  98. package/src/__tests__/noise-filter.test.ts +205 -0
  99. package/src/__tests__/obsidian-connector.test.ts +363 -0
  100. package/src/__tests__/omitted-reporting.test.ts +98 -0
  101. package/src/__tests__/pending-review.test.ts +77 -0
  102. package/src/__tests__/pii-redact.test.ts +314 -0
  103. package/src/__tests__/post-retrieval-filter.test.ts +178 -0
  104. package/src/__tests__/ppr-traversal.test.ts +256 -0
  105. package/src/__tests__/prediction-engine.test.ts +349 -0
  106. package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
  107. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  108. package/src/__tests__/preference-matcher.test.ts +230 -0
  109. package/src/__tests__/preference-slots.test.ts +205 -0
  110. package/src/__tests__/prospective-memory.test.ts +283 -0
  111. package/src/__tests__/query-entity-detector.test.ts +171 -0
  112. package/src/__tests__/query-expander-alias.test.ts +146 -0
  113. package/src/__tests__/query-expander.test.ts +34 -0
  114. package/src/__tests__/recall-governor.test.ts +227 -0
  115. package/src/__tests__/recall-mode.test.ts +64 -0
  116. package/src/__tests__/result-synthesizer.test.ts +198 -0
  117. package/src/__tests__/resume-context.test.ts +4898 -0
  118. package/src/__tests__/retrieval-trace.test.ts +213 -0
  119. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  120. package/src/__tests__/retroactive-boost.test.ts +223 -0
  121. package/src/__tests__/salience-scoring.test.ts +106 -0
  122. package/src/__tests__/scope-inventory.test.ts +200 -0
  123. package/src/__tests__/scope-policy.test.ts +69 -0
  124. package/src/__tests__/scope-suggester.test.ts +80 -0
  125. package/src/__tests__/session-checkpoint.test.ts +291 -0
  126. package/src/__tests__/session-distiller.test.ts +640 -0
  127. package/src/__tests__/session-gc.test.ts +215 -0
  128. package/src/__tests__/skill-engine.test.ts +552 -0
  129. package/src/__tests__/skill-promotion.test.ts +601 -0
  130. package/src/__tests__/skill-verifier.test.ts +122 -0
  131. package/src/__tests__/source-heartbeat.test.ts +250 -0
  132. package/src/__tests__/store-patch-metadata.test.ts +205 -0
  133. package/src/__tests__/store-scope-audit.test.ts +81 -0
  134. package/src/__tests__/store-upsert.test.ts +184 -0
  135. package/src/__tests__/supersede-chain.test.ts +154 -0
  136. package/src/__tests__/temporal-parser.test.ts +241 -0
  137. package/src/__tests__/temporal-validity.test.ts +160 -0
  138. package/src/__tests__/term-registry.test.ts +121 -0
  139. package/src/__tests__/tool-error-surfacing.test.ts +98 -0
  140. package/src/__tests__/tool-output-compressor.test.ts +160 -0
  141. package/src/__tests__/topic-tag.test.ts +113 -0
  142. package/src/__tests__/trusted-memory.test.ts +839 -0
  143. package/src/__tests__/usage-tracker.test.ts +287 -0
  144. package/src/__tests__/version-manager.test.ts +213 -0
  145. package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
  146. package/src/__tests__/workflow-observation.test.ts +433 -0
  147. package/src/__tests__/write-verifier.test.ts +122 -0
  148. package/src/access-tracker.ts +38 -32
  149. package/src/activity-counter.ts +46 -18
  150. package/src/aliases.ts +133 -0
  151. package/src/api-server.ts +27 -7
  152. package/src/audit-log.ts +4 -2
  153. package/src/auto-consolidation.ts +8 -3
  154. package/src/auto-gc.ts +85 -21
  155. package/src/capture-engine.ts +54 -21
  156. package/src/cli.ts +169 -37
  157. package/src/confidence-tracker.ts +36 -18
  158. package/src/consolidation-engine.ts +109 -11
  159. package/src/context-composer.ts +8 -1
  160. package/src/data-checkup.ts +95 -6
  161. package/src/distill-lock.ts +194 -15
  162. package/src/doctor.ts +9 -2
  163. package/src/dream-pipeline.ts +118 -34
  164. package/src/env-config.ts +77 -0
  165. package/src/error-signature.ts +153 -0
  166. package/src/error-taxonomy.ts +115 -0
  167. package/src/eval.ts +251 -6
  168. package/src/idempotency.ts +8 -0
  169. package/src/ingest.ts +31 -10
  170. package/src/kg-extractor.ts +6 -2
  171. package/src/kg-store.ts +236 -44
  172. package/src/llm-consolidation.ts +2 -1
  173. package/src/mcp-server.ts +85 -1799
  174. package/src/mcp-tool-deps.ts +48 -0
  175. package/src/mcp-tools-advanced.ts +941 -0
  176. package/src/mcp-tools-core.ts +633 -0
  177. package/src/mcp-tools-governance.ts +475 -0
  178. package/src/memory-boundaries.ts +3 -1
  179. package/src/memory-evolution.ts +73 -1
  180. package/src/memory-lint.ts +183 -10
  181. package/src/memory-output.ts +114 -3
  182. package/src/memory-promotion.ts +246 -0
  183. package/src/memory-schema.ts +25 -2
  184. package/src/memory-store-port.ts +43 -0
  185. package/src/multi-vector.ts +2 -1
  186. package/src/narrative-schema.ts +2 -1
  187. package/src/noise-filter.ts +57 -6
  188. package/src/pii-detector.ts +157 -5
  189. package/src/query-expander.ts +90 -9
  190. package/src/result-synthesizer.ts +2 -1
  191. package/src/retention-policy.ts +2 -1
  192. package/src/retriever.ts +259 -139
  193. package/src/runtime-config.ts +8 -1
  194. package/src/scope-suggester.ts +113 -0
  195. package/src/session-distiller.ts +23 -2
  196. package/src/session-engine.ts +2 -1
  197. package/src/session-schema.ts +1 -0
  198. package/src/session-store.ts +5 -0
  199. package/src/skill-engine.ts +78 -0
  200. package/src/skill-promotion.ts +247 -47
  201. package/src/skill-schema.ts +16 -3
  202. package/src/skill-verifier.ts +152 -0
  203. package/src/store.ts +267 -38
  204. package/src/tool-output-compressor.ts +3 -1
  205. package/src/ui-server.ts +2 -1
  206. package/src/usage-tracker.ts +242 -0
  207. package/src/version-manager.ts +3 -2
  208. package/src/workflow-observation-engine.ts +2 -1
  209. package/src/workflow-observation-schema.ts +2 -1
  210. package/src/workflow-observation-store.ts +6 -2
  211. package/.env +0 -4
  212. package/CLAUDE.md +0 -46
  213. package/config.json +0 -55
  214. package/daily-recall.sh +0 -110
  215. package/logs/daily-recall.error.log +0 -0
  216. package/logs/daily-recall.log +0 -60
  217. package/logs/ingest-2026-04-11.log +0 -110
  218. package/logs/ingest-2026-04-12.log +0 -90
  219. package/logs/ingest-2026-04-13.log +0 -87
  220. package/logs/launchagent-stderr.log +0 -0
  221. package/logs/launchagent-stdout.log +0 -0
@@ -1,7 +1,7 @@
1
1
  import type { Embedder } from "./embedder.js";
2
2
  import { detectLang, tokenizeFts } from "./language-hook.js";
3
3
  import { generateAnchor } from "./anchor-generator.js";
4
- import { incrementWriteCount } from "./activity-counter.js";
4
+ import { extractErrorSignatures } from "./error-signature.js";
5
5
  import { verifyWrite } from "./write-verifier.js";
6
6
  // batchInternalDedup is available in ingest.ts for large-batch scenarios;
7
7
  // persistMemoryBatch relies on per-item conflict detection (A-2) instead.
@@ -56,7 +56,7 @@ import {
56
56
  import { matchPreference, applyPreferenceMatch } from "./preference-matcher.js";
57
57
  import type { LLMClient } from "./llm-client.js";
58
58
  import type { KGExtractor } from "./kg-extractor.js";
59
- import { scanForPII } from "./pii-detector.js";
59
+ import { scanForPII, redactSecrets } from "./pii-detector.js";
60
60
  import type { AuditLogger } from "./audit-log.js";
61
61
  import { checkAdmission, type ScopeRateLimiter, type AdmissionConfig } from "./admission-control.js";
62
62
  import { tagNarrativeIfEnabled } from "./narrative-tagger.js";
@@ -699,6 +699,17 @@ function buildCaseMemoryMetadata(
699
699
  tools: input.tools,
700
700
  },
701
701
  };
702
+ // A1: error-signature 指纹(读写对称,P3-A 检索复用同一抽取器)。
703
+ // 在 build metadata 前算好作为 extra 写入,不 post-write update(Codex 验证要求)。
704
+ const errorSignature = extractErrorSignatures({
705
+ problem: input.problem,
706
+ context: input.context,
707
+ outcome: input.outcome,
708
+ solutionSteps: input.solutionSteps,
709
+ });
710
+ if (errorSignature.length > 0) cmExtra.error_signature = errorSignature;
711
+ // A1: break-loop 五维(可选,向后兼容;不提供则普通 case)。
712
+ if (input.debugFraming) cmExtra.debugFraming = input.debugFraming;
702
713
  const anchor = generateAnchor(input.title + ": " + input.problem, cmExtra);
703
714
  if (anchor) cmExtra.anchor = anchor;
704
715
  return buildStructuredMetadata({
@@ -746,7 +757,16 @@ export async function persistMemory(
746
757
  deps: PersistMemoryDeps,
747
758
  rawInput: unknown,
748
759
  ): Promise<StoredMemoryRecord> {
749
- const input = StoreMemoryInputSchema.parse(rawInput);
760
+ let input = StoreMemoryInputSchema.parse(rawInput);
761
+
762
+ // F-3b: Scan THEN redact, before the text reaches any external surface —
763
+ // LLM summary/importance calls, embedding API, storage, audit log, KG.
764
+ // Scan first so the piiWarning below still reflects what the raw text held.
765
+ const piiScan = scanForPII(input.text);
766
+ const redaction = redactSecrets(input.text);
767
+ if (redaction.redacted > 0) {
768
+ input = { ...input, text: redaction.text };
769
+ }
750
770
 
751
771
  // F3: Extract temporal validity params before Zod strips them
752
772
  const rawObj = rawInput as Record<string, unknown> | null;
@@ -801,6 +821,18 @@ export async function persistMemory(
801
821
  deps.admissionConfig,
802
822
  );
803
823
  if (admission.verdict === "rejected") {
824
+ // Gemini #26523 lesson: rejected writes must stay queryable after the fact,
825
+ // not just echo back to the caller — otherwise they become invisible noise.
826
+ try {
827
+ deps.auditLogger?.log({
828
+ operation: "reject",
829
+ scope: admissionScope,
830
+ actor: input.source,
831
+ details: `${admission.reason ?? "unknown"}: ${input.text.slice(0, 100)}`,
832
+ });
833
+ } catch {
834
+ // Audit must never block memory writes
835
+ }
804
836
  return {
805
837
  id: `rejected-${Date.now()}`,
806
838
  text: input.text,
@@ -812,6 +844,7 @@ export async function persistMemory(
812
844
  resolvedScope: admissionScope,
813
845
  storedAt: new Date().toISOString(),
814
846
  disposition: "rejected" as const,
847
+ ...(admission.reason ? { rejectionReason: admission.reason } : {}),
815
848
  };
816
849
  }
817
850
 
@@ -937,15 +970,16 @@ export async function persistMemory(
937
970
  }
938
971
 
939
972
  // F-3: PII detection — inject warning into metadata if PII found.
940
- // Non-blocking: PII does not prevent the write.
973
+ // Non-blocking: PII does not prevent the write. Uses the pre-redaction scan
974
+ // from the top of persistMemory (scanning here would see scrubbed text).
941
975
  let metadata = buildStoreMemoryMetadata(input, canonicalKey);
942
- const piiResult = scanForPII(input.text);
943
- if (piiResult.hasPII) {
976
+ if (piiScan.hasPII) {
944
977
  const parsed: Record<string, unknown> = JSON.parse(metadata);
945
978
  parsed.piiWarning = {
946
- summary: piiResult.summary,
947
- detections: piiResult.detections.length,
948
- severity: piiResult.detections.some(d => d.severity === "high") ? "high" : "medium",
979
+ summary: piiScan.summary,
980
+ detections: piiScan.detections.length,
981
+ severity: piiScan.detections.some(d => d.severity === "high") ? "high" : "medium",
982
+ ...(redaction.redacted > 0 ? { redacted: redaction.redacted } : {}),
949
983
  };
950
984
  metadata = JSON.stringify(parsed);
951
985
  }
@@ -1031,14 +1065,9 @@ export async function persistMemory(
1031
1065
  // Audit must never block memory writes
1032
1066
  }
1033
1067
 
1034
- // HP-3: Activity counter — track non-dedup writes for distill trigger
1035
- if (disposition !== "deduped") {
1036
- try {
1037
- incrementWriteCount();
1038
- } catch {
1039
- // Activity tracking must never block memory writes
1040
- }
1041
- }
1068
+ // HP-3 activity counter moved to the store layer (store/upsert/storeBatch count per
1069
+ // scope): that is the necessary chokepoint for every write source (capture/ingest/
1070
+ // store_memory/consolidation), and per-scope counts drive per-scope dream activation.
1042
1071
 
1043
1072
  // HP-2: Post-write verification — async, non-blocking
1044
1073
  if (disposition !== "deduped" && deps.store.get) {
@@ -1063,7 +1092,7 @@ export async function persistMemory(
1063
1092
  input.category !== "preferences" &&
1064
1093
  inferImplicitUsageSlot(input.text)
1065
1094
  ) {
1066
- const prefCanonicalKey = buildDefaultCanonicalKey("preferences", input.text);
1095
+ const prefCanonicalKey = buildDefaultCanonicalKey({ category: "preferences", text: input.text });
1067
1096
  const prefSlot = inferImplicitUsageSlot(input.text);
1068
1097
  const prefMetadata = buildStructuredMetadata({
1069
1098
  source: input.source,
@@ -1098,7 +1127,8 @@ export async function persistWorkflowPattern(
1098
1127
  const input = WorkflowPatternInputSchema.parse(rawInput);
1099
1128
  const resolvedScope = resolveScope(input);
1100
1129
  const tags = mergeTags(input.tags, ["workflow", "pattern"]);
1101
- const text = buildWorkflowPatternText(input);
1130
+ // F-3b: scrub before embedding/storage (this path bypasses persistMemory)
1131
+ const text = redactSecrets(buildWorkflowPatternText(input)).text;
1102
1132
  const vector = await deps.embedder.embedPassage(text);
1103
1133
  const language = detectLang(text);
1104
1134
  const fts_text = tokenizeFts(text, language);
@@ -1142,7 +1172,8 @@ export async function persistCaseMemory(
1142
1172
  const input = CaseMemoryInputSchema.parse(rawInput);
1143
1173
  const resolvedScope = resolveScope(input);
1144
1174
  const tags = mergeTags(input.tags, ["case", "solution"]);
1145
- const text = buildCaseMemoryText(input);
1175
+ // F-3b: scrub before embedding/storage (this path bypasses persistMemory)
1176
+ const text = redactSecrets(buildCaseMemoryText(input)).text;
1146
1177
  const vector = await deps.embedder.embedPassage(text);
1147
1178
  const language = detectLang(text);
1148
1179
  const fts_text = tokenizeFts(text, language);
@@ -1202,7 +1233,9 @@ export async function promoteMemory(
1202
1233
  }
1203
1234
 
1204
1235
  const category = inferPromotedCategory(input, sourceEntry);
1205
- const text = input.text || sourceEntry.text;
1236
+ // F-3b: scrub caller-supplied text AND legacy stored text (pre-redaction
1237
+ // era entries may still hold secrets; promotion is a natural wash point)
1238
+ const text = redactSecrets(input.text || sourceEntry.text).text;
1206
1239
  const canonicalKey = resolveCanonicalKey({
1207
1240
  category,
1208
1241
  text,
package/src/cli.ts CHANGED
@@ -11,7 +11,7 @@
11
11
 
12
12
  import { Command } from "commander";
13
13
  import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
14
- import { join, resolve } from "node:path";
14
+ import { basename, dirname, join, resolve } from "node:path";
15
15
  import { homedir } from "node:os";
16
16
 
17
17
  import { metaDir } from "./compat.js";
@@ -21,6 +21,8 @@ import { distillResults, formatExplainResults, formatSearchResults, selectBriefS
21
21
  import { archiveDirtyBriefAsset, assetSummaryLine, buildBriefAsset, buildPinAsset, getExportsDir, listDirtyBriefAssets, listMemoryAssets, listPinAssets, pinSummaryLine, saveBriefAsset, savePinAsset, writeExportArtifact } from "./memory-assets.js";
22
22
  import { indexAsset, indexPinnedAsset } from "./asset-sync.js";
23
23
  import { createComponents, createStoreOnly, expandHome, loadConfig, loadDotEnv, type LocalMemoryConfig } from "./runtime-config.js";
24
+ import { KGStore } from "./kg-store.js";
25
+ import { isKGModeEnabled } from "./kg-extractor.js";
24
26
  import {
25
27
  formatDedupReasonSummary,
26
28
  getDedupSkipRate,
@@ -34,6 +36,9 @@ import {
34
36
  } from "./ingest.js";
35
37
  import { runDoctor, formatDoctorResults } from "./doctor.js";
36
38
  import { persistCaseMemory, persistMemory, persistWorkflowPattern } from "./capture-engine.js";
39
+ import { scanMemoryPromotions, buildPromoteScanDeps, formatPromoteScanResult } from "./memory-promotion.js";
40
+ import { runDream, formatDreamResult, DEFAULT_DREAM_CONFIG } from "./dream-pipeline.js";
41
+ import { listScopesAboveThreshold } from "./activity-counter.js";
37
42
  import {
38
43
  CaseMemoryInputSchema,
39
44
  type CaseMemoryInput,
@@ -47,6 +52,7 @@ import { ConflictStatusSchema } from "./conflict-schema.js";
47
52
  import { resolveConflictCandidate } from "./conflict-engine.js";
48
53
  import { escalateConflicts } from "./conflict-escalation.js";
49
54
  import { ConflictCandidateStore } from "./conflict-store.js";
55
+ import { createAuditLogger } from "./audit-log.js";
50
56
  import { formatConflictAudit, formatConflictAuditMarkdown, formatConflictClusters, formatConflictEscalation, formatConflictList, formatConflictRecord, formatConflictResolution } from "./conflict-output.js";
51
57
  import { CONFLICT_ATTENTION_LEVELS, parseConflictAttention, summarizeConflictLifecycle } from "./conflict-lifecycle.js";
52
58
  import { buildConflictAuditSummary, clusterConflicts, summarizeConflictAdvice } from "./conflict-advisor.js";
@@ -63,49 +69,66 @@ import {
63
69
  } from "./workflow-observation-scope-review.js";
64
70
  import { collectScopeInventory, formatScopeInventoryReport } from "./scope-inventory.js";
65
71
  import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
72
+ import { createScopeSuggester } from "./scope-suggester.js";
66
73
 
67
74
  /**
68
- * Auto-detect Claude Code transcript directory.
69
- * Scans ~/.claude/projects/ for subdirectories containing .jsonl files.
75
+ * Auto-detect Claude Code transcript directories.
76
+ * Scans ~/.claude/projects/ for ALL subdirectories containing .jsonl files.
77
+ *
78
+ * CC 按 cwd 编码 project 子目录,凡是不在 ~ 下跑的会话(Codex spawn 的子进程 / cd 到 repo 跑 /
79
+ * bridge 临时工作目录)会落到 ~/.claude/projects/-XXX/ 别的子目录。旧版只挑 jsonl 最多的根
80
+ * 目录返回,导致 28+ 个子目录全失声。修复:返回所有有 jsonl 的子目录,调用方循环 ingest。
70
81
  */
71
- function detectCCPath(): string | null {
82
+ function detectCCPath(): string[] {
72
83
  const projectsDir = join(homedir(), ".claude", "projects");
73
- if (!existsSync(projectsDir)) return null;
84
+ if (!existsSync(projectsDir)) return [];
74
85
  try {
75
- const dirs = readdirSync(projectsDir, { withFileTypes: true })
86
+ return readdirSync(projectsDir, { withFileTypes: true })
76
87
  .filter(d => d.isDirectory())
77
- .map(d => join(projectsDir, d.name));
78
- // Pick the directory with most .jsonl files
79
- let best = "";
80
- let bestCount = 0;
81
- for (const dir of dirs) {
82
- const jsonlCount = readdirSync(dir).filter(f => f.endsWith(".jsonl")).length;
83
- if (jsonlCount > bestCount) {
84
- bestCount = jsonlCount;
85
- best = dir;
86
- }
87
- }
88
- return best || null;
88
+ .map(d => join(projectsDir, d.name))
89
+ .filter(dir => {
90
+ try { return readdirSync(dir).some(f => f.endsWith(".jsonl")); }
91
+ catch { return false; }
92
+ });
89
93
  } catch {
90
- return null;
94
+ return [];
95
+ }
96
+ }
97
+
98
+ /**
99
+ * Pick the CC project subdirectory with the most jsonl files (heuristic: user's primary cwd).
100
+ * Used by memory source which still expects a single directory.
101
+ */
102
+ function pickPrimaryCCPath(detected: string[]): string {
103
+ let best = detected[0];
104
+ let bestCount = 0;
105
+ for (const dir of detected) {
106
+ try {
107
+ const c = readdirSync(dir).filter(f => f.endsWith(".jsonl")).length;
108
+ if (c > bestCount) { bestCount = c; best = dir; }
109
+ } catch { /* skip */ }
91
110
  }
111
+ return best;
92
112
  }
93
113
 
94
114
  /**
95
115
  * Resolve "auto" paths in config to actual directories.
116
+ *
117
+ * Note: cc 分支只返回第一个 detect 到的目录(fallback),ingest 入口已改为直接调
118
+ * detectCCPath() 拿全量子目录循环处理。这里保留是为兼容潜在的其它调用方。
96
119
  */
97
120
  function resolveSourcePath(source: string, key: string): string {
98
121
  if (source !== "auto") return expandHome(source);
99
122
 
100
123
  if (key === "cc") {
101
124
  const detected = detectCCPath();
102
- if (!detected) throw new Error("Could not auto-detect Claude Code transcript path. Set sources.cc.path in config.json.");
103
- return detected;
125
+ if (detected.length === 0) throw new Error("Could not auto-detect Claude Code transcript path. Set sources.cc.path in config.json.");
126
+ return pickPrimaryCCPath(detected);
104
127
  }
105
128
  if (key === "memory") {
106
- const ccPath = detectCCPath();
107
- if (!ccPath) throw new Error("Could not auto-detect memory path. Set sources.memory.path in config.json.");
108
- return join(ccPath, "memory");
129
+ const detected = detectCCPath();
130
+ if (detected.length === 0) throw new Error("Could not auto-detect memory path. Set sources.memory.path in config.json.");
131
+ return join(pickPrimaryCCPath(detected), "memory");
109
132
  }
110
133
  throw new Error(`Cannot auto-detect path for source "${key}". Set it manually in config.json.`);
111
134
  }
@@ -147,7 +170,7 @@ async function runRetrievalView(
147
170
  },
148
171
  ) {
149
172
  const config = loadConfig();
150
- const { retriever, profile } = createComponents(config, options.profile);
173
+ const { retriever, profile, store } = createComponents(config, options.profile);
151
174
 
152
175
  const limit = parseLimitOption(options.limit, 5, 1, 20);
153
176
 
@@ -206,6 +229,17 @@ async function runRetrievalView(
206
229
  : formatSearchResults(results, context);
207
230
 
208
231
  console.log(rendered);
232
+
233
+ // P1-A: 显式 scope 0 命中 → 相近 scope 提示打到 stderr(不污染 stdout)
234
+ if (results.length === 0 && options.scope) {
235
+ try {
236
+ const suggest = createScopeSuggester(async () => (await store.stats()).scopeCounts);
237
+ const suggestions = (await suggest(options.scope)).filter(s => s !== options.scope);
238
+ if (suggestions.length > 0) {
239
+ console.error(`⚠️ scope '${options.scope}' 命中 0 条,相近 scope: [${suggestions.join(", ")}]`);
240
+ }
241
+ } catch { /* best-effort */ }
242
+ }
209
243
  }
210
244
 
211
245
  function parseLimitOption(value: string | undefined, fallback: number, min = 1, max = 50): number {
@@ -644,6 +678,28 @@ program
644
678
  ].join("\n"));
645
679
  });
646
680
 
681
+ program
682
+ .command("promote-scan")
683
+ .description("扫描 transcript 降级的 profile/preferences evidence,聚类高频高 importance 项晋升回 durable")
684
+ .requiredOption("--scope <scope>", "要扫描的 scope,如 cc:project:recallnest")
685
+ .option("--apply", "执行晋升;默认 dry-run 只预览")
686
+ .option("--min-occurrences <n>", "聚类最小成员数(默认 3)", "3")
687
+ .option("--min-importance <n>", "聚类最小平均 importance(默认 0.6)", "0.6")
688
+ .action(async (options) => {
689
+ const config = loadConfig();
690
+ const { store, embedder } = createComponents(config);
691
+ const conflictStore = new ConflictCandidateStore();
692
+ const deps = buildPromoteScanDeps({ store, embedder, conflictStore });
693
+ const minOccurrences = Number(options.minOccurrences);
694
+ const minImportance = Number(options.minImportance);
695
+ const result = await scanMemoryPromotions(deps, options.scope, {
696
+ dryRun: !options.apply,
697
+ ...(Number.isFinite(minOccurrences) ? { minOccurrences } : {}),
698
+ ...(Number.isFinite(minImportance) ? { minImportance } : {}),
699
+ });
700
+ console.log(formatPromoteScanResult(result));
701
+ });
702
+
647
703
  program
648
704
  .command("export <query>")
649
705
  .description("导出检索与蒸馏结果到 markdown/json")
@@ -1054,7 +1110,7 @@ program
1054
1110
  continue;
1055
1111
  }
1056
1112
 
1057
- const record = await persistWorkflowPattern({ store, embedder, conflictStore }, input);
1113
+ const record = await persistWorkflowPattern({ store, embedder, conflictStore, auditLogger: createAuditLogger() }, input);
1058
1114
  existingKeys.add(key);
1059
1115
  stored.push({ title: record.title, id: record.id, scope: record.resolvedScope });
1060
1116
  }
@@ -1130,7 +1186,7 @@ program
1130
1186
  continue;
1131
1187
  }
1132
1188
 
1133
- const record = await persistCaseMemory({ store, embedder, conflictStore }, input);
1189
+ const record = await persistCaseMemory({ store, embedder, conflictStore, auditLogger: createAuditLogger() }, input);
1134
1190
  existingKeys.add(key);
1135
1191
  stored.push({ title: record.title, id: record.id, scope: record.resolvedScope });
1136
1192
  }
@@ -1208,7 +1264,7 @@ program
1208
1264
  continue;
1209
1265
  }
1210
1266
 
1211
- const record = await persistMemory({ store, embedder, conflictStore }, input);
1267
+ const record = await persistMemory({ store, embedder, conflictStore, auditLogger: createAuditLogger() }, input);
1212
1268
  existingKeys.add(storeMemoryIdentity({
1213
1269
  category: input.category,
1214
1270
  scope: record.resolvedScope,
@@ -1345,8 +1401,10 @@ program
1345
1401
  console.log(` ✅ ${config.embedding.model} (${testResult.dimensions}d)`);
1346
1402
 
1347
1403
  // LLM status — respect --no-llm flag
1348
- const effectiveLlm = options.noLlm ? null : llm;
1349
- if (options.noLlm) {
1404
+ // commander 把 --no-llm 解析成 options.llm=false(不产生 options.noLlm),对齐 --no-dedup 的取反键写法
1405
+ const noLlm = options.llm === false;
1406
+ const effectiveLlm = noLlm ? null : llm;
1407
+ if (noLlm) {
1350
1408
  console.log(" ⚠️ LLM: 已通过 --no-llm 禁用,原始对话将跳过不存入");
1351
1409
  } else if (llm) {
1352
1410
  const llmTest = await llm.test();
@@ -1376,15 +1434,23 @@ program
1376
1434
  const ingestOpts = { limit, verbose, noDedup, llm: effectiveLlm, recentHours };
1377
1435
 
1378
1436
  // CC Transcripts
1437
+ // CC 按 cwd 编码 project 子目录,所有有 jsonl 的子目录都要 ingest(不只是 jsonl 最多的那个)
1379
1438
  if (source === "all" || source === "cc") {
1380
1439
  console.log("📝 导入 Claude Code 对话...");
1381
1440
  const ccSource = config.sources.cc;
1382
1441
  if (ccSource) {
1383
- const ccPath = resolveSourcePath(ccSource.path, "cc");
1384
- const r = await ingestCCTranscripts(store, embedder, ccPath, ingestOpts);
1385
- results.push(r);
1386
- console.log(` ✅ CC: ${formatIngestSummary(r)}`);
1387
- maybeWarnHighCCDedupRate(r);
1442
+ const ccPaths = ccSource.path === "auto"
1443
+ ? detectCCPath()
1444
+ : [expandHome(ccSource.path)];
1445
+ if (ccPaths.length === 0) {
1446
+ throw new Error("Could not auto-detect Claude Code transcript path. Set sources.cc.path in config.json.");
1447
+ }
1448
+ for (const ccPath of ccPaths) {
1449
+ const r = await ingestCCTranscripts(store, embedder, ccPath, ingestOpts);
1450
+ results.push(r);
1451
+ console.log(` ✅ CC [${basename(ccPath)}]: ${formatIngestSummary(r)}`);
1452
+ maybeWarnHighCCDedupRate(r);
1453
+ }
1388
1454
  }
1389
1455
  }
1390
1456
 
@@ -1403,7 +1469,7 @@ program
1403
1469
  limit,
1404
1470
  verbose,
1405
1471
  noDedup,
1406
- llm,
1472
+ llm: effectiveLlm,
1407
1473
  recentHours,
1408
1474
  });
1409
1475
  results.push(r);
@@ -1436,7 +1502,7 @@ program
1436
1502
  const r = await ingestMarkdownFiles(store, embedder, memPath, "memory", {
1437
1503
  verbose,
1438
1504
  noDedup,
1439
- llm,
1505
+ llm: effectiveLlm,
1440
1506
  recentHours,
1441
1507
  });
1442
1508
  results.push(r);
@@ -1502,6 +1568,70 @@ program
1502
1568
  console.log();
1503
1569
  });
1504
1570
 
1571
+ // ─── dream ───────────────────────────────────────────────────────────────────
1572
+ // P3 调度化入口:launchd 经 scripts/dream-consolidation.sh 调用本命令,
1573
+ // 输出 [[DREAM_STATUS]] ok|skip|blocked 标记供脚本判定(同 weekly-distill 的
1574
+ // [[DISTILL_STATUS]] 模式)。dream 是 auto-gc / usageStatus 快照 / consolidation
1575
+ // 的唯一生产触发点——没有它们整条维护链就是死代码(2026-06-11 临床重审实锤)。
1576
+
1577
+ program
1578
+ .command("dream")
1579
+ .description("运行 dream 离线整固管线 (Orient/Gather/Consolidate/Prune)")
1580
+ .option("--scope <scope>", "目标 scope(前缀或精确)", "cc")
1581
+ .option("--auto", "扫描 activity-counter 中写计数达标的所有 scope,逐个跑 dream(单 scope 失败不阻断其他)")
1582
+ .option("--force", "跳过 min-writes 门槛强制运行")
1583
+ .action(async (options: { scope: string; auto?: boolean; force?: boolean }) => {
1584
+ const config = loadConfig();
1585
+ const { store, embedder, llm } = createComponents(config);
1586
+ // Same activity-stats file the store writes per-scope counts to (beside the LanceDB dir).
1587
+ const activityStatsPath = join(dirname(store.dbPath), "activity-stats.json");
1588
+ // KG evidence for triple-overlap merging — same gate as the MCP server
1589
+ const kgStore = isKGModeEnabled() ? new KGStore({ dbPath: store.dbPath }) : null;
1590
+
1591
+ if (options.auto) {
1592
+ const scopes = listScopesAboveThreshold(DEFAULT_DREAM_CONFIG.minWritesForDream, { statsPath: activityStatsPath });
1593
+ if (scopes.length === 0) {
1594
+ console.log("dream --auto: 无达标 scope(写计数均低于门槛)");
1595
+ console.log("[[DREAM_STATUS]] skip");
1596
+ process.exit(0);
1597
+ }
1598
+ console.log(`dream --auto: ${scopes.length} 个达标 scope → ${scopes.join(", ")}`);
1599
+ let anyFailed = false;
1600
+ for (const scope of scopes) {
1601
+ try {
1602
+ const result = await runDream({ store, llm, embedder, scope, force: Boolean(options.force), activityStatsPath, kgStore });
1603
+ console.log(`[scope=${scope}] ${formatDreamResult(result)}`);
1604
+ } catch (err) {
1605
+ anyFailed = true;
1606
+ // 单 scope 失败不阻断其他 scope
1607
+ console.error(`[scope=${scope}] dream failed: ${err instanceof Error ? err.message : String(err)}`);
1608
+ }
1609
+ }
1610
+ console.log(anyFailed ? "[[DREAM_STATUS]] blocked" : "[[DREAM_STATUS]] ok");
1611
+ process.exit(anyFailed ? 1 : 0);
1612
+ }
1613
+
1614
+ try {
1615
+ const result = await runDream({
1616
+ store,
1617
+ llm,
1618
+ embedder,
1619
+ scope: options.scope,
1620
+ force: Boolean(options.force),
1621
+ activityStatsPath,
1622
+ kgStore,
1623
+ });
1624
+
1625
+ console.log(formatDreamResult(result));
1626
+ console.log(result.ran ? "[[DREAM_STATUS]] ok" : "[[DREAM_STATUS]] skip");
1627
+ process.exit(0);
1628
+ } catch (err) {
1629
+ console.error(`dream failed: ${err instanceof Error ? err.message : String(err)}`);
1630
+ console.log("[[DREAM_STATUS]] blocked");
1631
+ process.exit(1);
1632
+ }
1633
+ });
1634
+
1505
1635
  // ─── stats ───────────────────────────────────────────────────────────────────
1506
1636
 
1507
1637
  program
@@ -1723,6 +1853,7 @@ program
1723
1853
  .option("--task <task>", "相关任务")
1724
1854
  .option("--tags <tags>", "逗号分隔 tags")
1725
1855
  .option("--tools <tools>", "逗号分隔 tools")
1856
+ .option("--idempotency-key <key>", "同一请求重试时用于替换旧 observation 的稳定键")
1726
1857
  .option("--json", "输出 JSON")
1727
1858
  .action(async (workflowId, summary, options) => {
1728
1859
  const store = new WorkflowObservationStore();
@@ -1736,6 +1867,7 @@ program
1736
1867
  task: options.task,
1737
1868
  tags: splitCsvOption(options.tags),
1738
1869
  tools: splitCsvOption(options.tools),
1870
+ idempotencyKey: options.idempotencyKey,
1739
1871
  });
1740
1872
  const stored = await store.save(record);
1741
1873
  if (options.json) {
@@ -261,50 +261,68 @@ export function buildContradictPatch(
261
261
  // Convenience wrappers (for callers who DO have a store)
262
262
  // ---------------------------------------------------------------------------
263
263
 
264
+ /** 单写通道端口:读改写经 store.patchMetadata per-id 队列,防并发覆盖。 */
265
+ type ConfidenceStorePort = {
266
+ patchMetadata(
267
+ id: string,
268
+ patchFn: (meta: Record<string, unknown>, entry: MemoryEntry) => Record<string, unknown>,
269
+ scopeFilter?: string[],
270
+ ): Promise<MemoryEntry | null>;
271
+ };
272
+
273
+ function applyConfidencePatch(
274
+ store: ConfidenceStorePort,
275
+ entryId: string,
276
+ scope: string,
277
+ build: (entry: MemoryEntry) => ConfidencePatch | null,
278
+ ): Promise<ConfidenceUpdate | null> {
279
+ let update: ConfidenceUpdate | null = null;
280
+ return store
281
+ .patchMetadata(
282
+ entryId,
283
+ (meta, entry) => {
284
+ const patch = build(entry);
285
+ if (!patch) return meta;
286
+ update = patch.update;
287
+ return patch.metadata;
288
+ },
289
+ [scope],
290
+ )
291
+ .then(result => (result === null ? null : update));
292
+ }
293
+
264
294
  /**
265
295
  * Confirm a memory and write back to store. Convenience wrapper.
266
296
  */
267
297
  export async function confirmMemory(
268
- store: { getById(id: string): Promise<MemoryEntry | null>; update(id: string, data: { metadata: string }, scopes: string[]): Promise<void> },
298
+ store: ConfidenceStorePort,
269
299
  entryId: string,
270
300
  scope: string,
271
301
  ): Promise<ConfidenceUpdate | null> {
272
- const entry = await store.getById(entryId);
273
- const patch = buildConfirmPatch(entry);
274
- if (!patch) return null;
275
- await store.update(entryId, { metadata: JSON.stringify(patch.metadata) }, [scope]);
276
- return patch.update;
302
+ return applyConfidencePatch(store, entryId, scope, entry => buildConfirmPatch(entry));
277
303
  }
278
304
 
279
305
  /**
280
306
  * Correct a memory and write back to store. Convenience wrapper.
281
307
  */
282
308
  export async function correctMemory(
283
- store: { getById(id: string): Promise<MemoryEntry | null>; update(id: string, data: { metadata: string }, scopes: string[]): Promise<void> },
309
+ store: ConfidenceStorePort,
284
310
  entryId: string,
285
311
  scope: string,
286
312
  correctedById?: string,
287
313
  ): Promise<ConfidenceUpdate | null> {
288
- const entry = await store.getById(entryId);
289
- const patch = buildCorrectPatch(entry, correctedById);
290
- if (!patch) return null;
291
- await store.update(entryId, { metadata: JSON.stringify(patch.metadata) }, [scope]);
292
- return patch.update;
314
+ return applyConfidencePatch(store, entryId, scope, entry => buildCorrectPatch(entry, correctedById));
293
315
  }
294
316
 
295
317
  /**
296
318
  * Contradict a memory and write back to store. Convenience wrapper.
297
319
  */
298
320
  export async function contradictMemory(
299
- store: { getById(id: string): Promise<MemoryEntry | null>; update(id: string, data: { metadata: string }, scopes: string[]): Promise<void> },
321
+ store: ConfidenceStorePort,
300
322
  entryId: string,
301
323
  scope: string,
302
324
  ): Promise<ConfidenceUpdate | null> {
303
- const entry = await store.getById(entryId);
304
- const patch = buildContradictPatch(entry);
305
- if (!patch) return null;
306
- await store.update(entryId, { metadata: JSON.stringify(patch.metadata) }, [scope]);
307
- return patch.update;
325
+ return applyConfidencePatch(store, entryId, scope, entry => buildContradictPatch(entry));
308
326
  }
309
327
 
310
328
  // ---------------------------------------------------------------------------