recallnest 2.2.1 → 2.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/CHANGELOG.md +150 -0
  2. package/README.md +46 -35
  3. package/README_CN.md +41 -26
  4. package/assets/ui/app.js +1031 -0
  5. package/assets/ui/index.html +224 -0
  6. package/assets/ui/styles.css +781 -0
  7. package/config.json.example +4 -1
  8. package/connectors/examples/email-adapter.ts +73 -0
  9. package/connectors/examples/logs-adapter.ts +103 -0
  10. package/connectors/examples/rss-adapter.ts +95 -0
  11. package/docs/api-reference.md +1291 -0
  12. package/docs/architecture.md +168 -0
  13. package/docs/connector-spec.md +126 -0
  14. package/eval/cases-canary.json +58 -0
  15. package/eval/cases-scope-robustness.json +278 -0
  16. package/eval/cases.json +117 -20
  17. package/package.json +22 -1
  18. package/src/__tests__/access-tracker-novelty.test.ts +238 -0
  19. package/src/__tests__/activity-counter.test.ts +125 -0
  20. package/src/__tests__/adaptive-fidelity.test.ts +102 -0
  21. package/src/__tests__/admission-control.test.ts +206 -0
  22. package/src/__tests__/anchor-generator.test.ts +67 -0
  23. package/src/__tests__/auto-consolidation.test.ts +168 -0
  24. package/src/__tests__/auto-recall.test.ts +131 -0
  25. package/src/__tests__/babel-integration.test.ts +144 -0
  26. package/src/__tests__/batch-dedup.test.ts +65 -0
  27. package/src/__tests__/canary-eval.test.ts +114 -0
  28. package/src/__tests__/capture-engine.test.ts +1238 -0
  29. package/src/__tests__/capture-heuristic.test.ts +171 -0
  30. package/src/__tests__/cascade-forget.test.ts +210 -0
  31. package/src/__tests__/category-min-scores.test.ts +33 -0
  32. package/src/__tests__/cc8-cc1-essential-context.test.ts +298 -0
  33. package/src/__tests__/circuit-breaker.test.ts +99 -0
  34. package/src/__tests__/cluster-consolidation.test.ts +782 -0
  35. package/src/__tests__/confidence-metadata.test.ts +125 -0
  36. package/src/__tests__/confidence-tracker.test.ts +261 -0
  37. package/src/__tests__/conflict-advisor.test.ts +206 -0
  38. package/src/__tests__/conflict-escalation.test.ts +149 -0
  39. package/src/__tests__/conflict-lifecycle.test.ts +83 -0
  40. package/src/__tests__/conflict-output.test.ts +85 -0
  41. package/src/__tests__/conflict-store.test.ts +374 -0
  42. package/src/__tests__/connector-v1.test.ts +181 -0
  43. package/src/__tests__/consolidation-engine.test.ts +333 -0
  44. package/src/__tests__/constructive-retrieval.test.ts +509 -0
  45. package/src/__tests__/context-collapse-renderer.test.ts +214 -0
  46. package/src/__tests__/context-composer-pins.test.ts +64 -0
  47. package/src/__tests__/context-composer-task-selection.test.ts +24 -0
  48. package/src/__tests__/context-reconstructor.test.ts +65 -0
  49. package/src/__tests__/context-renderer.test.ts +91 -0
  50. package/src/__tests__/continuity-eval.test.ts +275 -0
  51. package/src/__tests__/continuity-schema.test.ts +97 -0
  52. package/src/__tests__/contracts.test.ts +216 -0
  53. package/src/__tests__/conversation-importer.test.ts +692 -0
  54. package/src/__tests__/core-recall-eval.test.ts +235 -0
  55. package/src/__tests__/core-summary.test.ts +109 -0
  56. package/src/__tests__/data-checkup.test.ts +276 -0
  57. package/src/__tests__/decay-exemption.test.ts +84 -0
  58. package/src/__tests__/dedup-actions.test.ts +58 -0
  59. package/src/__tests__/dedup-guard.test.ts +39 -0
  60. package/src/__tests__/deterministic-id.test.ts +39 -0
  61. package/src/__tests__/distill-lock.test.ts +305 -0
  62. package/src/__tests__/distill-session-argshape.test.ts +79 -0
  63. package/src/__tests__/doctor.test.ts +319 -0
  64. package/src/__tests__/dream-pipeline.test.ts +321 -0
  65. package/src/__tests__/drill-down.test.ts +108 -0
  66. package/src/__tests__/embedder.test.ts +122 -0
  67. package/src/__tests__/emotion-decay.test.ts +46 -0
  68. package/src/__tests__/emotion-detector.test.ts +56 -0
  69. package/src/__tests__/emotion-phase1-integration.test.ts +153 -0
  70. package/src/__tests__/emotion-retrieval.test.ts +56 -0
  71. package/src/__tests__/entity-resolver.test.ts +58 -0
  72. package/src/__tests__/env-config.test.ts +151 -0
  73. package/src/__tests__/error-signature-boost.test.ts +87 -0
  74. package/src/__tests__/error-signature.test.ts +95 -0
  75. package/src/__tests__/error-taxonomy.test.ts +129 -0
  76. package/src/__tests__/ethics-layer.test.ts +476 -0
  77. package/src/__tests__/event-segmenter.test.ts +153 -0
  78. package/src/__tests__/feature-flags-integration.test.ts +54 -0
  79. package/src/__tests__/frequency-tracker.test.ts +113 -0
  80. package/src/__tests__/graph-export.test.ts +489 -0
  81. package/src/__tests__/hotness-blend.test.ts +76 -0
  82. package/src/__tests__/ingest-chunking.test.ts +44 -0
  83. package/src/__tests__/ingest.test.ts +360 -0
  84. package/src/__tests__/interference-detector.test.ts +203 -0
  85. package/src/__tests__/kg-extractor.test.ts +192 -0
  86. package/src/__tests__/kg-store.test.ts +300 -0
  87. package/src/__tests__/llm-consolidation.test.ts +226 -0
  88. package/src/__tests__/maintenance-locks.test.ts +174 -0
  89. package/src/__tests__/mcp-efficiency.test.ts +288 -0
  90. package/src/__tests__/mcp-related-scopes.test.ts +151 -0
  91. package/src/__tests__/memory-boundaries.test.ts +250 -0
  92. package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +345 -0
  93. package/src/__tests__/memory-evolution.test.ts +271 -0
  94. package/src/__tests__/memory-health-rebalance.test.ts +144 -0
  95. package/src/__tests__/memory-lint.test.ts +421 -0
  96. package/src/__tests__/memory-output.test.ts +215 -0
  97. package/src/__tests__/memory-promotion.test.ts +315 -0
  98. package/src/__tests__/migrate-dedup-ids.test.ts +65 -0
  99. package/src/__tests__/multi-vector.test.ts +242 -0
  100. package/src/__tests__/narrative-architecture.test.ts +456 -0
  101. package/src/__tests__/noise-filter.test.ts +205 -0
  102. package/src/__tests__/obsidian-connector.test.ts +363 -0
  103. package/src/__tests__/omitted-reporting.test.ts +98 -0
  104. package/src/__tests__/pending-review.test.ts +77 -0
  105. package/src/__tests__/pii-redact.test.ts +314 -0
  106. package/src/__tests__/post-retrieval-filter.test.ts +178 -0
  107. package/src/__tests__/ppr-traversal.test.ts +256 -0
  108. package/src/__tests__/prediction-engine.test.ts +349 -0
  109. package/src/__tests__/predictive-prospective-memory.test.ts +416 -0
  110. package/src/__tests__/preference-guard-smoke.test.ts +211 -0
  111. package/src/__tests__/preference-matcher.test.ts +230 -0
  112. package/src/__tests__/preference-slots.test.ts +205 -0
  113. package/src/__tests__/prospective-memory.test.ts +283 -0
  114. package/src/__tests__/query-entity-detector.test.ts +171 -0
  115. package/src/__tests__/query-expander-alias.test.ts +146 -0
  116. package/src/__tests__/query-expander.test.ts +34 -0
  117. package/src/__tests__/recall-governor.test.ts +227 -0
  118. package/src/__tests__/recall-mode.test.ts +64 -0
  119. package/src/__tests__/result-synthesizer.test.ts +198 -0
  120. package/src/__tests__/resume-context.test.ts +4898 -0
  121. package/src/__tests__/retrieval-trace.test.ts +213 -0
  122. package/src/__tests__/retriever-boundary-weight.test.ts +136 -0
  123. package/src/__tests__/retroactive-boost.test.ts +223 -0
  124. package/src/__tests__/salience-scoring.test.ts +106 -0
  125. package/src/__tests__/scope-inventory.test.ts +200 -0
  126. package/src/__tests__/scope-policy.test.ts +69 -0
  127. package/src/__tests__/scope-suggester.test.ts +80 -0
  128. package/src/__tests__/session-checkpoint.test.ts +291 -0
  129. package/src/__tests__/session-distiller.test.ts +640 -0
  130. package/src/__tests__/session-gc.test.ts +215 -0
  131. package/src/__tests__/skill-engine.test.ts +552 -0
  132. package/src/__tests__/skill-promotion.test.ts +601 -0
  133. package/src/__tests__/skill-verifier.test.ts +122 -0
  134. package/src/__tests__/source-heartbeat.test.ts +250 -0
  135. package/src/__tests__/store-patch-metadata.test.ts +205 -0
  136. package/src/__tests__/store-scope-audit.test.ts +81 -0
  137. package/src/__tests__/store-upsert.test.ts +184 -0
  138. package/src/__tests__/supersede-chain.test.ts +154 -0
  139. package/src/__tests__/temporal-parser.test.ts +241 -0
  140. package/src/__tests__/temporal-validity.test.ts +160 -0
  141. package/src/__tests__/term-registry.test.ts +121 -0
  142. package/src/__tests__/tool-error-surfacing.test.ts +98 -0
  143. package/src/__tests__/tool-output-compressor.test.ts +160 -0
  144. package/src/__tests__/topic-tag.test.ts +113 -0
  145. package/src/__tests__/trusted-memory.test.ts +839 -0
  146. package/src/__tests__/usage-tracker.test.ts +287 -0
  147. package/src/__tests__/version-manager.test.ts +213 -0
  148. package/src/__tests__/workflow-observation-skill-outcome.test.ts +352 -0
  149. package/src/__tests__/workflow-observation.test.ts +433 -0
  150. package/src/__tests__/write-verifier.test.ts +122 -0
  151. package/src/access-tracker.ts +38 -32
  152. package/src/activity-counter.ts +46 -18
  153. package/src/aliases.ts +133 -0
  154. package/src/api-server.ts +27 -7
  155. package/src/audit-log.ts +4 -2
  156. package/src/auto-consolidation.ts +8 -3
  157. package/src/auto-gc.ts +85 -21
  158. package/src/capture-engine.ts +54 -21
  159. package/src/cli.ts +199 -37
  160. package/src/confidence-tracker.ts +36 -18
  161. package/src/connector-types.ts +57 -0
  162. package/src/consolidation-engine.ts +109 -11
  163. package/src/context-composer.ts +8 -1
  164. package/src/conversation-importer.ts +25 -1
  165. package/src/data-checkup.ts +126 -7
  166. package/src/distill-lock.ts +194 -15
  167. package/src/doctor.ts +38 -2
  168. package/src/dream-pipeline.ts +118 -34
  169. package/src/env-config.ts +77 -0
  170. package/src/error-signature.ts +153 -0
  171. package/src/error-taxonomy.ts +115 -0
  172. package/src/eval.ts +251 -6
  173. package/src/idempotency.ts +8 -0
  174. package/src/ingest.ts +234 -10
  175. package/src/kg-extractor.ts +6 -2
  176. package/src/kg-store.ts +236 -44
  177. package/src/llm-consolidation.ts +2 -1
  178. package/src/mcp-server.ts +85 -1799
  179. package/src/mcp-tool-deps.ts +48 -0
  180. package/src/mcp-tools-advanced.ts +941 -0
  181. package/src/mcp-tools-core.ts +633 -0
  182. package/src/mcp-tools-governance.ts +475 -0
  183. package/src/memory-boundaries.ts +3 -1
  184. package/src/memory-evolution.ts +73 -1
  185. package/src/memory-lint.ts +183 -10
  186. package/src/memory-output.ts +114 -3
  187. package/src/memory-promotion.ts +246 -0
  188. package/src/memory-schema.ts +25 -2
  189. package/src/memory-store-port.ts +43 -0
  190. package/src/multi-vector.ts +2 -1
  191. package/src/narrative-schema.ts +2 -1
  192. package/src/noise-filter.ts +57 -6
  193. package/src/obsidian-connector.ts +282 -0
  194. package/src/pii-detector.ts +157 -5
  195. package/src/query-expander.ts +90 -9
  196. package/src/result-synthesizer.ts +2 -1
  197. package/src/retention-policy.ts +2 -1
  198. package/src/retriever.ts +259 -139
  199. package/src/runtime-config.ts +8 -1
  200. package/src/scope-suggester.ts +113 -0
  201. package/src/session-distiller.ts +23 -2
  202. package/src/session-engine.ts +2 -1
  203. package/src/session-schema.ts +3 -2
  204. package/src/session-store.ts +5 -0
  205. package/src/skill-engine.ts +78 -0
  206. package/src/skill-promotion.ts +247 -47
  207. package/src/skill-schema.ts +16 -3
  208. package/src/skill-verifier.ts +152 -0
  209. package/src/source-heartbeat.ts +108 -0
  210. package/src/store.ts +267 -38
  211. package/src/tool-output-compressor.ts +3 -1
  212. package/src/ui-server.ts +2 -1
  213. package/src/usage-tracker.ts +242 -0
  214. package/src/version-manager.ts +3 -2
  215. package/src/workflow-observation-engine.ts +2 -1
  216. package/src/workflow-observation-schema.ts +2 -1
  217. package/src/workflow-observation-store.ts +6 -2
  218. package/.env +0 -4
  219. package/CLAUDE.md +0 -46
  220. package/config.json +0 -55
  221. package/daily-recall.sh +0 -110
  222. package/logs/daily-recall.error.log +0 -0
  223. package/logs/daily-recall.log +0 -30
  224. package/logs/ingest-2026-04-11.log +0 -110
  225. package/logs/ingest-2026-04-12.log +0 -90
  226. package/logs/launchagent-stderr.log +0 -0
  227. package/logs/launchagent-stdout.log +0 -0
@@ -1,7 +1,7 @@
1
1
  import type { Embedder } from "./embedder.js";
2
2
  import { detectLang, tokenizeFts } from "./language-hook.js";
3
3
  import { generateAnchor } from "./anchor-generator.js";
4
- import { incrementWriteCount } from "./activity-counter.js";
4
+ import { extractErrorSignatures } from "./error-signature.js";
5
5
  import { verifyWrite } from "./write-verifier.js";
6
6
  // batchInternalDedup is available in ingest.ts for large-batch scenarios;
7
7
  // persistMemoryBatch relies on per-item conflict detection (A-2) instead.
@@ -56,7 +56,7 @@ import {
56
56
  import { matchPreference, applyPreferenceMatch } from "./preference-matcher.js";
57
57
  import type { LLMClient } from "./llm-client.js";
58
58
  import type { KGExtractor } from "./kg-extractor.js";
59
- import { scanForPII } from "./pii-detector.js";
59
+ import { scanForPII, redactSecrets } from "./pii-detector.js";
60
60
  import type { AuditLogger } from "./audit-log.js";
61
61
  import { checkAdmission, type ScopeRateLimiter, type AdmissionConfig } from "./admission-control.js";
62
62
  import { tagNarrativeIfEnabled } from "./narrative-tagger.js";
@@ -699,6 +699,17 @@ function buildCaseMemoryMetadata(
699
699
  tools: input.tools,
700
700
  },
701
701
  };
702
+ // A1: error-signature 指纹(读写对称,P3-A 检索复用同一抽取器)。
703
+ // 在 build metadata 前算好作为 extra 写入,不 post-write update(Codex 验证要求)。
704
+ const errorSignature = extractErrorSignatures({
705
+ problem: input.problem,
706
+ context: input.context,
707
+ outcome: input.outcome,
708
+ solutionSteps: input.solutionSteps,
709
+ });
710
+ if (errorSignature.length > 0) cmExtra.error_signature = errorSignature;
711
+ // A1: break-loop 五维(可选,向后兼容;不提供则普通 case)。
712
+ if (input.debugFraming) cmExtra.debugFraming = input.debugFraming;
702
713
  const anchor = generateAnchor(input.title + ": " + input.problem, cmExtra);
703
714
  if (anchor) cmExtra.anchor = anchor;
704
715
  return buildStructuredMetadata({
@@ -746,7 +757,16 @@ export async function persistMemory(
746
757
  deps: PersistMemoryDeps,
747
758
  rawInput: unknown,
748
759
  ): Promise<StoredMemoryRecord> {
749
- const input = StoreMemoryInputSchema.parse(rawInput);
760
+ let input = StoreMemoryInputSchema.parse(rawInput);
761
+
762
+ // F-3b: Scan THEN redact, before the text reaches any external surface —
763
+ // LLM summary/importance calls, embedding API, storage, audit log, KG.
764
+ // Scan first so the piiWarning below still reflects what the raw text held.
765
+ const piiScan = scanForPII(input.text);
766
+ const redaction = redactSecrets(input.text);
767
+ if (redaction.redacted > 0) {
768
+ input = { ...input, text: redaction.text };
769
+ }
750
770
 
751
771
  // F3: Extract temporal validity params before Zod strips them
752
772
  const rawObj = rawInput as Record<string, unknown> | null;
@@ -801,6 +821,18 @@ export async function persistMemory(
801
821
  deps.admissionConfig,
802
822
  );
803
823
  if (admission.verdict === "rejected") {
824
+ // Gemini #26523 lesson: rejected writes must stay queryable after the fact,
825
+ // not just echo back to the caller — otherwise they become invisible noise.
826
+ try {
827
+ deps.auditLogger?.log({
828
+ operation: "reject",
829
+ scope: admissionScope,
830
+ actor: input.source,
831
+ details: `${admission.reason ?? "unknown"}: ${input.text.slice(0, 100)}`,
832
+ });
833
+ } catch {
834
+ // Audit must never block memory writes
835
+ }
804
836
  return {
805
837
  id: `rejected-${Date.now()}`,
806
838
  text: input.text,
@@ -812,6 +844,7 @@ export async function persistMemory(
812
844
  resolvedScope: admissionScope,
813
845
  storedAt: new Date().toISOString(),
814
846
  disposition: "rejected" as const,
847
+ ...(admission.reason ? { rejectionReason: admission.reason } : {}),
815
848
  };
816
849
  }
817
850
 
@@ -937,15 +970,16 @@ export async function persistMemory(
937
970
  }
938
971
 
939
972
  // F-3: PII detection — inject warning into metadata if PII found.
940
- // Non-blocking: PII does not prevent the write.
973
+ // Non-blocking: PII does not prevent the write. Uses the pre-redaction scan
974
+ // from the top of persistMemory (scanning here would see scrubbed text).
941
975
  let metadata = buildStoreMemoryMetadata(input, canonicalKey);
942
- const piiResult = scanForPII(input.text);
943
- if (piiResult.hasPII) {
976
+ if (piiScan.hasPII) {
944
977
  const parsed: Record<string, unknown> = JSON.parse(metadata);
945
978
  parsed.piiWarning = {
946
- summary: piiResult.summary,
947
- detections: piiResult.detections.length,
948
- severity: piiResult.detections.some(d => d.severity === "high") ? "high" : "medium",
979
+ summary: piiScan.summary,
980
+ detections: piiScan.detections.length,
981
+ severity: piiScan.detections.some(d => d.severity === "high") ? "high" : "medium",
982
+ ...(redaction.redacted > 0 ? { redacted: redaction.redacted } : {}),
949
983
  };
950
984
  metadata = JSON.stringify(parsed);
951
985
  }
@@ -1031,14 +1065,9 @@ export async function persistMemory(
1031
1065
  // Audit must never block memory writes
1032
1066
  }
1033
1067
 
1034
- // HP-3: Activity counter — track non-dedup writes for distill trigger
1035
- if (disposition !== "deduped") {
1036
- try {
1037
- incrementWriteCount();
1038
- } catch {
1039
- // Activity tracking must never block memory writes
1040
- }
1041
- }
1068
+ // HP-3 activity counter moved to the store layer (store/upsert/storeBatch count per
1069
+ // scope): that is the necessary chokepoint for every write source (capture/ingest/
1070
+ // store_memory/consolidation), and per-scope counts drive per-scope dream activation.
1042
1071
 
1043
1072
  // HP-2: Post-write verification — async, non-blocking
1044
1073
  if (disposition !== "deduped" && deps.store.get) {
@@ -1063,7 +1092,7 @@ export async function persistMemory(
1063
1092
  input.category !== "preferences" &&
1064
1093
  inferImplicitUsageSlot(input.text)
1065
1094
  ) {
1066
- const prefCanonicalKey = buildDefaultCanonicalKey("preferences", input.text);
1095
+ const prefCanonicalKey = buildDefaultCanonicalKey({ category: "preferences", text: input.text });
1067
1096
  const prefSlot = inferImplicitUsageSlot(input.text);
1068
1097
  const prefMetadata = buildStructuredMetadata({
1069
1098
  source: input.source,
@@ -1098,7 +1127,8 @@ export async function persistWorkflowPattern(
1098
1127
  const input = WorkflowPatternInputSchema.parse(rawInput);
1099
1128
  const resolvedScope = resolveScope(input);
1100
1129
  const tags = mergeTags(input.tags, ["workflow", "pattern"]);
1101
- const text = buildWorkflowPatternText(input);
1130
+ // F-3b: scrub before embedding/storage (this path bypasses persistMemory)
1131
+ const text = redactSecrets(buildWorkflowPatternText(input)).text;
1102
1132
  const vector = await deps.embedder.embedPassage(text);
1103
1133
  const language = detectLang(text);
1104
1134
  const fts_text = tokenizeFts(text, language);
@@ -1142,7 +1172,8 @@ export async function persistCaseMemory(
1142
1172
  const input = CaseMemoryInputSchema.parse(rawInput);
1143
1173
  const resolvedScope = resolveScope(input);
1144
1174
  const tags = mergeTags(input.tags, ["case", "solution"]);
1145
- const text = buildCaseMemoryText(input);
1175
+ // F-3b: scrub before embedding/storage (this path bypasses persistMemory)
1176
+ const text = redactSecrets(buildCaseMemoryText(input)).text;
1146
1177
  const vector = await deps.embedder.embedPassage(text);
1147
1178
  const language = detectLang(text);
1148
1179
  const fts_text = tokenizeFts(text, language);
@@ -1202,7 +1233,9 @@ export async function promoteMemory(
1202
1233
  }
1203
1234
 
1204
1235
  const category = inferPromotedCategory(input, sourceEntry);
1205
- const text = input.text || sourceEntry.text;
1236
+ // F-3b: scrub caller-supplied text AND legacy stored text (pre-redaction
1237
+ // era entries may still hold secrets; promotion is a natural wash point)
1238
+ const text = redactSecrets(input.text || sourceEntry.text).text;
1206
1239
  const canonicalKey = resolveCanonicalKey({
1207
1240
  category,
1208
1241
  text,
package/src/cli.ts CHANGED
@@ -11,7 +11,7 @@
11
11
 
12
12
  import { Command } from "commander";
13
13
  import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
14
- import { join, resolve } from "node:path";
14
+ import { basename, dirname, join, resolve } from "node:path";
15
15
  import { homedir } from "node:os";
16
16
 
17
17
  import { metaDir } from "./compat.js";
@@ -21,6 +21,8 @@ import { distillResults, formatExplainResults, formatSearchResults, selectBriefS
21
21
  import { archiveDirtyBriefAsset, assetSummaryLine, buildBriefAsset, buildPinAsset, getExportsDir, listDirtyBriefAssets, listMemoryAssets, listPinAssets, pinSummaryLine, saveBriefAsset, savePinAsset, writeExportArtifact } from "./memory-assets.js";
22
22
  import { indexAsset, indexPinnedAsset } from "./asset-sync.js";
23
23
  import { createComponents, createStoreOnly, expandHome, loadConfig, loadDotEnv, type LocalMemoryConfig } from "./runtime-config.js";
24
+ import { KGStore } from "./kg-store.js";
25
+ import { isKGModeEnabled } from "./kg-extractor.js";
24
26
  import {
25
27
  formatDedupReasonSummary,
26
28
  getDedupSkipRate,
@@ -29,9 +31,14 @@ import {
29
31
  ingestCodexSessions,
30
32
  ingestGeminiSessions,
31
33
  ingestMarkdownFiles,
34
+ ingestObsidianVault,
35
+ ingestConnectorFile,
32
36
  } from "./ingest.js";
33
37
  import { runDoctor, formatDoctorResults } from "./doctor.js";
34
38
  import { persistCaseMemory, persistMemory, persistWorkflowPattern } from "./capture-engine.js";
39
+ import { scanMemoryPromotions, buildPromoteScanDeps, formatPromoteScanResult } from "./memory-promotion.js";
40
+ import { runDream, formatDreamResult, DEFAULT_DREAM_CONFIG } from "./dream-pipeline.js";
41
+ import { listScopesAboveThreshold } from "./activity-counter.js";
35
42
  import {
36
43
  CaseMemoryInputSchema,
37
44
  type CaseMemoryInput,
@@ -45,6 +52,7 @@ import { ConflictStatusSchema } from "./conflict-schema.js";
45
52
  import { resolveConflictCandidate } from "./conflict-engine.js";
46
53
  import { escalateConflicts } from "./conflict-escalation.js";
47
54
  import { ConflictCandidateStore } from "./conflict-store.js";
55
+ import { createAuditLogger } from "./audit-log.js";
48
56
  import { formatConflictAudit, formatConflictAuditMarkdown, formatConflictClusters, formatConflictEscalation, formatConflictList, formatConflictRecord, formatConflictResolution } from "./conflict-output.js";
49
57
  import { CONFLICT_ATTENTION_LEVELS, parseConflictAttention, summarizeConflictLifecycle } from "./conflict-lifecycle.js";
50
58
  import { buildConflictAuditSummary, clusterConflicts, summarizeConflictAdvice } from "./conflict-advisor.js";
@@ -61,49 +69,66 @@ import {
61
69
  } from "./workflow-observation-scope-review.js";
62
70
  import { collectScopeInventory, formatScopeInventoryReport } from "./scope-inventory.js";
63
71
  import { buildRetrievalContext, resolveScopeSelection } from "./scope-policy.js";
72
+ import { createScopeSuggester } from "./scope-suggester.js";
64
73
 
65
74
  /**
66
- * Auto-detect Claude Code transcript directory.
67
- * Scans ~/.claude/projects/ for subdirectories containing .jsonl files.
75
+ * Auto-detect Claude Code transcript directories.
76
+ * Scans ~/.claude/projects/ for ALL subdirectories containing .jsonl files.
77
+ *
78
+ * CC 按 cwd 编码 project 子目录,凡是不在 ~ 下跑的会话(Codex spawn 的子进程 / cd 到 repo 跑 /
79
+ * bridge 临时工作目录)会落到 ~/.claude/projects/-XXX/ 别的子目录。旧版只挑 jsonl 最多的根
80
+ * 目录返回,导致 28+ 个子目录全失声。修复:返回所有有 jsonl 的子目录,调用方循环 ingest。
68
81
  */
69
- function detectCCPath(): string | null {
82
+ function detectCCPath(): string[] {
70
83
  const projectsDir = join(homedir(), ".claude", "projects");
71
- if (!existsSync(projectsDir)) return null;
84
+ if (!existsSync(projectsDir)) return [];
72
85
  try {
73
- const dirs = readdirSync(projectsDir, { withFileTypes: true })
86
+ return readdirSync(projectsDir, { withFileTypes: true })
74
87
  .filter(d => d.isDirectory())
75
- .map(d => join(projectsDir, d.name));
76
- // Pick the directory with most .jsonl files
77
- let best = "";
78
- let bestCount = 0;
79
- for (const dir of dirs) {
80
- const jsonlCount = readdirSync(dir).filter(f => f.endsWith(".jsonl")).length;
81
- if (jsonlCount > bestCount) {
82
- bestCount = jsonlCount;
83
- best = dir;
84
- }
85
- }
86
- return best || null;
88
+ .map(d => join(projectsDir, d.name))
89
+ .filter(dir => {
90
+ try { return readdirSync(dir).some(f => f.endsWith(".jsonl")); }
91
+ catch { return false; }
92
+ });
87
93
  } catch {
88
- return null;
94
+ return [];
89
95
  }
90
96
  }
91
97
 
98
+ /**
99
+ * Pick the CC project subdirectory with the most jsonl files (heuristic: user's primary cwd).
100
+ * Used by memory source which still expects a single directory.
101
+ */
102
+ function pickPrimaryCCPath(detected: string[]): string {
103
+ let best = detected[0];
104
+ let bestCount = 0;
105
+ for (const dir of detected) {
106
+ try {
107
+ const c = readdirSync(dir).filter(f => f.endsWith(".jsonl")).length;
108
+ if (c > bestCount) { bestCount = c; best = dir; }
109
+ } catch { /* skip */ }
110
+ }
111
+ return best;
112
+ }
113
+
92
114
  /**
93
115
  * Resolve "auto" paths in config to actual directories.
116
+ *
117
+ * Note: cc 分支只返回第一个 detect 到的目录(fallback),ingest 入口已改为直接调
118
+ * detectCCPath() 拿全量子目录循环处理。这里保留是为兼容潜在的其它调用方。
94
119
  */
95
120
  function resolveSourcePath(source: string, key: string): string {
96
121
  if (source !== "auto") return expandHome(source);
97
122
 
98
123
  if (key === "cc") {
99
124
  const detected = detectCCPath();
100
- if (!detected) throw new Error("Could not auto-detect Claude Code transcript path. Set sources.cc.path in config.json.");
101
- return detected;
125
+ if (detected.length === 0) throw new Error("Could not auto-detect Claude Code transcript path. Set sources.cc.path in config.json.");
126
+ return pickPrimaryCCPath(detected);
102
127
  }
103
128
  if (key === "memory") {
104
- const ccPath = detectCCPath();
105
- if (!ccPath) throw new Error("Could not auto-detect memory path. Set sources.memory.path in config.json.");
106
- return join(ccPath, "memory");
129
+ const detected = detectCCPath();
130
+ if (detected.length === 0) throw new Error("Could not auto-detect memory path. Set sources.memory.path in config.json.");
131
+ return join(pickPrimaryCCPath(detected), "memory");
107
132
  }
108
133
  throw new Error(`Cannot auto-detect path for source "${key}". Set it manually in config.json.`);
109
134
  }
@@ -145,7 +170,7 @@ async function runRetrievalView(
145
170
  },
146
171
  ) {
147
172
  const config = loadConfig();
148
- const { retriever, profile } = createComponents(config, options.profile);
173
+ const { retriever, profile, store } = createComponents(config, options.profile);
149
174
 
150
175
  const limit = parseLimitOption(options.limit, 5, 1, 20);
151
176
 
@@ -204,6 +229,17 @@ async function runRetrievalView(
204
229
  : formatSearchResults(results, context);
205
230
 
206
231
  console.log(rendered);
232
+
233
+ // P1-A: 显式 scope 0 命中 → 相近 scope 提示打到 stderr(不污染 stdout)
234
+ if (results.length === 0 && options.scope) {
235
+ try {
236
+ const suggest = createScopeSuggester(async () => (await store.stats()).scopeCounts);
237
+ const suggestions = (await suggest(options.scope)).filter(s => s !== options.scope);
238
+ if (suggestions.length > 0) {
239
+ console.error(`⚠️ scope '${options.scope}' 命中 0 条,相近 scope: [${suggestions.join(", ")}]`);
240
+ }
241
+ } catch { /* best-effort */ }
242
+ }
207
243
  }
208
244
 
209
245
  function parseLimitOption(value: string | undefined, fallback: number, min = 1, max = 50): number {
@@ -642,6 +678,28 @@ program
642
678
  ].join("\n"));
643
679
  });
644
680
 
681
+ program
682
+ .command("promote-scan")
683
+ .description("扫描 transcript 降级的 profile/preferences evidence,聚类高频高 importance 项晋升回 durable")
684
+ .requiredOption("--scope <scope>", "要扫描的 scope,如 cc:project:recallnest")
685
+ .option("--apply", "执行晋升;默认 dry-run 只预览")
686
+ .option("--min-occurrences <n>", "聚类最小成员数(默认 3)", "3")
687
+ .option("--min-importance <n>", "聚类最小平均 importance(默认 0.6)", "0.6")
688
+ .action(async (options) => {
689
+ const config = loadConfig();
690
+ const { store, embedder } = createComponents(config);
691
+ const conflictStore = new ConflictCandidateStore();
692
+ const deps = buildPromoteScanDeps({ store, embedder, conflictStore });
693
+ const minOccurrences = Number(options.minOccurrences);
694
+ const minImportance = Number(options.minImportance);
695
+ const result = await scanMemoryPromotions(deps, options.scope, {
696
+ dryRun: !options.apply,
697
+ ...(Number.isFinite(minOccurrences) ? { minOccurrences } : {}),
698
+ ...(Number.isFinite(minImportance) ? { minImportance } : {}),
699
+ });
700
+ console.log(formatPromoteScanResult(result));
701
+ });
702
+
645
703
  program
646
704
  .command("export <query>")
647
705
  .description("导出检索与蒸馏结果到 markdown/json")
@@ -1052,7 +1110,7 @@ program
1052
1110
  continue;
1053
1111
  }
1054
1112
 
1055
- const record = await persistWorkflowPattern({ store, embedder, conflictStore }, input);
1113
+ const record = await persistWorkflowPattern({ store, embedder, conflictStore, auditLogger: createAuditLogger() }, input);
1056
1114
  existingKeys.add(key);
1057
1115
  stored.push({ title: record.title, id: record.id, scope: record.resolvedScope });
1058
1116
  }
@@ -1128,7 +1186,7 @@ program
1128
1186
  continue;
1129
1187
  }
1130
1188
 
1131
- const record = await persistCaseMemory({ store, embedder, conflictStore }, input);
1189
+ const record = await persistCaseMemory({ store, embedder, conflictStore, auditLogger: createAuditLogger() }, input);
1132
1190
  existingKeys.add(key);
1133
1191
  stored.push({ title: record.title, id: record.id, scope: record.resolvedScope });
1134
1192
  }
@@ -1206,7 +1264,7 @@ program
1206
1264
  continue;
1207
1265
  }
1208
1266
 
1209
- const record = await persistMemory({ store, embedder, conflictStore }, input);
1267
+ const record = await persistMemory({ store, embedder, conflictStore, auditLogger: createAuditLogger() }, input);
1210
1268
  existingKeys.add(storeMemoryIdentity({
1211
1269
  category: input.category,
1212
1270
  scope: record.resolvedScope,
@@ -1306,6 +1364,8 @@ program
1306
1364
  .option("--no-dedup", "跳过向量去重")
1307
1365
  .option("--no-llm", "禁用 LLM 提取(原始对话将跳过不存入,排入待处理队列)")
1308
1366
  .option("--recent [hours]", "快速模式:只处理最近 N 小时修改的文件(默认 2 小时),跳过 ingested-files 检查")
1367
+ .option("--obsidian <vault-path>", "导入 Obsidian vault(自动检测 .obsidian/ 目录)")
1368
+ .option("--connector <file-path>", "导入 connector-v1 格式 JSON 文件")
1309
1369
  .action(async (options) => {
1310
1370
  const config = loadConfig();
1311
1371
  const { store, embedder, llm } = createComponents(config);
@@ -1341,8 +1401,10 @@ program
1341
1401
  console.log(` ✅ ${config.embedding.model} (${testResult.dimensions}d)`);
1342
1402
 
1343
1403
  // LLM status — respect --no-llm flag
1344
- const effectiveLlm = options.noLlm ? null : llm;
1345
- if (options.noLlm) {
1404
+ // commander 把 --no-llm 解析成 options.llm=false(不产生 options.noLlm),对齐 --no-dedup 的取反键写法
1405
+ const noLlm = options.llm === false;
1406
+ const effectiveLlm = noLlm ? null : llm;
1407
+ if (noLlm) {
1346
1408
  console.log(" ⚠️ LLM: 已通过 --no-llm 禁用,原始对话将跳过不存入");
1347
1409
  } else if (llm) {
1348
1410
  const llmTest = await llm.test();
@@ -1372,15 +1434,23 @@ program
1372
1434
  const ingestOpts = { limit, verbose, noDedup, llm: effectiveLlm, recentHours };
1373
1435
 
1374
1436
  // CC Transcripts
1437
+ // CC 按 cwd 编码 project 子目录,所有有 jsonl 的子目录都要 ingest(不只是 jsonl 最多的那个)
1375
1438
  if (source === "all" || source === "cc") {
1376
1439
  console.log("📝 导入 Claude Code 对话...");
1377
1440
  const ccSource = config.sources.cc;
1378
1441
  if (ccSource) {
1379
- const ccPath = resolveSourcePath(ccSource.path, "cc");
1380
- const r = await ingestCCTranscripts(store, embedder, ccPath, ingestOpts);
1381
- results.push(r);
1382
- console.log(` ✅ CC: ${formatIngestSummary(r)}`);
1383
- maybeWarnHighCCDedupRate(r);
1442
+ const ccPaths = ccSource.path === "auto"
1443
+ ? detectCCPath()
1444
+ : [expandHome(ccSource.path)];
1445
+ if (ccPaths.length === 0) {
1446
+ throw new Error("Could not auto-detect Claude Code transcript path. Set sources.cc.path in config.json.");
1447
+ }
1448
+ for (const ccPath of ccPaths) {
1449
+ const r = await ingestCCTranscripts(store, embedder, ccPath, ingestOpts);
1450
+ results.push(r);
1451
+ console.log(` ✅ CC [${basename(ccPath)}]: ${formatIngestSummary(r)}`);
1452
+ maybeWarnHighCCDedupRate(r);
1453
+ }
1384
1454
  }
1385
1455
  }
1386
1456
 
@@ -1399,7 +1469,7 @@ program
1399
1469
  limit,
1400
1470
  verbose,
1401
1471
  noDedup,
1402
- llm,
1472
+ llm: effectiveLlm,
1403
1473
  recentHours,
1404
1474
  });
1405
1475
  results.push(r);
@@ -1432,7 +1502,7 @@ program
1432
1502
  const r = await ingestMarkdownFiles(store, embedder, memPath, "memory", {
1433
1503
  verbose,
1434
1504
  noDedup,
1435
- llm,
1505
+ llm: effectiveLlm,
1436
1506
  recentHours,
1437
1507
  });
1438
1508
  results.push(r);
@@ -1440,6 +1510,32 @@ program
1440
1510
  }
1441
1511
  }
1442
1512
 
1513
+ // Obsidian vault (--obsidian <path>)
1514
+ if (options.obsidian) {
1515
+ console.log("🗃️ 导入 Obsidian vault...");
1516
+ const r = await ingestObsidianVault(store, embedder, options.obsidian, {
1517
+ verbose,
1518
+ noDedup,
1519
+ llm: effectiveLlm,
1520
+ });
1521
+ results.push(r);
1522
+ console.log(` ✅ Obsidian: ${formatIngestSummary(r)}`);
1523
+ }
1524
+
1525
+ // Connector-v1 file (--connector <path>)
1526
+ if (options.connector) {
1527
+ console.log("🔌 导入 connector-v1 文件...");
1528
+ const { readFileSync: readFs } = await import("node:fs");
1529
+ const connectorContent = readFs(options.connector, "utf-8");
1530
+ const r = await ingestConnectorFile(store, embedder, connectorContent, {
1531
+ verbose,
1532
+ noDedup,
1533
+ llm: effectiveLlm,
1534
+ });
1535
+ results.push(r);
1536
+ console.log(` ✅ Connector: ${formatIngestSummary(r)}`);
1537
+ }
1538
+
1443
1539
  // Summary
1444
1540
  console.log("\n📊 导入汇总:");
1445
1541
  let totalChunks = 0;
@@ -1472,6 +1568,70 @@ program
1472
1568
  console.log();
1473
1569
  });
1474
1570
 
1571
+ // ─── dream ───────────────────────────────────────────────────────────────────
1572
+ // P3 调度化入口:launchd 经 scripts/dream-consolidation.sh 调用本命令,
1573
+ // 输出 [[DREAM_STATUS]] ok|skip|blocked 标记供脚本判定(同 weekly-distill 的
1574
+ // [[DISTILL_STATUS]] 模式)。dream 是 auto-gc / usageStatus 快照 / consolidation
1575
+ // 的唯一生产触发点——没有它们整条维护链就是死代码(2026-06-11 临床重审实锤)。
1576
+
1577
+ program
1578
+ .command("dream")
1579
+ .description("运行 dream 离线整固管线 (Orient/Gather/Consolidate/Prune)")
1580
+ .option("--scope <scope>", "目标 scope(前缀或精确)", "cc")
1581
+ .option("--auto", "扫描 activity-counter 中写计数达标的所有 scope,逐个跑 dream(单 scope 失败不阻断其他)")
1582
+ .option("--force", "跳过 min-writes 门槛强制运行")
1583
+ .action(async (options: { scope: string; auto?: boolean; force?: boolean }) => {
1584
+ const config = loadConfig();
1585
+ const { store, embedder, llm } = createComponents(config);
1586
+ // Same activity-stats file the store writes per-scope counts to (beside the LanceDB dir).
1587
+ const activityStatsPath = join(dirname(store.dbPath), "activity-stats.json");
1588
+ // KG evidence for triple-overlap merging — same gate as the MCP server
1589
+ const kgStore = isKGModeEnabled() ? new KGStore({ dbPath: store.dbPath }) : null;
1590
+
1591
+ if (options.auto) {
1592
+ const scopes = listScopesAboveThreshold(DEFAULT_DREAM_CONFIG.minWritesForDream, { statsPath: activityStatsPath });
1593
+ if (scopes.length === 0) {
1594
+ console.log("dream --auto: 无达标 scope(写计数均低于门槛)");
1595
+ console.log("[[DREAM_STATUS]] skip");
1596
+ process.exit(0);
1597
+ }
1598
+ console.log(`dream --auto: ${scopes.length} 个达标 scope → ${scopes.join(", ")}`);
1599
+ let anyFailed = false;
1600
+ for (const scope of scopes) {
1601
+ try {
1602
+ const result = await runDream({ store, llm, embedder, scope, force: Boolean(options.force), activityStatsPath, kgStore });
1603
+ console.log(`[scope=${scope}] ${formatDreamResult(result)}`);
1604
+ } catch (err) {
1605
+ anyFailed = true;
1606
+ // 单 scope 失败不阻断其他 scope
1607
+ console.error(`[scope=${scope}] dream failed: ${err instanceof Error ? err.message : String(err)}`);
1608
+ }
1609
+ }
1610
+ console.log(anyFailed ? "[[DREAM_STATUS]] blocked" : "[[DREAM_STATUS]] ok");
1611
+ process.exit(anyFailed ? 1 : 0);
1612
+ }
1613
+
1614
+ try {
1615
+ const result = await runDream({
1616
+ store,
1617
+ llm,
1618
+ embedder,
1619
+ scope: options.scope,
1620
+ force: Boolean(options.force),
1621
+ activityStatsPath,
1622
+ kgStore,
1623
+ });
1624
+
1625
+ console.log(formatDreamResult(result));
1626
+ console.log(result.ran ? "[[DREAM_STATUS]] ok" : "[[DREAM_STATUS]] skip");
1627
+ process.exit(0);
1628
+ } catch (err) {
1629
+ console.error(`dream failed: ${err instanceof Error ? err.message : String(err)}`);
1630
+ console.log("[[DREAM_STATUS]] blocked");
1631
+ process.exit(1);
1632
+ }
1633
+ });
1634
+
1475
1635
  // ─── stats ───────────────────────────────────────────────────────────────────
1476
1636
 
1477
1637
  program
@@ -1693,6 +1853,7 @@ program
1693
1853
  .option("--task <task>", "相关任务")
1694
1854
  .option("--tags <tags>", "逗号分隔 tags")
1695
1855
  .option("--tools <tools>", "逗号分隔 tools")
1856
+ .option("--idempotency-key <key>", "同一请求重试时用于替换旧 observation 的稳定键")
1696
1857
  .option("--json", "输出 JSON")
1697
1858
  .action(async (workflowId, summary, options) => {
1698
1859
  const store = new WorkflowObservationStore();
@@ -1706,6 +1867,7 @@ program
1706
1867
  task: options.task,
1707
1868
  tags: splitCsvOption(options.tags),
1708
1869
  tools: splitCsvOption(options.tools),
1870
+ idempotencyKey: options.idempotencyKey,
1709
1871
  });
1710
1872
  const stored = await store.save(record);
1711
1873
  if (options.json) {
@@ -261,50 +261,68 @@ export function buildContradictPatch(
261
261
  // Convenience wrappers (for callers who DO have a store)
262
262
  // ---------------------------------------------------------------------------
263
263
 
264
+ /** 单写通道端口:读改写经 store.patchMetadata per-id 队列,防并发覆盖。 */
265
+ type ConfidenceStorePort = {
266
+ patchMetadata(
267
+ id: string,
268
+ patchFn: (meta: Record<string, unknown>, entry: MemoryEntry) => Record<string, unknown>,
269
+ scopeFilter?: string[],
270
+ ): Promise<MemoryEntry | null>;
271
+ };
272
+
273
+ function applyConfidencePatch(
274
+ store: ConfidenceStorePort,
275
+ entryId: string,
276
+ scope: string,
277
+ build: (entry: MemoryEntry) => ConfidencePatch | null,
278
+ ): Promise<ConfidenceUpdate | null> {
279
+ let update: ConfidenceUpdate | null = null;
280
+ return store
281
+ .patchMetadata(
282
+ entryId,
283
+ (meta, entry) => {
284
+ const patch = build(entry);
285
+ if (!patch) return meta;
286
+ update = patch.update;
287
+ return patch.metadata;
288
+ },
289
+ [scope],
290
+ )
291
+ .then(result => (result === null ? null : update));
292
+ }
293
+
264
294
  /**
265
295
  * Confirm a memory and write back to store. Convenience wrapper.
266
296
  */
267
297
  export async function confirmMemory(
268
- store: { getById(id: string): Promise<MemoryEntry | null>; update(id: string, data: { metadata: string }, scopes: string[]): Promise<void> },
298
+ store: ConfidenceStorePort,
269
299
  entryId: string,
270
300
  scope: string,
271
301
  ): Promise<ConfidenceUpdate | null> {
272
- const entry = await store.getById(entryId);
273
- const patch = buildConfirmPatch(entry);
274
- if (!patch) return null;
275
- await store.update(entryId, { metadata: JSON.stringify(patch.metadata) }, [scope]);
276
- return patch.update;
302
+ return applyConfidencePatch(store, entryId, scope, entry => buildConfirmPatch(entry));
277
303
  }
278
304
 
279
305
  /**
280
306
  * Correct a memory and write back to store. Convenience wrapper.
281
307
  */
282
308
  export async function correctMemory(
283
- store: { getById(id: string): Promise<MemoryEntry | null>; update(id: string, data: { metadata: string }, scopes: string[]): Promise<void> },
309
+ store: ConfidenceStorePort,
284
310
  entryId: string,
285
311
  scope: string,
286
312
  correctedById?: string,
287
313
  ): Promise<ConfidenceUpdate | null> {
288
- const entry = await store.getById(entryId);
289
- const patch = buildCorrectPatch(entry, correctedById);
290
- if (!patch) return null;
291
- await store.update(entryId, { metadata: JSON.stringify(patch.metadata) }, [scope]);
292
- return patch.update;
314
+ return applyConfidencePatch(store, entryId, scope, entry => buildCorrectPatch(entry, correctedById));
293
315
  }
294
316
 
295
317
  /**
296
318
  * Contradict a memory and write back to store. Convenience wrapper.
297
319
  */
298
320
  export async function contradictMemory(
299
- store: { getById(id: string): Promise<MemoryEntry | null>; update(id: string, data: { metadata: string }, scopes: string[]): Promise<void> },
321
+ store: ConfidenceStorePort,
300
322
  entryId: string,
301
323
  scope: string,
302
324
  ): Promise<ConfidenceUpdate | null> {
303
- const entry = await store.getById(entryId);
304
- const patch = buildContradictPatch(entry);
305
- if (!patch) return null;
306
- await store.update(entryId, { metadata: JSON.stringify(patch.metadata) }, [scope]);
307
- return patch.update;
325
+ return applyConfidencePatch(store, entryId, scope, entry => buildContradictPatch(entry));
308
326
  }
309
327
 
310
328
  // ---------------------------------------------------------------------------