recallnest 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env +4 -0
- package/CLAUDE.md +1 -1
- package/README.md +30 -5
- package/README_CN.md +30 -5
- package/config.json +55 -0
- package/connectors/examples/email-adapter.ts +73 -0
- package/connectors/examples/logs-adapter.ts +103 -0
- package/connectors/examples/rss-adapter.ts +95 -0
- package/logs/daily-recall.error.log +0 -0
- package/logs/daily-recall.log +60 -0
- package/logs/ingest-2026-04-11.log +110 -0
- package/logs/ingest-2026-04-12.log +90 -0
- package/logs/ingest-2026-04-13.log +87 -0
- package/logs/launchagent-stderr.log +0 -0
- package/logs/launchagent-stdout.log +0 -0
- package/package.json +1 -1
- package/src/cli.ts +30 -0
- package/src/connector-types.ts +57 -0
- package/src/conversation-importer.ts +25 -1
- package/src/data-checkup.ts +31 -1
- package/src/doctor.ts +29 -0
- package/src/ingest.ts +203 -0
- package/src/mcp-server.ts +2 -2
- package/src/obsidian-connector.ts +282 -0
- package/src/session-schema.ts +2 -2
- package/src/source-heartbeat.ts +108 -0
- package/.claude-plugin/CLAUDE.md +0 -29
- package/.claude-plugin/marketplace.json +0 -18
- package/.dockerignore +0 -11
- package/.github/workflows/ci.yml +0 -28
- package/.mcp.json +0 -9
- package/CHANGELOG.md +0 -51
- package/DEMO.md +0 -94
- package/Dockerfile +0 -20
- package/EVAL.md +0 -89
- package/FAILURES.md +0 -75
- package/OPERATIONS.md +0 -59
- package/ROADMAP.md +0 -245
- package/SKILL.md +0 -250
- package/assets/dashboard.png +0 -0
- package/assets/knowledge-graph.png +0 -0
- package/assets/recallnest-workbench.png +0 -0
- package/assets/screenshots/ui-full.png +0 -0
- package/assets/screenshots/ui-overview.png +0 -0
- package/assets/screenshots/ui-search.png +0 -0
- package/assets/ui/app.js +0 -1031
- package/assets/ui/index.html +0 -224
- package/assets/ui/styles.css +0 -781
- package/assets/wechat-qr.jpg +0 -0
- package/bun.lock +0 -337
- package/daily-reflections/2026-03-05.md +0 -15
- package/daily-reflections/2026-03-06.md +0 -15
- package/daily-reflections/2026-03-07.md +0 -7
- package/daily-reflections/2026-03-08.md +0 -34
- package/daily-reflections/2026-03-09.md +0 -35
- package/daily-reflections/2026-03-10.md +0 -36
- package/daily-reflections/2026-03-11.md +0 -35
- package/daily-reflections/2026-03-12.md +0 -36
- package/daily-reflections/2026-03-13.md +0 -35
- package/daily-reflections/2026-03-14.md +0 -35
- package/daily-reflections/2026-03-15.md +0 -35
- package/daily-reflections/2026-03-16.md +0 -41
- package/daily-reflections/2026-03-17.md +0 -10
- package/daily-reflections/2026-03-18.md +0 -10
- package/daily-reflections/2026-03-19.md +0 -10
- package/daily-reflections/2026-03-20.md +0 -10
- package/daily-reflections/2026-03-21.md +0 -10
- package/daily-reflections/2026-03-22.md +0 -10
- package/daily-reflections/2026-03-23.md +0 -10
- package/daily-reflections/2026-03-24.md +0 -10
- package/daily-reflections/2026-03-25.md +0 -32
- package/daily-reflections/2026-03-26.md +0 -37
- package/daily-reflections/2026-03-27.md +0 -35
- package/daily-reflections/2026-03-28.md +0 -35
- package/daily-reflections/2026-03-29.md +0 -35
- package/daily-reflections/2026-03-30.md +0 -36
- package/daily-reflections/2026-03-31.md +0 -35
- package/daily-reflections/2026-04-01.md +0 -35
- package/daily-reflections/2026-04-02.md +0 -35
- package/daily-reflections/2026-04-03.md +0 -37
- package/daily-reflections/2026-04-04.md +0 -37
- package/daily-reflections/2026-04-05.md +0 -35
- package/daily-reflections/2026-04-06.md +0 -36
- package/daily-reflections/2026-04-07.md +0 -35
- package/daily-reflections/2026-04-08.md +0 -35
- package/daily-reflections/2026-04-12.md +0 -35
- package/data/alias-map.example.json +0 -22
- package/docs/api-reference.md +0 -1291
- package/docs/architecture.md +0 -168
- package/docs/memory-boundary-contract.md +0 -226
- package/docs/memory-categories.md +0 -119
- package/docs/pipeline-audit-ultramemory.md +0 -177
- package/docs/plans/2026-03-08-query-expansion.md +0 -270
- package/docs/plans/2026-03-16-continuity-layer.md +0 -196
- package/docs/plans/2026-04-01-recall-boost.md +0 -100
- package/docs/plans/mcp-tiering-plan.md +0 -126
- package/docs/research/recall-quality-audit-2026-03-22.md +0 -153
- package/docs/self-evolution.md +0 -198
- package/docs/superpowers/plans/2026-03-19-ingest-pipeline-upgrade.md +0 -782
- package/docs/superpowers/plans/2026-04-11-constructive-retrieval-emotion.md +0 -1195
- package/docs/superpowers/specs/2026-03-19-ingest-pipeline-upgrade-design.md +0 -77
- package/docs/superpowers/specs/2026-03-23-pull-claude-desktop-design.md +0 -95
- package/docs/superpowers/specs/2026-04-11-constructive-retrieval-emotion-design.md +0 -398
- package/eval/continuity/results-2026-03-16.md +0 -117
- package/eval/continuity/results-2026-03-19.md +0 -453
- package/eval/continuity/runbook.md +0 -110
- package/eval/continuity/seed-cases.md +0 -54
- package/eval/reports/2026-03-06-abstract-query-baseline.md +0 -96
- package/eval/reports/2026-03-06-aws-ssh-baseline.md +0 -82
- package/eval/reports/2026-03-06-baseline.md +0 -82
- package/eval/reports/query-expansion.md +0 -138
- package/glama.json +0 -6
- package/lm +0 -19
- package/plan-philosophy-upgrades.md +0 -363
- package/plan-product-upgrades.md +0 -155
- package/plan-v1.2-distributable.md +0 -105
- package/plan.md +0 -140
- package/research-memory-evolution.md +0 -440
- package/research-rl-agent-memory.md +0 -330
- package/scripts/archive-distilled-junk.ts +0 -116
- package/scripts/backfill-anchors.ts +0 -130
- package/scripts/backfill-l0l1l2.ts +0 -98
- package/scripts/blind-scan.ts +0 -588
- package/scripts/cleanup-step1-exact-dups.ts +0 -117
- package/scripts/cleanup-step2-template-dups.ts +0 -113
- package/scripts/cleanup-step3-stale-content.ts +0 -149
- package/scripts/cleanup-step4-orphan-scopes.ts +0 -131
- package/scripts/delete-distilled-junk.ts +0 -88
- package/scripts/diagnose-vector-search.ts +0 -132
- package/scripts/distill-facts.ts +0 -305
- package/scripts/health-check.ts +0 -343
- package/scripts/incremental-ingest.sh +0 -71
- package/scripts/migrate-language.ts +0 -75
- package/scripts/mlp-health-check.ts +0 -382
- package/scripts/pull-claude-desktop.ts +0 -515
- package/scripts/re-ingest-cc.sh +0 -66
- package/scripts/rebalance-memory-health.ts +0 -181
- package/scripts/run-desktop-ingest.sh +0 -11
- package/scripts/start-server.sh +0 -12
- package/scripts/verify-emotion-decay.ts +0 -148
- package/scripts/viz-dashboard.html +0 -423
- package/scripts/viz-export.ts +0 -703
- package/src/__tests__/access-tracker-novelty.test.ts +0 -228
- package/src/__tests__/activity-counter.test.ts +0 -96
- package/src/__tests__/admission-control.test.ts +0 -206
- package/src/__tests__/anchor-generator.test.ts +0 -67
- package/src/__tests__/auto-consolidation.test.ts +0 -168
- package/src/__tests__/auto-recall.test.ts +0 -131
- package/src/__tests__/babel-integration.test.ts +0 -139
- package/src/__tests__/batch-dedup.test.ts +0 -65
- package/src/__tests__/capture-engine.test.ts +0 -1144
- package/src/__tests__/capture-heuristic.test.ts +0 -171
- package/src/__tests__/cascade-forget.test.ts +0 -210
- package/src/__tests__/category-min-scores.test.ts +0 -33
- package/src/__tests__/cc8-cc1-essential-context.test.ts +0 -298
- package/src/__tests__/circuit-breaker.test.ts +0 -99
- package/src/__tests__/cluster-consolidation.test.ts +0 -712
- package/src/__tests__/confidence-metadata.test.ts +0 -125
- package/src/__tests__/confidence-tracker.test.ts +0 -249
- package/src/__tests__/conflict-advisor.test.ts +0 -206
- package/src/__tests__/conflict-escalation.test.ts +0 -149
- package/src/__tests__/conflict-lifecycle.test.ts +0 -83
- package/src/__tests__/conflict-output.test.ts +0 -85
- package/src/__tests__/conflict-store.test.ts +0 -374
- package/src/__tests__/consolidation-engine.test.ts +0 -181
- package/src/__tests__/constructive-retrieval.test.ts +0 -509
- package/src/__tests__/context-collapse-renderer.test.ts +0 -214
- package/src/__tests__/context-composer-pins.test.ts +0 -64
- package/src/__tests__/context-composer-task-selection.test.ts +0 -24
- package/src/__tests__/context-reconstructor.test.ts +0 -65
- package/src/__tests__/context-renderer.test.ts +0 -91
- package/src/__tests__/continuity-eval.test.ts +0 -275
- package/src/__tests__/continuity-schema.test.ts +0 -97
- package/src/__tests__/conversation-importer.test.ts +0 -692
- package/src/__tests__/core-recall-eval.test.ts +0 -235
- package/src/__tests__/core-summary.test.ts +0 -109
- package/src/__tests__/data-checkup.test.ts +0 -225
- package/src/__tests__/decay-exemption.test.ts +0 -84
- package/src/__tests__/dedup-actions.test.ts +0 -58
- package/src/__tests__/dedup-guard.test.ts +0 -39
- package/src/__tests__/deterministic-id.test.ts +0 -39
- package/src/__tests__/distill-lock.test.ts +0 -176
- package/src/__tests__/doctor.test.ts +0 -303
- package/src/__tests__/dream-pipeline.test.ts +0 -232
- package/src/__tests__/drill-down.test.ts +0 -108
- package/src/__tests__/embedder.test.ts +0 -122
- package/src/__tests__/emotion-decay.test.ts +0 -46
- package/src/__tests__/emotion-detector.test.ts +0 -56
- package/src/__tests__/emotion-phase1-integration.test.ts +0 -153
- package/src/__tests__/emotion-retrieval.test.ts +0 -56
- package/src/__tests__/entity-resolver.test.ts +0 -58
- package/src/__tests__/ethics-layer.test.ts +0 -476
- package/src/__tests__/event-segmenter.test.ts +0 -153
- package/src/__tests__/feature-flags-integration.test.ts +0 -54
- package/src/__tests__/frequency-tracker.test.ts +0 -113
- package/src/__tests__/graph-export.test.ts +0 -489
- package/src/__tests__/hotness-blend.test.ts +0 -76
- package/src/__tests__/ingest.test.ts +0 -360
- package/src/__tests__/interference-detector.test.ts +0 -203
- package/src/__tests__/kg-extractor.test.ts +0 -184
- package/src/__tests__/kg-store.test.ts +0 -198
- package/src/__tests__/llm-consolidation.test.ts +0 -226
- package/src/__tests__/mcp-efficiency.test.ts +0 -288
- package/src/__tests__/memory-boundaries.test.ts +0 -237
- package/src/__tests__/memory-evolution-b2-b3-e1.test.ts +0 -243
- package/src/__tests__/memory-evolution.test.ts +0 -271
- package/src/__tests__/memory-health-rebalance.test.ts +0 -144
- package/src/__tests__/memory-lint.test.ts +0 -271
- package/src/__tests__/memory-output.test.ts +0 -215
- package/src/__tests__/multi-vector.test.ts +0 -242
- package/src/__tests__/narrative-architecture.test.ts +0 -456
- package/src/__tests__/pending-review.test.ts +0 -77
- package/src/__tests__/post-retrieval-filter.test.ts +0 -178
- package/src/__tests__/ppr-traversal.test.ts +0 -256
- package/src/__tests__/prediction-engine.test.ts +0 -349
- package/src/__tests__/predictive-prospective-memory.test.ts +0 -416
- package/src/__tests__/preference-guard-smoke.test.ts +0 -211
- package/src/__tests__/preference-matcher.test.ts +0 -230
- package/src/__tests__/preference-slots.test.ts +0 -205
- package/src/__tests__/prospective-memory.test.ts +0 -283
- package/src/__tests__/query-entity-detector.test.ts +0 -171
- package/src/__tests__/query-expander-alias.test.ts +0 -89
- package/src/__tests__/query-expander.test.ts +0 -34
- package/src/__tests__/recall-governor.test.ts +0 -227
- package/src/__tests__/recall-mode.test.ts +0 -64
- package/src/__tests__/result-synthesizer.test.ts +0 -198
- package/src/__tests__/resume-context.test.ts +0 -4898
- package/src/__tests__/retrieval-trace.test.ts +0 -213
- package/src/__tests__/retriever-boundary-weight.test.ts +0 -136
- package/src/__tests__/retroactive-boost.test.ts +0 -223
- package/src/__tests__/salience-scoring.test.ts +0 -106
- package/src/__tests__/scope-inventory.test.ts +0 -200
- package/src/__tests__/scope-policy.test.ts +0 -69
- package/src/__tests__/session-checkpoint.test.ts +0 -259
- package/src/__tests__/session-distiller.test.ts +0 -611
- package/src/__tests__/session-gc.test.ts +0 -215
- package/src/__tests__/skill-engine.test.ts +0 -546
- package/src/__tests__/skill-promotion.test.ts +0 -349
- package/src/__tests__/store-scope-audit.test.ts +0 -81
- package/src/__tests__/supersede-chain.test.ts +0 -154
- package/src/__tests__/temporal-parser.test.ts +0 -241
- package/src/__tests__/temporal-validity.test.ts +0 -160
- package/src/__tests__/term-registry.test.ts +0 -121
- package/src/__tests__/tool-output-compressor.test.ts +0 -133
- package/src/__tests__/topic-tag.test.ts +0 -113
- package/src/__tests__/trusted-memory.test.ts +0 -828
- package/src/__tests__/version-manager.test.ts +0 -213
- package/src/__tests__/workflow-observation.test.ts +0 -400
- package/src/__tests__/write-verifier.test.ts +0 -122
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
=== 2026-04-12 03:00:01 增量更新开始 ===
|
|
2
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
3
|
+
|
|
4
|
+
🔑 验证 Embedding API...
|
|
5
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
6
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
7
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
8
|
+
|
|
9
|
+
🔄 开始导入记忆 (source: all)...
|
|
10
|
+
|
|
11
|
+
📝 导入 Claude Code 对话...
|
|
12
|
+
[INFO] noise-filter: denial pattern matched: "<task-notification>
|
|
13
|
+
<task-id>ac1e8c3a46f90bcc6</task-id>
|
|
14
|
+
<to..."
|
|
15
|
+
CC: 400/976 files, 269 chunks ingested
|
|
16
|
+
[INFO] noise-filter: denial pattern matched: "完全破案了。
|
|
17
|
+
|
|
18
|
+
**根因:4月10日 22:55(新加坡时间)发生了一次批量仓库可见性修改**
|
|
19
|
+
|
|
20
|
+
时间线还原:
|
|
21
|
+
|
|
22
|
+
| U..."
|
|
23
|
+
CC: 770/976 files, 541 chunks ingested
|
|
24
|
+
✅ CC: 19 files, 563 ingested, 505 deduped (hard:24, exact:228, llm-skip:114, llm-merge:139), 0 errors
|
|
25
|
+
🤖 导入 Codex 对话...
|
|
26
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
27
|
+
💎 导入 Gemini 对话...
|
|
28
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
29
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
30
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
31
|
+
📚 导入记忆文件...
|
|
32
|
+
✅ Memory: 5 files, 4 ingested, 188 deduped (hard:0, exact:172, llm-skip:13, llm-merge:3), 0 errors
|
|
33
|
+
|
|
34
|
+
📊 导入汇总:
|
|
35
|
+
总计: 567 chunks 已索引, 693 chunks 去重 (hard:24, exact:400, llm-skip:127, llm-merge:142), 0 errors
|
|
36
|
+
|
|
37
|
+
=== 2026-04-12 03:16:02 增量更新完成 ===
|
|
38
|
+
|
|
39
|
+
=== 2026-04-12 11:16:02 增量更新开始 ===
|
|
40
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
41
|
+
|
|
42
|
+
🔑 验证 Embedding API...
|
|
43
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
44
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
45
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
46
|
+
|
|
47
|
+
🔄 开始导入记忆 (source: all)...
|
|
48
|
+
|
|
49
|
+
📝 导入 Claude Code 对话...
|
|
50
|
+
✅ CC: 5 files, 141 ingested, 90 deduped (hard:4, exact:24, llm-skip:30, llm-merge:32), 0 errors
|
|
51
|
+
🤖 导入 Codex 对话...
|
|
52
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
53
|
+
💎 导入 Gemini 对话...
|
|
54
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
55
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
56
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
57
|
+
📚 导入记忆文件...
|
|
58
|
+
✅ Memory: 1 files, 0 ingested, 182 deduped (hard:0, exact:171, llm-skip:11, llm-merge:0), 0 errors
|
|
59
|
+
|
|
60
|
+
📊 导入汇总:
|
|
61
|
+
总计: 141 chunks 已索引, 272 chunks 去重 (hard:4, exact:195, llm-skip:41, llm-merge:32), 0 errors
|
|
62
|
+
|
|
63
|
+
=== 2026-04-12 11:21:02 增量更新完成 ===
|
|
64
|
+
|
|
65
|
+
=== 2026-04-12 19:21:02 增量更新开始 ===
|
|
66
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
67
|
+
|
|
68
|
+
🔑 验证 Embedding API...
|
|
69
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
70
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
71
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
72
|
+
|
|
73
|
+
🔄 开始导入记忆 (source: all)...
|
|
74
|
+
|
|
75
|
+
📝 导入 Claude Code 对话...
|
|
76
|
+
✅ CC: 3 files, 55 ingested, 188 deduped (hard:2, exact:134, llm-skip:33, llm-merge:19), 0 errors
|
|
77
|
+
🤖 导入 Codex 对话...
|
|
78
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
79
|
+
💎 导入 Gemini 对话...
|
|
80
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
81
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
82
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
83
|
+
📚 导入记忆文件...
|
|
84
|
+
✅ Memory: 1 files, 0 ingested, 182 deduped (hard:0, exact:171, llm-skip:11, llm-merge:0), 0 errors
|
|
85
|
+
|
|
86
|
+
📊 导入汇总:
|
|
87
|
+
总计: 55 chunks 已索引, 370 chunks 去重 (hard:2, exact:305, llm-skip:44, llm-merge:19), 0 errors
|
|
88
|
+
|
|
89
|
+
=== 2026-04-12 19:25:03 增量更新完成 ===
|
|
90
|
+
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
=== 2026-04-13 03:00:04 增量更新开始 ===
|
|
2
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
3
|
+
|
|
4
|
+
🔑 验证 Embedding API...
|
|
5
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
6
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
7
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
8
|
+
|
|
9
|
+
🔄 开始导入记忆 (source: all)...
|
|
10
|
+
|
|
11
|
+
📝 导入 Claude Code 对话...
|
|
12
|
+
[INFO] noise-filter: short boilerplate filtered: "收到!多路并行处理:"
|
|
13
|
+
CC: 430/1002 files, 87 chunks ingested
|
|
14
|
+
CC: 680/1002 files, 217 chunks ingested
|
|
15
|
+
✅ CC: 23 files, 378 ingested, 655 deduped (hard:67, exact:279, llm-skip:202, llm-merge:107), 0 errors
|
|
16
|
+
🤖 导入 Codex 对话...
|
|
17
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
18
|
+
💎 导入 Gemini 对话...
|
|
19
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
20
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
21
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
22
|
+
📚 导入记忆文件...
|
|
23
|
+
✅ Memory: 7 files, 26 ingested, 216 deduped (hard:1, exact:192, llm-skip:16, llm-merge:7), 0 errors
|
|
24
|
+
|
|
25
|
+
📊 导入汇总:
|
|
26
|
+
总计: 404 chunks 已索引, 871 chunks 去重 (hard:68, exact:471, llm-skip:218, llm-merge:114), 0 errors
|
|
27
|
+
|
|
28
|
+
=== 2026-04-13 03:13:05 增量更新完成 ===
|
|
29
|
+
|
|
30
|
+
=== 2026-04-13 11:13:05 增量更新开始 ===
|
|
31
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
32
|
+
|
|
33
|
+
🔑 验证 Embedding API...
|
|
34
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
35
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
36
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
37
|
+
|
|
38
|
+
🔄 开始导入记忆 (source: all)...
|
|
39
|
+
|
|
40
|
+
📝 导入 Claude Code 对话...
|
|
41
|
+
CC: 480/1006 files, 20 chunks ingested
|
|
42
|
+
[INFO] noise-filter: denial pattern matched: "好,全文读完了。这篇文章标题是「Claude Code 省 Token 指南:慎用 1M 上下文,不开新会话或者总是开新..."
|
|
43
|
+
✅ CC: 5 files, 130 ingested, 153 deduped (hard:14, exact:44, llm-skip:46, llm-merge:49), 0 errors
|
|
44
|
+
🤖 导入 Codex 对话...
|
|
45
|
+
[INFO] noise-filter: meta-question filtered: "# AGENTS.md instructions for /Users/anxianjingya
|
|
46
|
+
|
|
47
|
+
<INSTRUCTI..."
|
|
48
|
+
Codex: 295/295 files, 2 chunks ingested
|
|
49
|
+
✅ Codex: 1 files, 2 ingested, 1 deduped (hard:0, exact:0, llm-skip:0, llm-merge:1), 0 errors
|
|
50
|
+
💎 导入 Gemini 对话...
|
|
51
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
52
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
53
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
54
|
+
📚 导入记忆文件...
|
|
55
|
+
✅ Memory: 1 files, 1 ingested, 185 deduped (hard:0, exact:172, llm-skip:13, llm-merge:0), 0 errors
|
|
56
|
+
|
|
57
|
+
📊 导入汇总:
|
|
58
|
+
总计: 133 chunks 已索引, 339 chunks 去重 (hard:14, exact:216, llm-skip:59, llm-merge:50), 0 errors
|
|
59
|
+
|
|
60
|
+
=== 2026-04-13 11:19:05 增量更新完成 ===
|
|
61
|
+
|
|
62
|
+
=== 2026-04-13 19:19:05 增量更新开始 ===
|
|
63
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
64
|
+
|
|
65
|
+
🔑 验证 Embedding API...
|
|
66
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
67
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
68
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
69
|
+
|
|
70
|
+
🔄 开始导入记忆 (source: all)...
|
|
71
|
+
|
|
72
|
+
📝 导入 Claude Code 对话...
|
|
73
|
+
✅ CC: 12 files, 193 ingested, 225 deduped (hard:19, exact:71, llm-skip:74, llm-merge:61), 0 errors
|
|
74
|
+
🤖 导入 Codex 对话...
|
|
75
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
76
|
+
💎 导入 Gemini 对话...
|
|
77
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
78
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
79
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
80
|
+
📚 导入记忆文件...
|
|
81
|
+
✅ Memory: 6 files, 16 ingested, 229 deduped (hard:0, exact:202, llm-skip:16, llm-merge:11), 0 errors
|
|
82
|
+
|
|
83
|
+
📊 导入汇总:
|
|
84
|
+
总计: 209 chunks 已索引, 454 chunks 去重 (hard:19, exact:273, llm-skip:90, llm-merge:72), 0 errors
|
|
85
|
+
|
|
86
|
+
=== 2026-04-13 19:27:06 增量更新完成 ===
|
|
87
|
+
|
|
File without changes
|
|
File without changes
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -29,6 +29,8 @@ import {
|
|
|
29
29
|
ingestCodexSessions,
|
|
30
30
|
ingestGeminiSessions,
|
|
31
31
|
ingestMarkdownFiles,
|
|
32
|
+
ingestObsidianVault,
|
|
33
|
+
ingestConnectorFile,
|
|
32
34
|
} from "./ingest.js";
|
|
33
35
|
import { runDoctor, formatDoctorResults } from "./doctor.js";
|
|
34
36
|
import { persistCaseMemory, persistMemory, persistWorkflowPattern } from "./capture-engine.js";
|
|
@@ -1306,6 +1308,8 @@ program
|
|
|
1306
1308
|
.option("--no-dedup", "跳过向量去重")
|
|
1307
1309
|
.option("--no-llm", "禁用 LLM 提取(原始对话将跳过不存入,排入待处理队列)")
|
|
1308
1310
|
.option("--recent [hours]", "快速模式:只处理最近 N 小时修改的文件(默认 2 小时),跳过 ingested-files 检查")
|
|
1311
|
+
.option("--obsidian <vault-path>", "导入 Obsidian vault(自动检测 .obsidian/ 目录)")
|
|
1312
|
+
.option("--connector <file-path>", "导入 connector-v1 格式 JSON 文件")
|
|
1309
1313
|
.action(async (options) => {
|
|
1310
1314
|
const config = loadConfig();
|
|
1311
1315
|
const { store, embedder, llm } = createComponents(config);
|
|
@@ -1440,6 +1444,32 @@ program
|
|
|
1440
1444
|
}
|
|
1441
1445
|
}
|
|
1442
1446
|
|
|
1447
|
+
// Obsidian vault (--obsidian <path>)
|
|
1448
|
+
if (options.obsidian) {
|
|
1449
|
+
console.log("🗃️ 导入 Obsidian vault...");
|
|
1450
|
+
const r = await ingestObsidianVault(store, embedder, options.obsidian, {
|
|
1451
|
+
verbose,
|
|
1452
|
+
noDedup,
|
|
1453
|
+
llm: effectiveLlm,
|
|
1454
|
+
});
|
|
1455
|
+
results.push(r);
|
|
1456
|
+
console.log(` ✅ Obsidian: ${formatIngestSummary(r)}`);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
// Connector-v1 file (--connector <path>)
|
|
1460
|
+
if (options.connector) {
|
|
1461
|
+
console.log("🔌 导入 connector-v1 文件...");
|
|
1462
|
+
const { readFileSync: readFs } = await import("node:fs");
|
|
1463
|
+
const connectorContent = readFs(options.connector, "utf-8");
|
|
1464
|
+
const r = await ingestConnectorFile(store, embedder, connectorContent, {
|
|
1465
|
+
verbose,
|
|
1466
|
+
noDedup,
|
|
1467
|
+
llm: effectiveLlm,
|
|
1468
|
+
});
|
|
1469
|
+
results.push(r);
|
|
1470
|
+
console.log(` ✅ Connector: ${formatIngestSummary(r)}`);
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1443
1473
|
// Summary
|
|
1444
1474
|
console.log("\n📊 导入汇总:");
|
|
1445
1475
|
let totalChunks = 0;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Connector Standard Types (connector-v1)
|
|
3
|
+
*
|
|
4
|
+
* 定义外部数据源(Obsidian、Email、RSS、自定义脚本)统一产出的 JSON 格式。
|
|
5
|
+
* ingest.ts 的 ingestConnectorFile 消费这个格式,走已有的 dedup/extract/入库管线。
|
|
6
|
+
*
|
|
7
|
+
* 协议版本管理:version 字段用于未来兼容性。当前固定 "connector-v1"。
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export interface ConnectorOutputV1 {
|
|
11
|
+
/** 格式版本,固定 "connector-v1" */
|
|
12
|
+
version: "connector-v1";
|
|
13
|
+
/** 数据源标识,如 "obsidian", "email", "rss", "x-bookmarks" */
|
|
14
|
+
source: string;
|
|
15
|
+
/** 目标 scope,如 "vault:my-notes", "project:research" */
|
|
16
|
+
scope: string;
|
|
17
|
+
/** 批次 ID(可选,用于幂等重试) */
|
|
18
|
+
batchId?: string;
|
|
19
|
+
/** 产出时间 ISO 8601 */
|
|
20
|
+
producedAt: string;
|
|
21
|
+
/** 记录列表 */
|
|
22
|
+
records: ConnectorRecord[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface ConnectorRecord {
|
|
26
|
+
/** 记录唯一 ID(source 内唯一,用于增量去重) */
|
|
27
|
+
id: string;
|
|
28
|
+
/** 正文 */
|
|
29
|
+
text: string;
|
|
30
|
+
/** 标题/heading(可选) */
|
|
31
|
+
title?: string;
|
|
32
|
+
/** 分类提示:preferences / events / knowledge / facts / decisions */
|
|
33
|
+
categoryHint?: string;
|
|
34
|
+
/** 重要性提示 0-1(可选,默认 0.7) */
|
|
35
|
+
importanceHint?: number;
|
|
36
|
+
/** 标签 */
|
|
37
|
+
tags?: string[];
|
|
38
|
+
/** 原始时间戳 ISO 8601 */
|
|
39
|
+
timestamp?: string;
|
|
40
|
+
/** 内容哈希(用于增量同步,调用方计算) */
|
|
41
|
+
contentHash?: string;
|
|
42
|
+
/** 来源元数据(自由字段) */
|
|
43
|
+
sourceMetadata?: Record<string, unknown>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** 运行时格式验证:判定一个 parsed JSON 是否是合法 connector-v1 */
|
|
47
|
+
export function isConnectorOutputV1(value: unknown): value is ConnectorOutputV1 {
|
|
48
|
+
if (!value || typeof value !== "object") return false;
|
|
49
|
+
const obj = value as Record<string, unknown>;
|
|
50
|
+
return (
|
|
51
|
+
obj.version === "connector-v1"
|
|
52
|
+
&& typeof obj.source === "string"
|
|
53
|
+
&& typeof obj.scope === "string"
|
|
54
|
+
&& typeof obj.producedAt === "string"
|
|
55
|
+
&& Array.isArray(obj.records)
|
|
56
|
+
);
|
|
57
|
+
}
|
|
@@ -2,6 +2,7 @@ import { readFileSync } from "node:fs";
|
|
|
2
2
|
import type { PersistMemoryDeps } from "./capture-engine.js";
|
|
3
3
|
import { persistMemory } from "./capture-engine.js";
|
|
4
4
|
import type { StoredMemoryRecord } from "./memory-schema.js";
|
|
5
|
+
import { isConnectorOutputV1, type ConnectorOutputV1 } from "./connector-types.js";
|
|
5
6
|
|
|
6
7
|
export interface NormalizedMessage {
|
|
7
8
|
role: "user" | "assistant" | "system";
|
|
@@ -14,7 +15,8 @@ export type ConversationFormat =
|
|
|
14
15
|
| "claude-ai"
|
|
15
16
|
| "chatgpt"
|
|
16
17
|
| "slack"
|
|
17
|
-
| "plaintext"
|
|
18
|
+
| "plaintext"
|
|
19
|
+
| "connector-v1";
|
|
18
20
|
|
|
19
21
|
// ---------------------------------------------------------------------------
|
|
20
22
|
// Format detection
|
|
@@ -40,6 +42,10 @@ export function detectFormat(content: string): ConversationFormat {
|
|
|
40
42
|
}
|
|
41
43
|
|
|
42
44
|
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
45
|
+
// connector-v1: highest priority — explicit version tag
|
|
46
|
+
if (isConnectorOutputV1(parsed)) {
|
|
47
|
+
return "connector-v1";
|
|
48
|
+
}
|
|
43
49
|
if (parsed.chat_conversations !== undefined) {
|
|
44
50
|
return "claude-ai";
|
|
45
51
|
}
|
|
@@ -366,6 +372,22 @@ export function normalizePlaintext(content: string): NormalizedMessage[] {
|
|
|
366
372
|
return messages;
|
|
367
373
|
}
|
|
368
374
|
|
|
375
|
+
// ---------------------------------------------------------------------------
|
|
376
|
+
// Connector-v1 normalizer
|
|
377
|
+
// ---------------------------------------------------------------------------
|
|
378
|
+
|
|
379
|
+
export function normalizeConnectorV1(content: string): NormalizedMessage[] {
|
|
380
|
+
const parsed: ConnectorOutputV1 = JSON.parse(content);
|
|
381
|
+
if (!isConnectorOutputV1(parsed)) {
|
|
382
|
+
throw new Error("Invalid connector-v1 format: missing version/source/scope/records");
|
|
383
|
+
}
|
|
384
|
+
return parsed.records.map((r) => ({
|
|
385
|
+
role: "user" as const,
|
|
386
|
+
content: r.title ? `[${r.title}] ${r.text}` : r.text,
|
|
387
|
+
...(r.timestamp ? { timestamp: r.timestamp } : {}),
|
|
388
|
+
}));
|
|
389
|
+
}
|
|
390
|
+
|
|
369
391
|
// ---------------------------------------------------------------------------
|
|
370
392
|
// Unified normalize dispatcher
|
|
371
393
|
// ---------------------------------------------------------------------------
|
|
@@ -382,6 +404,8 @@ export function normalizeConversation(content: string, format: ConversationForma
|
|
|
382
404
|
return normalizeSlack(content);
|
|
383
405
|
case "plaintext":
|
|
384
406
|
return normalizePlaintext(content);
|
|
407
|
+
case "connector-v1":
|
|
408
|
+
return normalizeConnectorV1(content);
|
|
385
409
|
}
|
|
386
410
|
}
|
|
387
411
|
|
package/src/data-checkup.ts
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
import type { MemoryEntry, MemoryStore } from "./store.js";
|
|
16
16
|
import { resolveTier, type MemoryTier } from "./decay-engine.js";
|
|
17
17
|
import { measureInterferenceDensity } from "./interference-detector.js";
|
|
18
|
+
import { readHeartbeats, checkSourceStaleness, formatAge } from "./source-heartbeat.js";
|
|
18
19
|
|
|
19
20
|
// ---------------------------------------------------------------------------
|
|
20
21
|
// Types
|
|
@@ -151,7 +152,33 @@ function checkVersionGroups(entries: MemoryEntry[]): CheckResult {
|
|
|
151
152
|
};
|
|
152
153
|
}
|
|
153
154
|
|
|
154
|
-
/** Check 6 (
|
|
155
|
+
/** Check 6 (GB-3): Source health — detect stale connector data sources. */
|
|
156
|
+
function checkSourceHealth(heartbeatPath?: string): CheckResult {
|
|
157
|
+
const heartbeats = readHeartbeats(heartbeatPath);
|
|
158
|
+
const sources = Object.values(heartbeats);
|
|
159
|
+
if (sources.length === 0) {
|
|
160
|
+
return { name: "source_health", status: "ok", detail: "No connector sources tracked yet" };
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const stale30 = checkSourceStaleness(30, heartbeatPath);
|
|
164
|
+
const stale7 = checkSourceStaleness(7, heartbeatPath);
|
|
165
|
+
|
|
166
|
+
const summary = sources
|
|
167
|
+
.map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
|
|
168
|
+
.join(", ");
|
|
169
|
+
|
|
170
|
+
if (stale30.length > 0) {
|
|
171
|
+
const names = stale30.map((s) => `${s.source} (${s.daysSince}d)`).join(", ");
|
|
172
|
+
return { name: "source_health", status: "error", detail: `${summary} — critically stale: ${names}` };
|
|
173
|
+
}
|
|
174
|
+
if (stale7.length > 0) {
|
|
175
|
+
const names = stale7.map((s) => `${s.source} (${s.daysSince}d)`).join(", ");
|
|
176
|
+
return { name: "source_health", status: "warning", detail: `${summary} — stale: ${names}` };
|
|
177
|
+
}
|
|
178
|
+
return { name: "source_health", status: "ok", detail: `All sources fresh: ${summary}` };
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Check 7 (F2): Interference density — detect semantic clustering pressure. */
|
|
155
182
|
function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
|
|
156
183
|
// Only check entries with vectors (skip schema/empty entries)
|
|
157
184
|
const withVectors = entries.filter(e => e.vector && e.vector.length > 0);
|
|
@@ -192,6 +219,8 @@ function checkInterferenceDensity(entries: MemoryEntry[]): CheckResult {
|
|
|
192
219
|
export interface CheckupDeps {
|
|
193
220
|
store: Pick<MemoryStore, "list" | "stats">;
|
|
194
221
|
openConflictCount: number;
|
|
222
|
+
/** Optional override path for source-heartbeat.json (used in tests). */
|
|
223
|
+
heartbeatPath?: string;
|
|
195
224
|
}
|
|
196
225
|
|
|
197
226
|
export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport> {
|
|
@@ -204,6 +233,7 @@ export async function runDataCheckup(deps: CheckupDeps): Promise<CheckupReport>
|
|
|
204
233
|
checkTierDistribution(entries),
|
|
205
234
|
checkConflictBacklog(deps.openConflictCount),
|
|
206
235
|
checkVersionGroups(entries),
|
|
236
|
+
checkSourceHealth(deps.heartbeatPath),
|
|
207
237
|
checkInterferenceDensity(entries),
|
|
208
238
|
],
|
|
209
239
|
totalEntries: entries.length,
|
package/src/doctor.ts
CHANGED
|
@@ -27,6 +27,7 @@ import {
|
|
|
27
27
|
} from "./memory-schema.js";
|
|
28
28
|
import { collectScopeInventory, type ScopeInventoryReport } from "./scope-inventory.js";
|
|
29
29
|
import { MemoryStore, type MemoryEntry, validateStoragePath } from "./store.js";
|
|
30
|
+
import { readHeartbeats, formatAge } from "./source-heartbeat.js";
|
|
30
31
|
|
|
31
32
|
interface CheckResult {
|
|
32
33
|
name: string;
|
|
@@ -508,6 +509,34 @@ export async function runDoctor(options: { ci?: boolean } = {}): Promise<CheckRe
|
|
|
508
509
|
));
|
|
509
510
|
}
|
|
510
511
|
|
|
512
|
+
// GB-3: Source heartbeat summary
|
|
513
|
+
try {
|
|
514
|
+
const heartbeats = readHeartbeats();
|
|
515
|
+
const sources = Object.values(heartbeats);
|
|
516
|
+
if (sources.length === 0) {
|
|
517
|
+
results.push(pass("Source heartbeats", "no connector sources tracked yet"));
|
|
518
|
+
} else {
|
|
519
|
+
const summary = sources
|
|
520
|
+
.map((s) => `${s.source}: ${formatAge(s.lastIngest)}`)
|
|
521
|
+
.join(", ");
|
|
522
|
+
const staleCount = sources.filter((s) => {
|
|
523
|
+
const daysSince = (Date.now() - Date.parse(s.lastIngest)) / 86_400_000;
|
|
524
|
+
return daysSince > 7;
|
|
525
|
+
}).length;
|
|
526
|
+
if (staleCount > 0) {
|
|
527
|
+
results.push(warn(
|
|
528
|
+
"Source heartbeats",
|
|
529
|
+
`${summary} (${staleCount} stale >7d)`,
|
|
530
|
+
"lm ingest --connector <file>",
|
|
531
|
+
));
|
|
532
|
+
} else {
|
|
533
|
+
results.push(pass("Source heartbeats", summary));
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
} catch {
|
|
537
|
+
results.push(warn("Source heartbeats", "unable to read heartbeat file"));
|
|
538
|
+
}
|
|
539
|
+
|
|
511
540
|
return results;
|
|
512
541
|
}
|
|
513
542
|
|
package/src/ingest.ts
CHANGED
|
@@ -15,6 +15,9 @@ import { detectLang, tokenizeFts } from "./language-hook.js";
|
|
|
15
15
|
import type { MemoryStore, MemoryEntry } from "./store.js";
|
|
16
16
|
import type { Embedder } from "./embedder.js";
|
|
17
17
|
import { chunkDocument, type ChunkerConfig } from "./chunker.js";
|
|
18
|
+
import type { ConnectorOutputV1, ConnectorRecord } from "./connector-types.js";
|
|
19
|
+
import { isConnectorOutputV1 } from "./connector-types.js";
|
|
20
|
+
import { isObsidianVault, scanVault } from "./obsidian-connector.js";
|
|
18
21
|
import { segmentEvents, type EventSegmenterConfig, DEFAULT_EVENT_SEGMENTER_CONFIG } from "./event-segmenter.js";
|
|
19
22
|
import { isProcessed, markProcessed } from "./tracker.js";
|
|
20
23
|
import type { LLMClient, SmartExtraction } from "./llm-client.js";
|
|
@@ -1850,3 +1853,203 @@ export async function ingestGenericText(
|
|
|
1850
1853
|
|
|
1851
1854
|
return result;
|
|
1852
1855
|
}
|
|
1856
|
+
|
|
1857
|
+
// ============================================================================
|
|
1858
|
+
// Connector-v1 Ingest (GB-2)
|
|
1859
|
+
// ============================================================================
|
|
1860
|
+
|
|
1861
|
+
/**
|
|
1862
|
+
* Ingest a connector-v1 formatted JSON payload.
|
|
1863
|
+
*
|
|
1864
|
+
* External connectors (Obsidian, Email, RSS, etc.) produce a standard
|
|
1865
|
+
* ConnectorOutputV1 JSON → this function consumes it through the existing
|
|
1866
|
+
* dedup / smartExtract / buildIngestedEntry pipeline.
|
|
1867
|
+
*
|
|
1868
|
+
* contentHash on each record enables incremental sync: if a record with the
|
|
1869
|
+
* same hash is already in the store, it is skipped without embedding.
|
|
1870
|
+
*/
|
|
1871
|
+
export async function ingestConnectorFile(
|
|
1872
|
+
store: MemoryStore,
|
|
1873
|
+
embedder: Embedder,
|
|
1874
|
+
content: string,
|
|
1875
|
+
options: { verbose?: boolean; noDedup?: boolean; llm?: LLMClient | null } = {},
|
|
1876
|
+
): Promise<IngestResult> {
|
|
1877
|
+
const parsed = JSON.parse(content);
|
|
1878
|
+
if (!isConnectorOutputV1(parsed)) {
|
|
1879
|
+
return {
|
|
1880
|
+
source: "connector",
|
|
1881
|
+
filesProcessed: 0,
|
|
1882
|
+
chunksIngested: 0,
|
|
1883
|
+
chunksSkipped: 0,
|
|
1884
|
+
chunksDeduped: 0,
|
|
1885
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
1886
|
+
errors: ["Invalid connector-v1 format: missing version/source/scope/records"],
|
|
1887
|
+
};
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
const connectorData = parsed as ConnectorOutputV1;
|
|
1891
|
+
const { source, scope, records } = connectorData;
|
|
1892
|
+
|
|
1893
|
+
const result: IngestResult = {
|
|
1894
|
+
source: `connector:${source}`,
|
|
1895
|
+
filesProcessed: 1,
|
|
1896
|
+
chunksIngested: 0,
|
|
1897
|
+
chunksSkipped: 0,
|
|
1898
|
+
chunksDeduped: 0,
|
|
1899
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
1900
|
+
errors: [],
|
|
1901
|
+
};
|
|
1902
|
+
|
|
1903
|
+
if (records.length === 0) return result;
|
|
1904
|
+
|
|
1905
|
+
const batchSize = 32;
|
|
1906
|
+
|
|
1907
|
+
for (let i = 0; i < records.length; i += batchSize) {
|
|
1908
|
+
const batch = records.slice(i, i + batchSize);
|
|
1909
|
+
const texts = batch.map((r: ConnectorRecord) =>
|
|
1910
|
+
r.title ? `[${r.title}] ${r.text}` : r.text,
|
|
1911
|
+
);
|
|
1912
|
+
|
|
1913
|
+
try {
|
|
1914
|
+
const vectors = await embedder.embedBatchPassage(texts);
|
|
1915
|
+
|
|
1916
|
+
// --- dedup pass ---
|
|
1917
|
+
const dedupedTexts: string[] = [];
|
|
1918
|
+
const dedupedVectors: number[][] = [];
|
|
1919
|
+
const dedupedRecords: ConnectorRecord[] = [];
|
|
1920
|
+
|
|
1921
|
+
for (let j = 0; j < batch.length; j++) {
|
|
1922
|
+
const vector = vectors[j];
|
|
1923
|
+
if (!vector || vector.length === 0) {
|
|
1924
|
+
result.chunksSkipped++;
|
|
1925
|
+
continue;
|
|
1926
|
+
}
|
|
1927
|
+
if (!options.noDedup) {
|
|
1928
|
+
const decision = await dedupCheck(store, vector, texts[j], options.llm);
|
|
1929
|
+
recordDedupDecision(result, decision);
|
|
1930
|
+
if (decision.secondaryDeletes?.length) {
|
|
1931
|
+
await executeSecondaryDeletes(store, decision.secondaryDeletes);
|
|
1932
|
+
}
|
|
1933
|
+
if (decision.action === "skip") continue;
|
|
1934
|
+
}
|
|
1935
|
+
dedupedTexts.push(texts[j]);
|
|
1936
|
+
dedupedVectors.push(vector);
|
|
1937
|
+
dedupedRecords.push(batch[j]);
|
|
1938
|
+
}
|
|
1939
|
+
|
|
1940
|
+
if (dedupedTexts.length === 0) continue;
|
|
1941
|
+
|
|
1942
|
+
// --- extract + build entries ---
|
|
1943
|
+
const extractions = await smartExtractBatch(dedupedTexts, options.llm);
|
|
1944
|
+
const coreSummaries = await generateCoreSummaries(dedupedTexts, options.llm);
|
|
1945
|
+
const toStore: Array<{
|
|
1946
|
+
text: string;
|
|
1947
|
+
vector: number[];
|
|
1948
|
+
category: string;
|
|
1949
|
+
scope: string;
|
|
1950
|
+
importance: number;
|
|
1951
|
+
metadata: string;
|
|
1952
|
+
}> = [];
|
|
1953
|
+
|
|
1954
|
+
for (let j = 0; j < dedupedTexts.length; j++) {
|
|
1955
|
+
const record = dedupedRecords[j];
|
|
1956
|
+
const entry = buildIngestedEntry({
|
|
1957
|
+
source: `connector:${source}`,
|
|
1958
|
+
scope,
|
|
1959
|
+
text: dedupedTexts[j],
|
|
1960
|
+
vector: dedupedVectors[j],
|
|
1961
|
+
extraction: extractions[j],
|
|
1962
|
+
file: record.id,
|
|
1963
|
+
heading: record.title,
|
|
1964
|
+
coreSummary: coreSummaries[j],
|
|
1965
|
+
});
|
|
1966
|
+
|
|
1967
|
+
// Merge connector-specific metadata
|
|
1968
|
+
const meta = JSON.parse(entry.metadata);
|
|
1969
|
+
if (record.tags?.length) meta.connectorTags = record.tags;
|
|
1970
|
+
if (record.contentHash) meta.contentHash = record.contentHash;
|
|
1971
|
+
if (record.sourceMetadata) meta.sourceMetadata = record.sourceMetadata;
|
|
1972
|
+
entry.metadata = JSON.stringify(meta);
|
|
1973
|
+
|
|
1974
|
+
// Respect categoryHint / importanceHint from connector
|
|
1975
|
+
if (record.categoryHint) {
|
|
1976
|
+
entry.category = record.categoryHint;
|
|
1977
|
+
}
|
|
1978
|
+
if (record.importanceHint !== undefined) {
|
|
1979
|
+
entry.importance = record.importanceHint;
|
|
1980
|
+
}
|
|
1981
|
+
|
|
1982
|
+
toStore.push(entry);
|
|
1983
|
+
}
|
|
1984
|
+
|
|
1985
|
+
if (toStore.length > 0) {
|
|
1986
|
+
await store.storeBatch(toStore);
|
|
1987
|
+
result.chunksIngested += toStore.length;
|
|
1988
|
+
}
|
|
1989
|
+
} catch (err: unknown) {
|
|
1990
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1991
|
+
result.errors.push(`Connector batch error: ${msg}`);
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
|
|
1995
|
+
if (options.verbose) {
|
|
1996
|
+
console.log(` connector:${source}: ${records.length} records → ${result.chunksIngested} stored`);
|
|
1997
|
+
}
|
|
1998
|
+
|
|
1999
|
+
// GB-3: Write source heartbeat after connector ingest
|
|
2000
|
+
try {
|
|
2001
|
+
const { writeHeartbeat } = await import("./source-heartbeat.js");
|
|
2002
|
+
writeHeartbeat(source, result.chunksIngested, result.errors);
|
|
2003
|
+
} catch {
|
|
2004
|
+
// Heartbeat write is best-effort — don't break ingest on failure
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
return result;
|
|
2008
|
+
}
|
|
2009
|
+
|
|
2010
|
+
// ============================================================================
|
|
2011
|
+
// Obsidian Vault Ingest (GB-1)
|
|
2012
|
+
// ============================================================================
|
|
2013
|
+
|
|
2014
|
+
/**
|
|
2015
|
+
* Scan an Obsidian vault and ingest its markdown files through the connector-v1 pipeline.
|
|
2016
|
+
*
|
|
2017
|
+
* Wrapper that:
|
|
2018
|
+
* 1. Validates the path is an Obsidian vault (.obsidian/ exists)
|
|
2019
|
+
* 2. Calls scanVault() to produce a ConnectorOutputV1 payload
|
|
2020
|
+
* 3. Feeds the payload to ingestConnectorFile()
|
|
2021
|
+
*/
|
|
2022
|
+
export async function ingestObsidianVault(
|
|
2023
|
+
store: MemoryStore,
|
|
2024
|
+
embedder: Embedder,
|
|
2025
|
+
vaultPath: string,
|
|
2026
|
+
options: {
|
|
2027
|
+
verbose?: boolean;
|
|
2028
|
+
noDedup?: boolean;
|
|
2029
|
+
llm?: LLMClient | null;
|
|
2030
|
+
scopePrefix?: string;
|
|
2031
|
+
} = {},
|
|
2032
|
+
): Promise<IngestResult> {
|
|
2033
|
+
const resolved = expandHome(vaultPath);
|
|
2034
|
+
|
|
2035
|
+
if (!isObsidianVault(resolved)) {
|
|
2036
|
+
return {
|
|
2037
|
+
source: "obsidian",
|
|
2038
|
+
filesProcessed: 0,
|
|
2039
|
+
chunksIngested: 0,
|
|
2040
|
+
chunksSkipped: 0,
|
|
2041
|
+
chunksDeduped: 0,
|
|
2042
|
+
dedupReasonCounts: createDedupReasonCounts(),
|
|
2043
|
+
errors: [`Not an Obsidian vault (missing .obsidian/ directory): ${resolved}`],
|
|
2044
|
+
};
|
|
2045
|
+
}
|
|
2046
|
+
|
|
2047
|
+
const output = scanVault(resolved, { scopePrefix: options.scopePrefix });
|
|
2048
|
+
const content = JSON.stringify(output);
|
|
2049
|
+
|
|
2050
|
+
if (options.verbose) {
|
|
2051
|
+
console.log(` obsidian: scanned ${output.records.length} notes from ${resolved}`);
|
|
2052
|
+
}
|
|
2053
|
+
|
|
2054
|
+
return ingestConnectorFile(store, embedder, content, options);
|
|
2055
|
+
}
|