recallnest 2.2.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +1 -1
- package/README.md +30 -5
- package/README_CN.md +30 -5
- package/connectors/examples/email-adapter.ts +73 -0
- package/connectors/examples/logs-adapter.ts +103 -0
- package/connectors/examples/rss-adapter.ts +95 -0
- package/logs/daily-recall.log +30 -0
- package/logs/ingest-2026-04-13.log +87 -0
- package/package.json +1 -1
- package/src/cli.ts +30 -0
- package/src/connector-types.ts +57 -0
- package/src/conversation-importer.ts +25 -1
- package/src/data-checkup.ts +31 -1
- package/src/doctor.ts +29 -0
- package/src/ingest.ts +203 -0
- package/src/mcp-server.ts +2 -2
- package/src/obsidian-connector.ts +282 -0
- package/src/session-schema.ts +2 -2
- package/src/source-heartbeat.ts +108 -0
package/CLAUDE.md
CHANGED
package/README.md
CHANGED
|
@@ -10,10 +10,10 @@ A local-first memory system backed by LanceDB that turns scattered conversation
|
|
|
10
10
|
|
|
11
11
|
[](https://github.com/AliceLJY/recallnest)
|
|
12
12
|
[](LICENSE)
|
|
13
|
-
[](https://bun.sh)
|
|
14
14
|
[](https://lancedb.com)
|
|
15
15
|
[](https://modelcontextprotocol.io)
|
|
16
|
-
[](https://github.com/AliceLJY/recallnest)
|
|
17
17
|
[](https://github.com/AliceLJY/recallnest)
|
|
18
18
|
|
|
19
19
|
**English** | [简体中文](README_CN.md) | [Roadmap](ROADMAP.md)
|
|
@@ -54,9 +54,20 @@ Wins or ties in **all 6 categories**, with no regression. The hybrid retrieval p
|
|
|
54
54
|
|
|
55
55
|
RecallNest starts automatically with Claude Code. No manual MCP config needed.
|
|
56
56
|
|
|
57
|
-
> **Requires:** [Bun](https://bun.sh)
|
|
57
|
+
> **Requires:** [Bun](https://bun.sh) (recommended) or Node.js 18+. Dependencies install on first start.
|
|
58
58
|
|
|
59
|
-
### Option B:
|
|
59
|
+
### Option B: npm install
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
npx recallnest --help # run directly
|
|
63
|
+
# or
|
|
64
|
+
npm install -g recallnest # install globally
|
|
65
|
+
recallnest doctor
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Works with Node.js 18+ (via tsx) or Bun. No git clone needed.
|
|
69
|
+
|
|
70
|
+
### Option C: Manual setup
|
|
60
71
|
|
|
61
72
|
```bash
|
|
62
73
|
git clone https://github.com/AliceLJY/recallnest.git
|
|
@@ -186,9 +197,11 @@ bun run src/ui-server.ts
|
|
|
186
197
|
| **Dashboard** | Web UI with stats, category distribution, growth trends, and health |
|
|
187
198
|
| **Workflow Observation** | Dedicated append-only workflow health records, outside regular memory |
|
|
188
199
|
| **Structured Assets** | Pins, briefs, and distilled summaries — not just raw logs |
|
|
189
|
-
| **Data Checkup** | Data quality health checks on the memory store |
|
|
200
|
+
| **Data Checkup** | Data quality health checks on the memory store (including source health) |
|
|
201
|
+
| **Source Heartbeats** | Automatic ingest health tracking per data source with staleness alerts |
|
|
190
202
|
| **Export Graph** | Export interactive HTML knowledge graph visualization |
|
|
191
203
|
| **Batch Operations** | Store up to 20 memories in a single call with dedup |
|
|
204
|
+
| **Connector Framework** | Standard connector-v1 format for external data sources with example adapters |
|
|
192
205
|
|
|
193
206
|
---
|
|
194
207
|
|
|
@@ -222,6 +235,18 @@ v2.1 added philosophy-informed behavior; v2.2 closes the last three engine-layer
|
|
|
222
235
|
|
|
223
236
|
---
|
|
224
237
|
|
|
238
|
+
## New in v2.3: Connector Ecosystem + Source Health
|
|
239
|
+
|
|
240
|
+
v2.2 hardened retrieval quality; v2.3 opens RecallNest to external data sources with a standard connector framework and operational health monitoring.
|
|
241
|
+
|
|
242
|
+
- **Connector-v1 Standard** *(GB-2)* — A JSON format (`ConnectorOutputV1`) that any external script can produce. Obsidian vaults, emails, RSS feeds, log files — normalize once, ingest through the full dedup/embed/extract pipeline. See [`docs/connector-spec.md`](docs/connector-spec.md) for the specification and [`connectors/examples/`](connectors/examples/) for adapter skeletons (email, logs, RSS).
|
|
243
|
+
|
|
244
|
+
- **Obsidian Vault Ingestion** *(GB-1)* — First-party Obsidian connector: scans `.md` files, extracts frontmatter + wikilinks, maps folder structure to tags. One command: `lm ingest --obsidian /path/to/vault`.
|
|
245
|
+
|
|
246
|
+
- **Source Health Monitoring** *(GB-3)* — Every connector ingest writes a heartbeat to `data/source-heartbeat.json`. `data_checkup` flags stale sources (>7d warning, >30d error). `doctor --ci` shows a per-source heartbeat summary with human-readable age.
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
225
250
|
## Architecture
|
|
226
251
|
|
|
227
252
|
```
|
package/README_CN.md
CHANGED
|
@@ -10,10 +10,10 @@
|
|
|
10
10
|
|
|
11
11
|
[](https://github.com/AliceLJY/recallnest)
|
|
12
12
|
[](LICENSE)
|
|
13
|
-
[](https://bun.sh)
|
|
14
14
|
[](https://lancedb.com)
|
|
15
15
|
[](https://modelcontextprotocol.io)
|
|
16
|
-
[](https://github.com/AliceLJY/recallnest)
|
|
17
17
|
[](https://github.com/AliceLJY/recallnest)
|
|
18
18
|
|
|
19
19
|
[English](README.md) | **简体中文** | [Roadmap](ROADMAP.md)
|
|
@@ -54,9 +54,20 @@ RecallNest 的解法:**一个 LanceDB 驱动的记忆层,三个终端共读
|
|
|
54
54
|
|
|
55
55
|
RecallNest 随 Claude Code 自动启动,无需手动配置 MCP。
|
|
56
56
|
|
|
57
|
-
> **前置要求:**
|
|
57
|
+
> **前置要求:** [Bun](https://bun.sh)(推荐)或 Node.js 18+。首次启动自动安装依赖。
|
|
58
58
|
|
|
59
|
-
### 方式 B
|
|
59
|
+
### 方式 B:npm 安装
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
npx recallnest --help # 直接运行
|
|
63
|
+
# 或
|
|
64
|
+
npm install -g recallnest # 全局安装
|
|
65
|
+
recallnest doctor
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
支持 Node.js 18+(通过 tsx)或 Bun,无需 clone 仓库。
|
|
69
|
+
|
|
70
|
+
### 方式 C:手动安装
|
|
60
71
|
|
|
61
72
|
```bash
|
|
62
73
|
git clone https://github.com/AliceLJY/recallnest.git
|
|
@@ -186,9 +197,11 @@ bun run src/ui-server.ts
|
|
|
186
197
|
| **Dashboard** | Web UI 首页:统计卡片、类别分布、增长趋势、健康概览 |
|
|
187
198
|
| **Workflow Observation** | 专门的 append-only 工作流观测层,不混入普通 memory |
|
|
188
199
|
| **结构化资产** | Pin、Brief、Distill —— 不只是原始日志 |
|
|
189
|
-
| **Data Checkup** |
|
|
200
|
+
| **Data Checkup** | 记忆存储数据质量健康检查(含数据源健康) |
|
|
201
|
+
| **数据源心跳** | 按数据源自动追踪 ingest 健康状态,过期告警 |
|
|
190
202
|
| **导出图谱** | 导出交互式 HTML 知识图谱可视化 |
|
|
191
203
|
| **批量操作** | 单次调用存储最多 20 条记忆,自带去重 |
|
|
204
|
+
| **Connector 框架** | 标准 connector-v1 格式接入外部数据源,附带适配器示例 |
|
|
192
205
|
|
|
193
206
|
---
|
|
194
207
|
|
|
@@ -222,6 +235,18 @@ v2.1 加入了哲学记忆行为;v2.2 补齐前沿扫描(ACC、PI-LLM、TSM
|
|
|
222
235
|
|
|
223
236
|
---
|
|
224
237
|
|
|
238
|
+
## v2.3 新增:Connector 生态 + 数据源健康
|
|
239
|
+
|
|
240
|
+
v2.2 强化了检索质量;v2.3 通过标准 Connector 框架和运维健康监控,把 RecallNest 开放给外部数据源。
|
|
241
|
+
|
|
242
|
+
- **Connector-v1 标准** *(GB-2)* —— 一套 JSON 格式(`ConnectorOutputV1`),任何外部脚本都可以输出。Obsidian 笔记、邮件、RSS、日志文件——标准化一次,走完整的 dedup/embed/extract 管线入库。规范见 [`docs/connector-spec.md`](docs/connector-spec.md),适配器骨架见 [`connectors/examples/`](connectors/examples/)(email、logs、RSS)。
|
|
243
|
+
|
|
244
|
+
- **Obsidian Vault 接入** *(GB-1)* —— 官方 Obsidian 连接器:扫描 `.md` 文件,提取 frontmatter + wikilink,文件夹结构映射为 tag。一条命令:`lm ingest --obsidian /path/to/vault`。
|
|
245
|
+
|
|
246
|
+
- **数据源健康监控** *(GB-3)* —— 每次 connector ingest 写心跳到 `data/source-heartbeat.json`。`data_checkup` 标记过期数据源(>7 天 warning,>30 天 error)。`doctor --ci` 按数据源展示心跳摘要(如"obsidian: 2h ago")。
|
|
247
|
+
|
|
248
|
+
---
|
|
249
|
+
|
|
225
250
|
## 架构
|
|
226
251
|
|
|
227
252
|
```
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Email Connector Adapter (skeleton)
|
|
4
|
+
*
|
|
5
|
+
* Demonstrates how to produce a ConnectorOutputV1 JSON file from an email
|
|
6
|
+
* source (IMAP, Gmail API, local mbox, etc.) for RecallNest ingestion.
|
|
7
|
+
*
|
|
8
|
+
* Usage:
|
|
9
|
+
* bun run connectors/examples/email-adapter.ts > email-output.json
|
|
10
|
+
* lm ingest --connector email-output.json
|
|
11
|
+
*
|
|
12
|
+
* Customize:
|
|
13
|
+
* 1. Replace the fetchEmails() stub with your IMAP/API client
|
|
14
|
+
* 2. Adjust scope to match your project naming
|
|
15
|
+
* 3. Set categoryHint based on email type (meeting invite → "events", etc.)
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { ConnectorOutputV1 } from "../../src/connector-types.js";
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// TODO: Replace with real email fetching logic
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
interface RawEmail {
|
|
26
|
+
messageId: string;
|
|
27
|
+
subject: string;
|
|
28
|
+
body: string;
|
|
29
|
+
from: string;
|
|
30
|
+
date: string; // ISO 8601
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function fetchEmails(_since: Date): Promise<RawEmail[]> {
|
|
34
|
+
// Example: use imapflow, googleapis, or read from .mbox
|
|
35
|
+
// const client = new ImapFlow({ host: 'imap.example.com', ... });
|
|
36
|
+
// await client.connect();
|
|
37
|
+
// const messages = await client.fetch(...);
|
|
38
|
+
return []; // Replace with real implementation
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
// Transform → ConnectorOutputV1
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
async function main() {
|
|
46
|
+
const since = new Date(Date.now() - 7 * 86_400_000); // last 7 days
|
|
47
|
+
const emails = await fetchEmails(since);
|
|
48
|
+
|
|
49
|
+
const output: ConnectorOutputV1 = {
|
|
50
|
+
version: "connector-v1",
|
|
51
|
+
source: "email",
|
|
52
|
+
scope: "inbox:personal", // Customize: "inbox:work", "project:xyz"
|
|
53
|
+
producedAt: new Date().toISOString(),
|
|
54
|
+
records: emails.map((email) => ({
|
|
55
|
+
id: `email:${email.messageId}`,
|
|
56
|
+
text: email.body,
|
|
57
|
+
title: email.subject,
|
|
58
|
+
categoryHint: "events", // Customize per email type
|
|
59
|
+
importanceHint: 0.6, // Customize: 0.9 for flagged/starred
|
|
60
|
+
tags: ["email", `from:${email.from}`],
|
|
61
|
+
timestamp: email.date,
|
|
62
|
+
contentHash: createHash("sha256").update(email.body).digest("hex"),
|
|
63
|
+
sourceMetadata: {
|
|
64
|
+
from: email.from,
|
|
65
|
+
messageId: email.messageId,
|
|
66
|
+
},
|
|
67
|
+
})),
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
console.log(JSON.stringify(output, null, 2));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
main().catch(console.error);
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* Log File Connector Adapter (skeleton)
|
|
4
|
+
*
|
|
5
|
+
* Demonstrates how to produce a ConnectorOutputV1 JSON file from application
|
|
6
|
+
* log files for RecallNest ingestion. Useful for tracking deploy events,
|
|
7
|
+
* error patterns, and operational decisions.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* bun run connectors/examples/logs-adapter.ts /var/log/myapp/*.log > logs-output.json
|
|
11
|
+
* lm ingest --connector logs-output.json
|
|
12
|
+
*
|
|
13
|
+
* Customize:
|
|
14
|
+
* 1. Replace parseLogLine() with your log format parser
|
|
15
|
+
* 2. Adjust the importance/category logic for your use case
|
|
16
|
+
* 3. Set scope to match your project
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { ConnectorOutputV1 } from "../../src/connector-types.js";
|
|
20
|
+
import { readFileSync } from "node:fs";
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
// TODO: Replace with your log format parser
|
|
25
|
+
// ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
interface ParsedLogEntry {
|
|
28
|
+
timestamp: string; // ISO 8601
|
|
29
|
+
level: "info" | "warn" | "error" | "debug";
|
|
30
|
+
message: string;
|
|
31
|
+
source?: string; // e.g. module name, service
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function parseLogLine(line: string): ParsedLogEntry | null {
|
|
35
|
+
// Example: "2026-04-14T10:00:00Z [ERROR] auth: login failed for user@example.com"
|
|
36
|
+
const match = line.match(
|
|
37
|
+
/^(\d{4}-\d{2}-\d{2}T[\d:.]+Z?)\s+\[(\w+)]\s+(?:(\w+):\s+)?(.+)$/,
|
|
38
|
+
);
|
|
39
|
+
if (!match) return null;
|
|
40
|
+
return {
|
|
41
|
+
timestamp: match[1],
|
|
42
|
+
level: match[2].toLowerCase() as ParsedLogEntry["level"],
|
|
43
|
+
source: match[3],
|
|
44
|
+
message: match[4],
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function levelToImportance(level: string): number {
|
|
49
|
+
switch (level) {
|
|
50
|
+
case "error": return 0.9;
|
|
51
|
+
case "warn": return 0.7;
|
|
52
|
+
case "info": return 0.5;
|
|
53
|
+
default: return 0.3;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// ---------------------------------------------------------------------------
|
|
58
|
+
// Transform → ConnectorOutputV1
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
function main() {
|
|
62
|
+
const files = process.argv.slice(2);
|
|
63
|
+
if (files.length === 0) {
|
|
64
|
+
console.error("Usage: logs-adapter.ts <logfile> [logfile...]");
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const records: ConnectorOutputV1["records"] = [];
|
|
69
|
+
|
|
70
|
+
for (const file of files) {
|
|
71
|
+
const lines = readFileSync(file, "utf-8").split("\n").filter(Boolean);
|
|
72
|
+
for (const line of lines) {
|
|
73
|
+
const entry = parseLogLine(line);
|
|
74
|
+
if (!entry) continue;
|
|
75
|
+
|
|
76
|
+
// Skip debug-level by default — too noisy for memory storage
|
|
77
|
+
if (entry.level === "debug") continue;
|
|
78
|
+
|
|
79
|
+
records.push({
|
|
80
|
+
id: `log:${createHash("md5").update(line).digest("hex")}`,
|
|
81
|
+
text: entry.message,
|
|
82
|
+
title: entry.source ? `[${entry.level.toUpperCase()}] ${entry.source}` : undefined,
|
|
83
|
+
categoryHint: entry.level === "error" ? "cases" : "events",
|
|
84
|
+
importanceHint: levelToImportance(entry.level),
|
|
85
|
+
tags: ["log", entry.level, ...(entry.source ? [`module:${entry.source}`] : [])],
|
|
86
|
+
timestamp: entry.timestamp,
|
|
87
|
+
contentHash: createHash("sha256").update(line).digest("hex"),
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const output: ConnectorOutputV1 = {
|
|
93
|
+
version: "connector-v1",
|
|
94
|
+
source: "logs",
|
|
95
|
+
scope: "ops:myapp", // Customize: "project:xyz", "ops:production"
|
|
96
|
+
producedAt: new Date().toISOString(),
|
|
97
|
+
records,
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
console.log(JSON.stringify(output, null, 2));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
main();
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* RSS Feed Connector Adapter (skeleton)
|
|
4
|
+
*
|
|
5
|
+
* Demonstrates how to produce a ConnectorOutputV1 JSON file from RSS/Atom
|
|
6
|
+
* feeds for RecallNest ingestion. Useful for tracking industry news,
|
|
7
|
+
* blog posts, and research updates.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* bun run connectors/examples/rss-adapter.ts > rss-output.json
|
|
11
|
+
* lm ingest --connector rss-output.json
|
|
12
|
+
*
|
|
13
|
+
* Customize:
|
|
14
|
+
* 1. Replace fetchFeed() with your RSS parsing library (e.g. rss-parser)
|
|
15
|
+
* 2. Adjust feed URLs and scope
|
|
16
|
+
* 3. Set categoryHint / importanceHint based on feed type
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { ConnectorOutputV1 } from "../../src/connector-types.js";
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
// TODO: Replace with real RSS/Atom parsing
|
|
24
|
+
// ---------------------------------------------------------------------------
|
|
25
|
+
|
|
26
|
+
interface FeedItem {
|
|
27
|
+
guid: string;
|
|
28
|
+
title: string;
|
|
29
|
+
content: string; // HTML stripped to plain text
|
|
30
|
+
link: string;
|
|
31
|
+
pubDate: string; // ISO 8601
|
|
32
|
+
categories?: string[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface FeedConfig {
|
|
36
|
+
url: string;
|
|
37
|
+
scope: string; // e.g. "feeds:ai-news"
|
|
38
|
+
categoryHint?: string;
|
|
39
|
+
importanceHint?: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const FEEDS: FeedConfig[] = [
|
|
43
|
+
// Add your feeds here:
|
|
44
|
+
// { url: "https://example.com/feed.xml", scope: "feeds:example" },
|
|
45
|
+
];
|
|
46
|
+
|
|
47
|
+
async function fetchFeed(_url: string): Promise<FeedItem[]> {
|
|
48
|
+
// Example with rss-parser:
|
|
49
|
+
// import Parser from "rss-parser";
|
|
50
|
+
// const parser = new Parser();
|
|
51
|
+
// const feed = await parser.parseURL(url);
|
|
52
|
+
// return feed.items.map(item => ({ ... }));
|
|
53
|
+
return []; // Replace with real implementation
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
// Transform → ConnectorOutputV1
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
|
|
60
|
+
async function main() {
|
|
61
|
+
for (const feed of FEEDS) {
|
|
62
|
+
const items = await fetchFeed(feed.url);
|
|
63
|
+
|
|
64
|
+
const output: ConnectorOutputV1 = {
|
|
65
|
+
version: "connector-v1",
|
|
66
|
+
source: "rss",
|
|
67
|
+
scope: feed.scope,
|
|
68
|
+
producedAt: new Date().toISOString(),
|
|
69
|
+
records: items.map((item) => ({
|
|
70
|
+
id: `rss:${createHash("md5").update(item.guid || item.link).digest("hex")}`,
|
|
71
|
+
text: item.content,
|
|
72
|
+
title: item.title,
|
|
73
|
+
categoryHint: feed.categoryHint ?? "knowledge",
|
|
74
|
+
importanceHint: feed.importanceHint ?? 0.6,
|
|
75
|
+
tags: [
|
|
76
|
+
"rss",
|
|
77
|
+
...(item.categories ?? []),
|
|
78
|
+
`link:${item.link}`,
|
|
79
|
+
],
|
|
80
|
+
timestamp: item.pubDate,
|
|
81
|
+
contentHash: createHash("sha256").update(item.content).digest("hex"),
|
|
82
|
+
sourceMetadata: {
|
|
83
|
+
feedUrl: feed.url,
|
|
84
|
+
link: item.link,
|
|
85
|
+
guid: item.guid,
|
|
86
|
+
},
|
|
87
|
+
})),
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
// Each feed outputs one JSON — pipe to separate files if needed
|
|
91
|
+
console.log(JSON.stringify(output, null, 2));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
main().catch(console.error);
|
package/logs/daily-recall.log
CHANGED
|
@@ -28,3 +28,33 @@ Deleted : 0
|
|
|
28
28
|
Errors : 0
|
|
29
29
|
=== 2026-04-12 08:00:13 daily-recall 结束 ===
|
|
30
30
|
=== 2026-04-12 08:00:13 daily-recall 结束 ===
|
|
31
|
+
=== 2026-04-13 08:00:01 daily-recall 开始 ===
|
|
32
|
+
=== 2026-04-13 08:00:01 daily-recall 开始 ===
|
|
33
|
+
今日主题 (#1): 成长 经历 改变了什么
|
|
34
|
+
今日主题 (#1): 成长 经历 改变了什么
|
|
35
|
+
检查 distill gate...
|
|
36
|
+
检查 distill gate...
|
|
37
|
+
distill gate:
|
|
38
|
+
distill gate:
|
|
39
|
+
执行 distill...
|
|
40
|
+
执行 distill...
|
|
41
|
+
distill 完成,31 行
|
|
42
|
+
distill 完成,31 行
|
|
43
|
+
[完成] 回顾已写入: /Users/anxianjingya/recallnest/daily-reflections/2026-04-13.md
|
|
44
|
+
[完成] 回顾已写入: /Users/anxianjingya/recallnest/daily-reflections/2026-04-13.md
|
|
45
|
+
执行 checkpoint-gc...
|
|
46
|
+
执行 checkpoint-gc...
|
|
47
|
+
checkpoint-gc: === Checkpoint GC ===
|
|
48
|
+
|
|
49
|
+
Kept : 218
|
|
50
|
+
Archived : 0
|
|
51
|
+
Deleted : 0
|
|
52
|
+
Errors : 0
|
|
53
|
+
checkpoint-gc: === Checkpoint GC ===
|
|
54
|
+
|
|
55
|
+
Kept : 218
|
|
56
|
+
Archived : 0
|
|
57
|
+
Deleted : 0
|
|
58
|
+
Errors : 0
|
|
59
|
+
=== 2026-04-13 08:00:22 daily-recall 结束 ===
|
|
60
|
+
=== 2026-04-13 08:00:22 daily-recall 结束 ===
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
=== 2026-04-13 03:00:04 增量更新开始 ===
|
|
2
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
3
|
+
|
|
4
|
+
🔑 验证 Embedding API...
|
|
5
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
6
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
7
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
8
|
+
|
|
9
|
+
🔄 开始导入记忆 (source: all)...
|
|
10
|
+
|
|
11
|
+
📝 导入 Claude Code 对话...
|
|
12
|
+
[INFO] noise-filter: short boilerplate filtered: "收到!多路并行处理:"
|
|
13
|
+
CC: 430/1002 files, 87 chunks ingested
|
|
14
|
+
CC: 680/1002 files, 217 chunks ingested
|
|
15
|
+
✅ CC: 23 files, 378 ingested, 655 deduped (hard:67, exact:279, llm-skip:202, llm-merge:107), 0 errors
|
|
16
|
+
🤖 导入 Codex 对话...
|
|
17
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
18
|
+
💎 导入 Gemini 对话...
|
|
19
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
20
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
21
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
22
|
+
📚 导入记忆文件...
|
|
23
|
+
✅ Memory: 7 files, 26 ingested, 216 deduped (hard:1, exact:192, llm-skip:16, llm-merge:7), 0 errors
|
|
24
|
+
|
|
25
|
+
📊 导入汇总:
|
|
26
|
+
总计: 404 chunks 已索引, 871 chunks 去重 (hard:68, exact:471, llm-skip:218, llm-merge:114), 0 errors
|
|
27
|
+
|
|
28
|
+
=== 2026-04-13 03:13:05 增量更新完成 ===
|
|
29
|
+
|
|
30
|
+
=== 2026-04-13 11:13:05 增量更新开始 ===
|
|
31
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
32
|
+
|
|
33
|
+
🔑 验证 Embedding API...
|
|
34
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
35
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
36
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
37
|
+
|
|
38
|
+
🔄 开始导入记忆 (source: all)...
|
|
39
|
+
|
|
40
|
+
📝 导入 Claude Code 对话...
|
|
41
|
+
CC: 480/1006 files, 20 chunks ingested
|
|
42
|
+
[INFO] noise-filter: denial pattern matched: "好,全文读完了。这篇文章标题是「Claude Code 省 Token 指南:慎用 1M 上下文,不开新会话或者总是开新..."
|
|
43
|
+
✅ CC: 5 files, 130 ingested, 153 deduped (hard:14, exact:44, llm-skip:46, llm-merge:49), 0 errors
|
|
44
|
+
🤖 导入 Codex 对话...
|
|
45
|
+
[INFO] noise-filter: meta-question filtered: "# AGENTS.md instructions for /Users/anxianjingya
|
|
46
|
+
|
|
47
|
+
<INSTRUCTI..."
|
|
48
|
+
Codex: 295/295 files, 2 chunks ingested
|
|
49
|
+
✅ Codex: 1 files, 2 ingested, 1 deduped (hard:0, exact:0, llm-skip:0, llm-merge:1), 0 errors
|
|
50
|
+
💎 导入 Gemini 对话...
|
|
51
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
52
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
53
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
54
|
+
📚 导入记忆文件...
|
|
55
|
+
✅ Memory: 1 files, 1 ingested, 185 deduped (hard:0, exact:172, llm-skip:13, llm-merge:0), 0 errors
|
|
56
|
+
|
|
57
|
+
📊 导入汇总:
|
|
58
|
+
总计: 133 chunks 已索引, 339 chunks 去重 (hard:14, exact:216, llm-skip:59, llm-merge:50), 0 errors
|
|
59
|
+
|
|
60
|
+
=== 2026-04-13 11:19:05 增量更新完成 ===
|
|
61
|
+
|
|
62
|
+
=== 2026-04-13 19:19:05 增量更新开始 ===
|
|
63
|
+
[INFO] LLM client initialized: qwen-turbo @ https://dashscope.aliyuncs.com/compatible-mode/v1
|
|
64
|
+
|
|
65
|
+
🔑 验证 Embedding API...
|
|
66
|
+
✅ jina-embeddings-v5-text-small (1024d)
|
|
67
|
+
✅ LLM: qwen-turbo (L0 摘要 + 语义去重)
|
|
68
|
+
✅ 两阶段去重: vector + LLM 语义判断
|
|
69
|
+
|
|
70
|
+
🔄 开始导入记忆 (source: all)...
|
|
71
|
+
|
|
72
|
+
📝 导入 Claude Code 对话...
|
|
73
|
+
✅ CC: 12 files, 193 ingested, 225 deduped (hard:19, exact:71, llm-skip:74, llm-merge:61), 0 errors
|
|
74
|
+
🤖 导入 Codex 对话...
|
|
75
|
+
✅ Codex: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
76
|
+
💎 导入 Gemini 对话...
|
|
77
|
+
✅ Gemini: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
78
|
+
🖥️ 导入 Claude Desktop 对话...
|
|
79
|
+
✅ Desktop: 0 files, 0 ingested, 0 deduped (hard:0, exact:0, llm-skip:0, llm-merge:0), 0 errors
|
|
80
|
+
📚 导入记忆文件...
|
|
81
|
+
✅ Memory: 6 files, 16 ingested, 229 deduped (hard:0, exact:202, llm-skip:16, llm-merge:11), 0 errors
|
|
82
|
+
|
|
83
|
+
📊 导入汇总:
|
|
84
|
+
总计: 209 chunks 已索引, 454 chunks 去重 (hard:19, exact:273, llm-skip:90, llm-merge:72), 0 errors
|
|
85
|
+
|
|
86
|
+
=== 2026-04-13 19:27:06 增量更新完成 ===
|
|
87
|
+
|
package/package.json
CHANGED
package/src/cli.ts
CHANGED
|
@@ -29,6 +29,8 @@ import {
|
|
|
29
29
|
ingestCodexSessions,
|
|
30
30
|
ingestGeminiSessions,
|
|
31
31
|
ingestMarkdownFiles,
|
|
32
|
+
ingestObsidianVault,
|
|
33
|
+
ingestConnectorFile,
|
|
32
34
|
} from "./ingest.js";
|
|
33
35
|
import { runDoctor, formatDoctorResults } from "./doctor.js";
|
|
34
36
|
import { persistCaseMemory, persistMemory, persistWorkflowPattern } from "./capture-engine.js";
|
|
@@ -1306,6 +1308,8 @@ program
|
|
|
1306
1308
|
.option("--no-dedup", "跳过向量去重")
|
|
1307
1309
|
.option("--no-llm", "禁用 LLM 提取(原始对话将跳过不存入,排入待处理队列)")
|
|
1308
1310
|
.option("--recent [hours]", "快速模式:只处理最近 N 小时修改的文件(默认 2 小时),跳过 ingested-files 检查")
|
|
1311
|
+
.option("--obsidian <vault-path>", "导入 Obsidian vault(自动检测 .obsidian/ 目录)")
|
|
1312
|
+
.option("--connector <file-path>", "导入 connector-v1 格式 JSON 文件")
|
|
1309
1313
|
.action(async (options) => {
|
|
1310
1314
|
const config = loadConfig();
|
|
1311
1315
|
const { store, embedder, llm } = createComponents(config);
|
|
@@ -1440,6 +1444,32 @@ program
|
|
|
1440
1444
|
}
|
|
1441
1445
|
}
|
|
1442
1446
|
|
|
1447
|
+
// Obsidian vault (--obsidian <path>)
|
|
1448
|
+
if (options.obsidian) {
|
|
1449
|
+
console.log("🗃️ 导入 Obsidian vault...");
|
|
1450
|
+
const r = await ingestObsidianVault(store, embedder, options.obsidian, {
|
|
1451
|
+
verbose,
|
|
1452
|
+
noDedup,
|
|
1453
|
+
llm: effectiveLlm,
|
|
1454
|
+
});
|
|
1455
|
+
results.push(r);
|
|
1456
|
+
console.log(` ✅ Obsidian: ${formatIngestSummary(r)}`);
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
// Connector-v1 file (--connector <path>)
|
|
1460
|
+
if (options.connector) {
|
|
1461
|
+
console.log("🔌 导入 connector-v1 文件...");
|
|
1462
|
+
const { readFileSync: readFs } = await import("node:fs");
|
|
1463
|
+
const connectorContent = readFs(options.connector, "utf-8");
|
|
1464
|
+
const r = await ingestConnectorFile(store, embedder, connectorContent, {
|
|
1465
|
+
verbose,
|
|
1466
|
+
noDedup,
|
|
1467
|
+
llm: effectiveLlm,
|
|
1468
|
+
});
|
|
1469
|
+
results.push(r);
|
|
1470
|
+
console.log(` ✅ Connector: ${formatIngestSummary(r)}`);
|
|
1471
|
+
}
|
|
1472
|
+
|
|
1443
1473
|
// Summary
|
|
1444
1474
|
console.log("\n📊 导入汇总:");
|
|
1445
1475
|
let totalChunks = 0;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Connector Standard Types (connector-v1)
|
|
3
|
+
*
|
|
4
|
+
* 定义外部数据源(Obsidian、Email、RSS、自定义脚本)统一产出的 JSON 格式。
|
|
5
|
+
* ingest.ts 的 ingestConnectorFile 消费这个格式,走已有的 dedup/extract/入库管线。
|
|
6
|
+
*
|
|
7
|
+
* 协议版本管理:version 字段用于未来兼容性。当前固定 "connector-v1"。
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export interface ConnectorOutputV1 {
|
|
11
|
+
/** 格式版本,固定 "connector-v1" */
|
|
12
|
+
version: "connector-v1";
|
|
13
|
+
/** 数据源标识,如 "obsidian", "email", "rss", "x-bookmarks" */
|
|
14
|
+
source: string;
|
|
15
|
+
/** 目标 scope,如 "vault:my-notes", "project:research" */
|
|
16
|
+
scope: string;
|
|
17
|
+
/** 批次 ID(可选,用于幂等重试) */
|
|
18
|
+
batchId?: string;
|
|
19
|
+
/** 产出时间 ISO 8601 */
|
|
20
|
+
producedAt: string;
|
|
21
|
+
/** 记录列表 */
|
|
22
|
+
records: ConnectorRecord[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface ConnectorRecord {
|
|
26
|
+
/** 记录唯一 ID(source 内唯一,用于增量去重) */
|
|
27
|
+
id: string;
|
|
28
|
+
/** 正文 */
|
|
29
|
+
text: string;
|
|
30
|
+
/** 标题/heading(可选) */
|
|
31
|
+
title?: string;
|
|
32
|
+
/** 分类提示:preferences / events / knowledge / facts / decisions */
|
|
33
|
+
categoryHint?: string;
|
|
34
|
+
/** 重要性提示 0-1(可选,默认 0.7) */
|
|
35
|
+
importanceHint?: number;
|
|
36
|
+
/** 标签 */
|
|
37
|
+
tags?: string[];
|
|
38
|
+
/** 原始时间戳 ISO 8601 */
|
|
39
|
+
timestamp?: string;
|
|
40
|
+
/** 内容哈希(用于增量同步,调用方计算) */
|
|
41
|
+
contentHash?: string;
|
|
42
|
+
/** 来源元数据(自由字段) */
|
|
43
|
+
sourceMetadata?: Record<string, unknown>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** 运行时格式验证:判定一个 parsed JSON 是否是合法 connector-v1 */
|
|
47
|
+
export function isConnectorOutputV1(value: unknown): value is ConnectorOutputV1 {
|
|
48
|
+
if (!value || typeof value !== "object") return false;
|
|
49
|
+
const obj = value as Record<string, unknown>;
|
|
50
|
+
return (
|
|
51
|
+
obj.version === "connector-v1"
|
|
52
|
+
&& typeof obj.source === "string"
|
|
53
|
+
&& typeof obj.scope === "string"
|
|
54
|
+
&& typeof obj.producedAt === "string"
|
|
55
|
+
&& Array.isArray(obj.records)
|
|
56
|
+
);
|
|
57
|
+
}
|