recallnest 1.4.1

package/plan-v1.2-distributable.md

@@ -0,0 +1,105 @@
+# RecallNest v1.2: 首个可传播版本 Plan
+
+> 目标：别人 15 分钟能跑起来，30 分钟能感知价值
+> 负责人：CC (实现) + 用户 (审批)
+> 预计：3-4 天
+
+---
+
+## 阻塞项（必须修，按优先级排）
+
+### B1: `recallnest doctor` 命令（新增） ✅ DONE
+**问题**：用户装完后没有一键验证手段，每个环节都可能静默失败
+**方案**：新增 `lm doctor` 子命令，检查清单：
+- [x] Bun 运行时可用？（检测版本）
+- [x] Jina API key 已配置？（读 .env）
+- [x] Jina API key 有效？（发一个测试 embedding 请求）
+- [x] CC transcript 路径可访问？（auto-detect 或 config）
+- [x] LanceDB 数据目录可写？
+- [x] 已有索引？多少条？
+- 输出：绿色 PASS / 红色 FAIL + 修复建议
+
+### B2: README 安装体验重写 ✅ DONE
+**问题**：Bun 未提及、Jina key 获取无引导、步骤跳跃
+**方案**：
+- [x] Prerequisites 段：明确列出 Bun ≥1.0 + Node ≥20（Bun 优先，Node 作为 fallback 说明）
+- [x] 快速开始改为 5 步 copy-paste：clone → install → 配 key → doctor → search
+- [x] 每步附带预期输出（用户知道"成功了"长什么样）
+- [x] 添加 "Troubleshooting" 段：Bun not found / Jina 401 / 路径不存在
+
+### B3: Jina API key 前置验证 ✅ DONE
+**问题**：config 加载成功，但第一次 ingest 才报 embedding 失败，错误信息是 OpenAI SDK 的原始 401
+**方案**：
+- [x] `ingest` 命令启动时，先调 embedder 发一个测试请求（embed "test"）
+- [x] 失败则立即报错：`Jina API key invalid or missing. Get one at https://jina.ai/embeddings/`
+- [x] `doctor` 命令复用同一验证逻辑
+
+### B4: Gemini 支持诚实化 ✅ DONE
+**问题**：README 说支持 Gemini CLI，config 里有 gemini 源，但实际 Gemini 会话是加密 protobuf，解析不了
+**方案**：
+- [x] README 中 Gemini 行标注为 `Coming soon`（不是 ✅）
+- [x] config.json.example 中移除 gemini source（新用户不会误配）
+- [x] ingest 遇到 gemini source 时打印明确 warning：`Gemini CLI sessions are encrypted protobuf; skipping`
+- [x] doctor 显示 Gemini 状态为 warn（已有配置的用户看到明确提示）
+- 不删代码，保留未来扩展能力
+
+### B5: 配置路径健壮化 ✅ DONE
+**问题**：`./data/lancedb` 是相对路径，换目录就炸；auto-detect 失败后建议不够具体
+**方案**：
+- [x] config.json.example 默认路径改为 `~/.recallnest/data/lancedb`（绝对，用户无感）
+- [x] 首次运行自动创建目录（validateStoragePath 已有 mkdirSync recursive）
+- [x] auto-detect 失败时，给出完整示例（含用户名的真实路径）
+- [x] config.json 加入 .gitignore，用户配置不被 git 覆盖
+- [x] findConfigPath 失败时提示 `cp config.json.example config.json`
+
+---
+
+## 加分项（做了更好，不阻塞发布）
+
+### P1: Demo 查询内置 ✅ DONE
+- [x] `lm demo` 命令：用 3 个内置 query 跑搜索，展示结果格式
+- [x] 显示索引统计 + 每条结果的分数/来源/预览
+
+### P2: 一键安装脚本
+- [ ] `curl -fsSL https://raw.githubusercontent.com/.../install.sh | bash`
+- [ ] 检测 Bun → clone → npm install → 提示配 key → 跑 doctor
+- [ ] 可以 Q2 再做，现在 README 手动步骤够用
+
+### P3: CI 基础 ✅ DONE
+- [x] GitHub Actions：push/PR 时跑 `doctor --ci` + TypeScript check
+- [x] `lm doctor --ci` 模式跳过 API key 在线验证
+
+### P4: CHANGELOG.md ✅ DONE
+- [x] v1.0.0 到 v1.2.0 的变更记录
+- [x] 后续每个 release 维护
+
+---
+
+## 不做的事（明确排除）
+
+- ❌ 不做 Docker 化（用户群是 CLI/MCP 用户，不是 Docker 用户）
+- ❌ 不做 NPM 发布（先 git clone 模式验证需求）
+- ❌ 不做 UI 重写（现有 UI 够用，不是本次重点）
+- ❌ 不做多用户支持（单 operator 场景）
+- ❌ 不重写检索逻辑（已经够好，不是本次范围）
+
+---
+
+## 执行顺序
+
+```
+Day 1: B1 (doctor) + B3 (Jina 验证) — 核心安全网 ✅
+Day 2: B5 (路径健壮化) + B4 (Gemini 诚实化) + B2 (README 重写) ✅
+Day 3: P1 (demo 查询) + P3 (CI) + P4 (CHANGELOG) ✅
+Day 4: 自测完整流程
+```
+
+---
+
+## 验收标准
+
+1. 全新 macOS 机器（有 Bun），从 clone 到第一条搜索结果 ≤ 15 分钟
+2. `lm doctor` 全绿
+3. Jina key 错误时，3 秒内给出明确修复建议
+4. README 无需翻代码即可完成安装
+5. `lm search "telegram bridge"` 返回有意义结果

package/scripts/archive-distilled-junk.ts

@@ -0,0 +1,116 @@
+#!/usr/bin/env bun
+/**
+ * Archive raw fact fragments that have already been distilled.
+ * Targets: entries in distill-completed scopes with importance < 0.4, no L0, category=fact.
+ * Action: set tier="archived", importance=0.05 — they won't surface but data isn't lost.
+ *
+ * Usage:
+ *   bun scripts/archive-distilled-junk.ts --dry-run   # preview
+ *   bun scripts/archive-distilled-junk.ts              # execute
+ */
+
+import fs from "fs";
+import path from "path";
+import { loadDotEnv, loadConfig, expandHome } from "../src/runtime-config.js";
+import { loadLanceDB } from "../src/store.js";
+
+loadDotEnv();
+const config = loadConfig();
+const dryRun = process.argv.includes("--dry-run");
+
+const PROGRESS_FILE = path.join(import.meta.dir, "..", "data", "distill-progress.json");
+
+async function main() {
+  // Load distill progress to know which scopes are done
+  const progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, "utf-8"));
+  const completedScopes = new Set<string>(progress.completedScopes);
+  console.log(`=== Archive distilled junk${dryRun ? " (dry run)" : ""} ===\n`);
+  console.log(`  Distill-completed scopes: ${completedScopes.size}`);
+
+  const lancedb = await loadLanceDB();
+  const dbPath = expandHome(config.database?.path || "data/lancedb");
+  const db = await lancedb.connect(dbPath);
+  const table = await db.openTable("memories");
+
+  // Read all fact entries
+  const allFacts = await table.query()
+    .where("category = 'fact'")
+    .select(["id", "text", "vector", "category", "scope", "importance", "timestamp", "metadata"])
+    .toArray();
+
+  console.log(`  Total fact entries: ${allFacts.length}`);
+
+  // Filter: in completed scope, importance < 0.4, no L0
+  const toArchive: any[] = [];
+  const sampleJunk: string[] = [];
+
+  for (const row of allFacts) {
+    const scope = (row.scope as string) || "";
+    if (!completedScopes.has(scope)) continue;
+
+    const imp = Number(row.importance);
+    if (imp >= 0.4) continue;
+
+    const metaStr = (row.metadata as string) || "{}";
+    let meta: Record<string, unknown>;
+    try { meta = JSON.parse(metaStr); } catch { continue; }
+
+    // Has L0 = already a quality entry, skip
+    if (typeof meta.l0_abstract === "string" && meta.l0_abstract.length > 0) continue;
+
+    // Mark as archived
+    meta.tier = "archived";
+    meta.archived_reason = "distilled-junk-cleanup-2026-03-26";
+
+    toArchive.push({
+      id: row.id,
+      text: row.text,
+      vector: Array.from(row.vector as Iterable<number>),
+      category: row.category,
+      scope: scope,
+      importance: 0.05, // near-zero so it never surfaces
+      timestamp: Number(row.timestamp),
+      metadata: JSON.stringify(meta),
+    });
+
+    // Collect samples
+    if (sampleJunk.length < 10) {
+      const text = (row.text as string).slice(0, 80);
+      sampleJunk.push(`  ${(row.id as string).slice(0, 8)} imp=${imp.toFixed(2)} scope=${scope.slice(0, 15)} | ${text}`);
+    }
+  }
+
+  console.log(`  Junk to archive: ${toArchive.length}`);
+  console.log(`  Would keep: ${allFacts.length - toArchive.length} fact entries\n`);
+
+  if (sampleJunk.length > 0) {
+    console.log(`  Sample junk:\n${sampleJunk.join("\n")}\n`);
+  }
+
+  if (dryRun) {
+    console.log(`  (dry run — no changes made)`);
+    return;
+  }
+
+  if (toArchive.length === 0) {
+    console.log(`  Nothing to archive.`);
+    return;
+  }
+
+  // Batch update
+  const batchSize = 1000;
+  for (let i = 0; i < toArchive.length; i += batchSize) {
+    const batch = toArchive.slice(i, i + batchSize);
+    const ids = batch.map((r: any) => `'${r.id.replace(/'/g, "''")}'`).join(",");
+    await table.delete(`id IN (${ids})`);
+    await table.add(batch);
+    console.log(`  Batch ${Math.floor(i / batchSize) + 1}: archived ${batch.length} entries`);
+  }
+
+  console.log(`\nDone. Archived ${toArchive.length} junk fact entries (importance → 0.05, tier → archived).`);
+}
+
+main().catch(err => {
+  console.error("Archive failed:", err);
+  process.exit(1);
+});

package/scripts/backfill-l0l1l2.ts

@@ -0,0 +1,98 @@
+#!/usr/bin/env bun
+/**
+ * One-time backfill: normalize legacy l0/l1 short names to l0_abstract/l1_overview/l2_content
+ * in metadata. Uses batch read + batch write for performance.
+ *
+ * Usage:
+ *   bun scripts/backfill-l0l1l2.ts --dry-run   # preview
+ *   bun scripts/backfill-l0l1l2.ts              # execute
+ */
+
+import { loadDotEnv, loadConfig, expandHome } from "../src/runtime-config.js";
+import { loadLanceDB } from "../src/store.js";
+
+loadDotEnv();
+const config = loadConfig();
+const dryRun = process.argv.includes("--dry-run");
+
+async function main() {
+  const lancedb = await loadLanceDB();
+  const dbPath = expandHome(config.database?.path || "data/lancedb");
+  const db = await lancedb.connect(dbPath);
+  const table = await db.openTable("memories");
+
+  console.log(`=== L0/L1/L2 metadata backfill${dryRun ? " (dry run)" : ""} ===\n`);
+
+  // Read all rows with metadata
+  const allRows = await table.query()
+    .select(["id", "text", "vector", "category", "scope", "importance", "timestamp", "metadata"])
+    .toArray();
+
+  console.log(`  Total entries: ${allRows.length}`);
+
+  const toMigrate: any[] = [];
+
+  for (const row of allRows) {
+    const metaStr = row.metadata as string;
+    if (!metaStr) continue;
+
+    let meta: Record<string, unknown>;
+    try { meta = JSON.parse(metaStr); } catch { continue; }
+
+    const hasShortL0 = typeof meta.l0 === "string" && (meta.l0 as string).length > 0;
+    const hasShortL1 = typeof meta.l1 === "string" && (meta.l1 as string).length > 0;
+    const hasLongL0 = typeof meta.l0_abstract === "string";
+    const hasLongL1 = typeof meta.l1_overview === "string";
+
+    if (!hasShortL0 && !hasShortL1) continue;
+    if (hasLongL0 && hasLongL1) continue;
+
+    // Migrate field names
+    if (hasShortL0 && !hasLongL0) meta.l0_abstract = meta.l0;
+    if (hasShortL1 && !hasLongL1) meta.l1_overview = meta.l1;
+    if (!meta.l2_content && row.text) meta.l2_content = row.text;
+    delete meta.l0;
+    delete meta.l1;
+
+    toMigrate.push({
+      id: row.id,
+      text: row.text,
+      vector: Array.from(row.vector as Iterable<number>),
+      category: row.category,
+      scope: row.scope ?? "",
+      importance: Number(row.importance),
+      timestamp: Number(row.timestamp),
+      metadata: JSON.stringify(meta),
+    });
+  }
+
+  console.log(`  Need migration: ${toMigrate.length}`);
+  console.log(`  Skipped: ${allRows.length - toMigrate.length}`);
+
+  if (dryRun) {
+    console.log(`\n  (dry run — no changes made)`);
+    return;
+  }
+
+  if (toMigrate.length === 0) {
+    console.log(`\n  Nothing to do.`);
+    return;
+  }
+
+  // Batch delete + batch add
+  const batchSize = 1000;
+  for (let i = 0; i < toMigrate.length; i += batchSize) {
+    const batch = toMigrate.slice(i, i + batchSize);
+    const ids = batch.map((r: any) => `'${r.id.replace(/'/g, "''")}'`).join(",");
+    await table.delete(`id IN (${ids})`);
+    await table.add(batch);
+    console.log(`  Batch ${Math.floor(i / batchSize) + 1}: migrated ${batch.length} entries`);
+  }
+
+  console.log(`\nDone. Migrated ${toMigrate.length} entries.`);
+}
+
+main().catch(err => {
+  console.error("Backfill failed:", err);
+  process.exit(1);
+});