recallnest 1.4.1

package/scripts/pull-claude-desktop.ts

@@ -0,0 +1,515 @@
+#!/usr/bin/env bun
+/**
+ * pull-claude-desktop.ts
+ *
+ * 从 claude.ai 拉取 Claude Desktop / 网页版对话历史，
+ * 转成 CC transcript .jsonl 格式，供 `lm ingest --source desktop` 消费。
+ *
+ * 用法：
+ *   CLAUDE_SESSION_KEY="sk-ant-..." bun scripts/pull-claude-desktop.ts
+ *   CLAUDE_SESSION_KEY="sk-ant-..." bun scripts/pull-claude-desktop.ts --limit 10
+ *   CLAUDE_SESSION_KEY="sk-ant-..." bun scripts/pull-claude-desktop.ts --since 2025-01-01
+ */
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { parseArgs } from "node:util";
+
+// ============================================================================
+// Config
+// ============================================================================
+
+const BASE_URL = "https://claude.ai";
+const PAGE_SIZE = 50;
+const THROTTLE_MS = 200;
+const MAX_RETRIES = 3;
+const OUTPUT_DIR = join(import.meta.dir, "..", "data", "desktop-import");
+const SYNC_STATE_PATH = join(OUTPUT_DIR, ".sync-state.json");
+
+// ============================================================================
+// Types
+// ============================================================================
+
+interface SyncState {
+  lastSyncedAt: string;
+  conversations: Record<
+    string,
+    {
+      messageCount: number;
+      updatedAt: string;
+      pulledAt: string;
+      name: string;
+    }
+  >;
+}
+
+interface ConversationSummary {
+  uuid: string;
+  name: string;
+  created_at: string;
+  updated_at: string;
+}
+
+interface ChatMessage {
+  uuid: string;
+  text: string;
+  sender: "human" | "assistant";
+  index: number;
+  created_at: string;
+  parent_message_uuid: string;
+  content: Array<{
+    type: "text" | "thinking" | "tool_use" | "tool_result";
+    text?: string;
+    name?: string;
+    input?: unknown;
+  }>;
+  attachments?: Array<{
+    file_name: string;
+    file_type?: string;
+    file_size?: number;
+  }>;
+}
+
+interface Conversation {
+  uuid: string;
+  name: string;
+  created_at: string;
+  updated_at: string;
+  chat_messages: ChatMessage[];
+}
+
+// ============================================================================
+// HTTP Client
+// ============================================================================
+
+class ClaudeApiClient {
+  private sessionKey: string;
+  private orgId: string | null = null;
+  private throttleMs = THROTTLE_MS;
+
+  constructor(sessionKey: string) {
+    this.sessionKey = sessionKey;
+  }
+
+  private async request(path: string): Promise<any> {
+    let lastError: Error | null = null;
+
+    for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
+      try {
+        const res = await fetch(`${BASE_URL}${path}`, {
+          headers: {
+            Cookie: `sessionKey=${this.sessionKey}`,
+            "Content-Type": "application/json",
+            Accept: "application/json",
+          },
+        });
+
+        if (res.status === 401 || res.status === 403) {
+          throw new Error(
+            `认证失败 (${res.status})。sessionKey 可能已过期，请从浏览器重新获取。`,
+          );
+        }
+
+        if (res.status === 429) {
+          this.throttleMs = Math.min(this.throttleMs * 2, 5000);
+          console.log(`  ⚠️  限流，等待 ${this.throttleMs}ms...`);
+          await sleep(this.throttleMs);
+          continue;
+        }
+
+        if (res.status >= 500) {
+          const waitMs = Math.pow(2, attempt) * 1000;
+          console.log(
+            `  ⚠️  服务器错误 ${res.status}，${waitMs}ms 后重试...`,
+          );
+          await sleep(waitMs);
+          continue;
+        }
+
+        if (!res.ok) {
+          throw new Error(`HTTP ${res.status}: ${await res.text()}`);
+        }
+
+        // Reset throttle on success
+        this.throttleMs = THROTTLE_MS;
+        return await res.json();
+      } catch (err: any) {
+        if (
+          err.message?.includes("认证失败") ||
+          err.message?.startsWith("HTTP ")
+        ) {
+          throw err;
+        }
+        lastError = err;
+        const waitMs = Math.pow(2, attempt) * 1000;
+        console.log(`  ⚠️  网络错误: ${err.message}，${waitMs}ms 后重试...`);
+        await sleep(waitMs);
+      }
+    }
+
+    throw lastError || new Error("请求失败，已重试 3 次");
+  }
+
+  async getOrgId(): Promise<string> {
+    if (this.orgId) return this.orgId;
+    const orgs = await this.request("/api/organizations");
+    if (!Array.isArray(orgs) || orgs.length === 0) {
+      throw new Error("获取 organization 失败。请确认 sessionKey 有效。");
+    }
+    this.orgId = orgs[0].uuid;
+    return this.orgId!;
+  }
+
+  async listConversations(
+    offset: number = 0,
+  ): Promise<ConversationSummary[]> {
+    const orgId = await this.getOrgId();
+    return await this.request(
+      `/api/organizations/${orgId}/chat_conversations?limit=${PAGE_SIZE}&offset=${offset}`,
+    );
+  }
+
+  async getConversation(chatId: string): Promise<Conversation> {
+    const orgId = await this.getOrgId();
+    return await this.request(
+      `/api/organizations/${orgId}/chat_conversations/${chatId}?tree=True&rendering_mode=messages&render_all_tools=true`,
+    );
+  }
+
+  async throttle(): Promise<void> {
+    await sleep(this.throttleMs);
+  }
+}
+
+// ============================================================================
+// Branch Extraction
+// ============================================================================
+
+function extractLatestBranch(messages: ChatMessage[]): ChatMessage[] {
+  if (messages.length === 0) return [];
+
+  const byUuid = new Map<string, ChatMessage>();
+  for (const msg of messages) {
+    byUuid.set(msg.uuid, msg);
+  }
+
+  // Find all parent UUIDs
+  const parentUuids = new Set<string>();
+  for (const msg of messages) {
+    if (msg.parent_message_uuid) {
+      parentUuids.add(msg.parent_message_uuid);
+    }
+  }
+
+  // Leaves = messages that are not anyone's parent
+  const leaves = messages.filter((m) => !parentUuids.has(m.uuid));
+  if (leaves.length === 0) return messages;
+
+  // Pick the leaf with the latest timestamp
+  const latestLeaf = leaves.reduce((a, b) =>
+    new Date(a.created_at).getTime() >= new Date(b.created_at).getTime()
+      ? a
+      : b,
+  );
+
+  // Walk backwards from leaf to root
+  const branch: ChatMessage[] = [];
+  let current: ChatMessage | undefined = latestLeaf;
+  const visited = new Set<string>();
+
+  while (current && !visited.has(current.uuid)) {
+    visited.add(current.uuid);
+    branch.push(current);
+    current = current.parent_message_uuid
+      ? byUuid.get(current.parent_message_uuid)
+      : undefined;
+  }
+
+  branch.reverse();
+  return branch;
+}
+
+// ============================================================================
+// Format Conversion
+// ============================================================================
+
+function messageToText(msg: ChatMessage): string {
+  const parts: string[] = [];
+
+  if (Array.isArray(msg.content) && msg.content.length > 0) {
+    for (const block of msg.content) {
+      if (block.type === "text" && block.text) {
+        parts.push(block.text);
+      }
+      // Skip thinking, tool_use, tool_result — noise for memory
+    }
+  } else if (msg.text) {
+    // Fallback: use raw text field
+    parts.push(msg.text);
+  }
+
+  // Mark attachments
+  if (Array.isArray(msg.attachments)) {
+    for (const att of msg.attachments) {
+      parts.push(`[Attachment: ${att.file_name}]`);
+    }
+  }
+
+  return parts.join("\n").trim();
+}
+
+function convertToCCTranscript(
+  conversation: Conversation,
+): string[] {
+  const branch = extractLatestBranch(conversation.chat_messages || []);
+  const lines: string[] = [];
+
+  for (const msg of branch) {
+    const text = messageToText(msg);
+    if (!text || text.length < 5) continue;
+
+    const type = msg.sender === "human" ? "user" : "assistant";
+    const entry = {
+      type,
+      message: {
+        role: type,
+        content: text,
+      },
+      sessionId: conversation.uuid,
+      timestamp: msg.created_at,
+      uuid: msg.uuid,
+      parentUuid: msg.parent_message_uuid || undefined,
+    };
+
+    lines.push(JSON.stringify(entry));
+  }
+
+  return lines;
+}
+
+// ============================================================================
+// Sync State
+// ============================================================================
+
+function loadSyncState(): SyncState {
+  if (existsSync(SYNC_STATE_PATH)) {
+    try {
+      return JSON.parse(readFileSync(SYNC_STATE_PATH, "utf-8"));
+    } catch {
+      // Corrupted, start fresh
+    }
+  }
+  return { lastSyncedAt: "", conversations: {} };
+}
+
+function saveSyncState(state: SyncState): void {
+  writeFileSync(SYNC_STATE_PATH, JSON.stringify(state, null, 2));
+}
+
+function needsPull(
+  state: SyncState,
+  conv: ConversationSummary,
+): boolean {
+  const existing = state.conversations[conv.uuid];
+  if (!existing) return true;
+  // Re-pull if conversation was updated since last pull
+  return conv.updated_at !== existing.updatedAt;
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function main() {
+  const { values } = parseArgs({
+    args: process.argv.slice(2),
+    options: {
+      "session-key": { type: "string" },
+      limit: { type: "string" },
+      since: { type: "string" },
+      verbose: { type: "boolean", default: false },
+    },
+    strict: false,
+  });
+
+  const sessionKey =
+    values["session-key"] ||
+    process.env.CLAUDE_SESSION_KEY ||
+    process.env.CLAUDE_DESKTOP_SESSION_KEY;
+
+  if (!sessionKey) {
+    console.error(
+      "❌ 需要 sessionKey。用法：\n" +
+        "   CLAUDE_SESSION_KEY='...' bun scripts/pull-claude-desktop.ts\n\n" +
+        "获取方式：浏览器登录 claude.ai → DevTools → Application → Cookies → sessionKey",
+    );
+    process.exit(1);
+  }
+
+  const limit = values.limit ? parseInt(values.limit, 10) : Infinity;
+  const sinceDate = values.since ? new Date(values.since) : null;
+  const verbose = values.verbose ?? false;
+
+  // Ensure output directory
+  mkdirSync(OUTPUT_DIR, { recursive: true });
+
+  const client = new ClaudeApiClient(sessionKey);
+  const state = loadSyncState();
+
+  // Step 1: Verify auth
+  console.log("🔑 验证 sessionKey...");
+  try {
+    const orgId = await client.getOrgId();
+    console.log(`  ✅ 认证成功 (org: ${orgId.slice(0, 8)}...)`);
+  } catch (err: any) {
+    console.error(`  ❌ ${err.message}`);
+    process.exit(1);
+  }
+
+  // Step 2: List all conversations (paginated)
+  console.log("\n📋 获取对话列表...");
+  const allConversations: ConversationSummary[] = [];
+  let offset = 0;
+  let pageCount = 0;
+
+  while (true) {
+    const page = await client.listConversations(offset);
+    if (!Array.isArray(page) || page.length === 0) break;
+
+    // Filter by --since if specified
+    for (const conv of page) {
+      if (sinceDate && new Date(conv.updated_at) < sinceDate) {
+        // API returns newest first; once we hit older, stop
+        console.log(
+          `  ⏭️  跳过 ${conv.updated_at} 之前的对话 (--since ${values.since})`,
+        );
+        break;
+      }
+      allConversations.push(conv);
+    }
+
+    pageCount++;
+    process.stdout.write(
+      `  已扫描 ${allConversations.length} 条对话 (第 ${pageCount} 页)...\r`,
+    );
+
+    // Stop if we hit the since boundary or got a short page
+    if (sinceDate) {
+      const lastOnPage = page[page.length - 1];
+      if (new Date(lastOnPage.updated_at) < sinceDate) break;
+    }
+    if (page.length < PAGE_SIZE) break;
+
+    offset += PAGE_SIZE;
+    await client.throttle();
+  }
+
+  console.log(`  ✅ 共发现 ${allConversations.length} 条对话`);
+
+  // Step 3: Filter — skip already-synced
+  const toPull = allConversations.filter((c) => needsPull(state, c));
+  const toSkip = allConversations.length - toPull.length;
+
+  if (toSkip > 0) {
+    console.log(`  ⏭️  ${toSkip} 条已同步，跳过`);
+  }
+
+  // Apply --limit
+  const batch = toPull.slice(0, limit);
+  if (batch.length === 0) {
+    console.log("\n✅ 没有需要拉取的新对话。");
+    return;
+  }
+
+  console.log(`\n🔄 开始拉取 ${batch.length} 条对话...\n`);
+
+  // Step 4: Pull each conversation
+  let pulled = 0;
+  let skipped = 0;
+  let failed = 0;
+  const errors: string[] = [];
+
+  for (let i = 0; i < batch.length; i++) {
+    const conv = batch[i];
+    const label = conv.name?.slice(0, 40) || conv.uuid.slice(0, 8);
+
+    try {
+      process.stdout.write(
+        `  [${i + 1}/${batch.length}] ${label}...`,
+      );
+
+      const full = await client.getConversation(conv.uuid);
+      const lines = convertToCCTranscript(full);
+
+      if (lines.length === 0) {
+        console.log(` ⏭️  空对话`);
+        skipped++;
+      } else {
+        const outPath = join(OUTPUT_DIR, `${conv.uuid}.jsonl`);
+        writeFileSync(outPath, lines.join("\n") + "\n");
+        console.log(` ✅ ${lines.length} 条消息`);
+        pulled++;
+      }
+
+      // Update sync state
+      state.conversations[conv.uuid] = {
+        messageCount: lines.length,
+        updatedAt: conv.updated_at,
+        pulledAt: new Date().toISOString(),
+        name: conv.name || "",
+      };
+
+      // Save state periodically (every 10 conversations)
+      if ((i + 1) % 10 === 0) {
+        state.lastSyncedAt = new Date().toISOString();
+        saveSyncState(state);
+      }
+
+      await client.throttle();
+    } catch (err: any) {
+      console.log(` ❌ ${err.message}`);
+      errors.push(`${label}: ${err.message}`);
+      failed++;
+
+      // Don't abort on single conversation failure
+      await client.throttle();
+    }
+  }
+
+  // Final save
+  state.lastSyncedAt = new Date().toISOString();
+  saveSyncState(state);
+
+  // Step 5: Summary
+  console.log("\n📊 拉取汇总:");
+  console.log(`  总对话数: ${allConversations.length}`);
+  console.log(`  已同步跳过: ${toSkip}`);
+  console.log(`  本次拉取: ${pulled}`);
+  console.log(`  空对话跳过: ${skipped}`);
+  console.log(`  失败: ${failed}`);
+  console.log(`  输出目录: ${OUTPUT_DIR}`);
+
+  if (errors.length > 0) {
+    console.log("\n⚠️  失败详情:");
+    for (const e of errors.slice(0, 10)) {
+      console.log(`  - ${e}`);
+    }
+  }
+
+  if (pulled > 0) {
+    console.log(
+      "\n🎯 下一步：运行以下命令将对话导入 RecallNest：",
+    );
+    console.log("   bun run src/cli.ts ingest --source desktop");
+  }
+}
+
+main().catch((err) => {
+  console.error(`\n❌ 致命错误: ${err.message}`);
+  if (err.stack) console.error(err.stack);
+  process.exit(1);
+});

package/scripts/re-ingest-cc.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# re-ingest-cc.sh — 全量重建 RecallNest 索引
+#
+# 背景：旧 dedup 阈值 (hard=0.80, soft=0.68) 导致 72% 内容被误杀，
+#       且旧数据中可能有噪音。最干净的做法是清空后用新阈值从头导入。
+#
+# 用法：
+#   cd ~/recallnest && bash scripts/re-ingest-cc.sh
+#
+# 步骤：
+#   1. 备份 LanceDB + ingested-files.json + distill-progress.json
+#   2. reset（清空 LanceDB）
+#   3. 清空 ingest 和 distill 进度记录
+#   4. 全量 ingest（所有源）
+#   5. 完成后提示手动跑 distill-facts
+#
+# 预计耗时：3-5 小时（后台跑）
+# 预计 API 成本：~$3-5
+
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+BACKUP_DIR="data/backup-$TIMESTAMP"
+LOG_FILE="logs/re-ingest-$TIMESTAMP.log"
+
+echo "=== RecallNest 全量重建 ===" | tee "$LOG_FILE"
+echo "开始时间: $(date)" | tee -a "$LOG_FILE"
+
+# Step 1: 备份
+echo "[1/4] 备份现有数据..." | tee -a "$LOG_FILE"
+mkdir -p "$BACKUP_DIR"
+cp data/ingested-files.json "$BACKUP_DIR/" 2>/dev/null || true
+cp data/distill-progress.json "$BACKUP_DIR/" 2>/dev/null || true
+# LanceDB 太大不整体备份，只记录 stats
+echo "  LanceDB 当前大小: $(du -sh data/lancedb 2>/dev/null | cut -f1)" | tee -a "$LOG_FILE"
+echo "  备份到: $BACKUP_DIR/" | tee -a "$LOG_FILE"
+
+# Step 2: 清空 LanceDB
+echo "[2/4] 清空 LanceDB 索引..." | tee -a "$LOG_FILE"
+bun run src/cli.ts reset --yes 2>&1 | tee -a "$LOG_FILE"
+
+# Step 3: 清空进度记录
+echo "[3/4] 清空 ingest/distill 进度记录..." | tee -a "$LOG_FILE"
+# Reset ingested-files.json — 让 ingest 认为所有文件都没处理过
+echo '{"files":{}}' > data/ingested-files.json
+# Reset distill progress
+echo '{"startedAt":"","lastUpdated":"","completedScopes":[],"stats":{}}' > data/distill-progress.json
+echo "  进度已重置" | tee -a "$LOG_FILE"
+
+# Step 4: 全量 ingest
+echo "[4/4] 开始全量导入（这一步需要 3-5 小时）..." | tee -a "$LOG_FILE"
+echo "  日志实时写入: $LOG_FILE" | tee -a "$LOG_FILE"
+echo "  查看进度: tail -f $LOG_FILE" | tee -a "$LOG_FILE"
+echo "" | tee -a "$LOG_FILE"
+
+bun run src/cli.ts ingest 2>&1 | tee -a "$LOG_FILE"
+
+echo "" | tee -a "$LOG_FILE"
+echo "=== 全量导入完成 ===" | tee -a "$LOG_FILE"
+echo "结束时间: $(date)" | tee -a "$LOG_FILE"
+echo "" | tee -a "$LOG_FILE"
+echo "下一步：" | tee -a "$LOG_FILE"
+echo "  1. 检查日志确认入库率（目标 50-65%，旧阈值是 27%）" | tee -a "$LOG_FILE"
+echo "  2. 跑 distill: bun scripts/distill-facts.ts" | tee -a "$LOG_FILE"
+echo "  3. 旧数据备份在: $BACKUP_DIR/" | tee -a "$LOG_FILE"