recallnest 2.2.0 → 2.2.1

package/scripts/pull-claude-desktop.ts

@@ -1,515 +0,0 @@
-#!/usr/bin/env bun
-/**
- * pull-claude-desktop.ts
- *
- * 从 claude.ai 拉取 Claude Desktop / 网页版对话历史，
- * 转成 CC transcript .jsonl 格式，供 `lm ingest --source desktop` 消费。
- *
- * 用法：
- *   CLAUDE_SESSION_KEY="sk-ant-..." bun scripts/pull-claude-desktop.ts
- *   CLAUDE_SESSION_KEY="sk-ant-..." bun scripts/pull-claude-desktop.ts --limit 10
- *   CLAUDE_SESSION_KEY="sk-ant-..." bun scripts/pull-claude-desktop.ts --since 2025-01-01
- */
-
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { parseArgs } from "node:util";
-
-// ============================================================================
-// Config
-// ============================================================================
-
-const BASE_URL = "https://claude.ai";
-const PAGE_SIZE = 50;
-const THROTTLE_MS = 200;
-const MAX_RETRIES = 3;
-const OUTPUT_DIR = join(import.meta.dir, "..", "data", "desktop-import");
-const SYNC_STATE_PATH = join(OUTPUT_DIR, ".sync-state.json");
-
-// ============================================================================
-// Types
-// ============================================================================
-
-interface SyncState {
-  lastSyncedAt: string;
-  conversations: Record<
-    string,
-    {
-      messageCount: number;
-      updatedAt: string;
-      pulledAt: string;
-      name: string;
-    }
-  >;
-}
-
-interface ConversationSummary {
-  uuid: string;
-  name: string;
-  created_at: string;
-  updated_at: string;
-}
-
-interface ChatMessage {
-  uuid: string;
-  text: string;
-  sender: "human" | "assistant";
-  index: number;
-  created_at: string;
-  parent_message_uuid: string;
-  content: Array<{
-    type: "text" | "thinking" | "tool_use" | "tool_result";
-    text?: string;
-    name?: string;
-    input?: unknown;
-  }>;
-  attachments?: Array<{
-    file_name: string;
-    file_type?: string;
-    file_size?: number;
-  }>;
-}
-
-interface Conversation {
-  uuid: string;
-  name: string;
-  created_at: string;
-  updated_at: string;
-  chat_messages: ChatMessage[];
-}
-
-// ============================================================================
-// HTTP Client
-// ============================================================================
-
-class ClaudeApiClient {
-  private sessionKey: string;
-  private orgId: string | null = null;
-  private throttleMs = THROTTLE_MS;
-
-  constructor(sessionKey: string) {
-    this.sessionKey = sessionKey;
-  }
-
-  private async request(path: string): Promise<any> {
-    let lastError: Error | null = null;
-
-    for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
-      try {
-        const res = await fetch(`${BASE_URL}${path}`, {
-          headers: {
-            Cookie: `sessionKey=${this.sessionKey}`,
-            "Content-Type": "application/json",
-            Accept: "application/json",
-          },
-        });
-
-        if (res.status === 401 || res.status === 403) {
-          throw new Error(
-            `认证失败 (${res.status})。sessionKey 可能已过期，请从浏览器重新获取。`,
-          );
-        }
-
-        if (res.status === 429) {
-          this.throttleMs = Math.min(this.throttleMs * 2, 5000);
-          console.log(`  ⚠️  限流，等待 ${this.throttleMs}ms...`);
-          await sleep(this.throttleMs);
-          continue;
-        }
-
-        if (res.status >= 500) {
-          const waitMs = Math.pow(2, attempt) * 1000;
-          console.log(
-            `  ⚠️  服务器错误 ${res.status}，${waitMs}ms 后重试...`,
-          );
-          await sleep(waitMs);
-          continue;
-        }
-
-        if (!res.ok) {
-          throw new Error(`HTTP ${res.status}: ${await res.text()}`);
-        }
-
-        // Reset throttle on success
-        this.throttleMs = THROTTLE_MS;
-        return await res.json();
-      } catch (err: any) {
-        if (
-          err.message?.includes("认证失败") ||
-          err.message?.startsWith("HTTP ")
-        ) {
-          throw err;
-        }
-        lastError = err;
-        const waitMs = Math.pow(2, attempt) * 1000;
-        console.log(`  ⚠️  网络错误: ${err.message}，${waitMs}ms 后重试...`);
-        await sleep(waitMs);
-      }
-    }
-
-    throw lastError || new Error("请求失败，已重试 3 次");
-  }
-
-  async getOrgId(): Promise<string> {
-    if (this.orgId) return this.orgId;
-    const orgs = await this.request("/api/organizations");
-    if (!Array.isArray(orgs) || orgs.length === 0) {
-      throw new Error("获取 organization 失败。请确认 sessionKey 有效。");
-    }
-    this.orgId = orgs[0].uuid;
-    return this.orgId!;
-  }
-
-  async listConversations(
-    offset: number = 0,
-  ): Promise<ConversationSummary[]> {
-    const orgId = await this.getOrgId();
-    return await this.request(
-      `/api/organizations/${orgId}/chat_conversations?limit=${PAGE_SIZE}&offset=${offset}`,
-    );
-  }
-
-  async getConversation(chatId: string): Promise<Conversation> {
-    const orgId = await this.getOrgId();
-    return await this.request(
-      `/api/organizations/${orgId}/chat_conversations/${chatId}?tree=True&rendering_mode=messages&render_all_tools=true`,
-    );
-  }
-
-  async throttle(): Promise<void> {
-    await sleep(this.throttleMs);
-  }
-}
-
-// ============================================================================
-// Branch Extraction
-// ============================================================================
-
-function extractLatestBranch(messages: ChatMessage[]): ChatMessage[] {
-  if (messages.length === 0) return [];
-
-  const byUuid = new Map<string, ChatMessage>();
-  for (const msg of messages) {
-    byUuid.set(msg.uuid, msg);
-  }
-
-  // Find all parent UUIDs
-  const parentUuids = new Set<string>();
-  for (const msg of messages) {
-    if (msg.parent_message_uuid) {
-      parentUuids.add(msg.parent_message_uuid);
-    }
-  }
-
-  // Leaves = messages that are not anyone's parent
-  const leaves = messages.filter((m) => !parentUuids.has(m.uuid));
-  if (leaves.length === 0) return messages;
-
-  // Pick the leaf with the latest timestamp
-  const latestLeaf = leaves.reduce((a, b) =>
-    new Date(a.created_at).getTime() >= new Date(b.created_at).getTime()
-      ? a
-      : b,
-  );
-
-  // Walk backwards from leaf to root
-  const branch: ChatMessage[] = [];
-  let current: ChatMessage | undefined = latestLeaf;
-  const visited = new Set<string>();
-
-  while (current && !visited.has(current.uuid)) {
-    visited.add(current.uuid);
-    branch.push(current);
-    current = current.parent_message_uuid
-      ? byUuid.get(current.parent_message_uuid)
-      : undefined;
-  }
-
-  branch.reverse();
-  return branch;
-}
-
-// ============================================================================
-// Format Conversion
-// ============================================================================
-
-function messageToText(msg: ChatMessage): string {
-  const parts: string[] = [];
-
-  if (Array.isArray(msg.content) && msg.content.length > 0) {
-    for (const block of msg.content) {
-      if (block.type === "text" && block.text) {
-        parts.push(block.text);
-      }
-      // Skip thinking, tool_use, tool_result — noise for memory
-    }
-  } else if (msg.text) {
-    // Fallback: use raw text field
-    parts.push(msg.text);
-  }
-
-  // Mark attachments
-  if (Array.isArray(msg.attachments)) {
-    for (const att of msg.attachments) {
-      parts.push(`[Attachment: ${att.file_name}]`);
-    }
-  }
-
-  return parts.join("\n").trim();
-}
-
-function convertToCCTranscript(
-  conversation: Conversation,
-): string[] {
-  const branch = extractLatestBranch(conversation.chat_messages || []);
-  const lines: string[] = [];
-
-  for (const msg of branch) {
-    const text = messageToText(msg);
-    if (!text || text.length < 5) continue;
-
-    const type = msg.sender === "human" ? "user" : "assistant";
-    const entry = {
-      type,
-      message: {
-        role: type,
-        content: text,
-      },
-      sessionId: conversation.uuid,
-      timestamp: msg.created_at,
-      uuid: msg.uuid,
-      parentUuid: msg.parent_message_uuid || undefined,
-    };
-
-    lines.push(JSON.stringify(entry));
-  }
-
-  return lines;
-}
-
-// ============================================================================
-// Sync State
-// ============================================================================
-
-function loadSyncState(): SyncState {
-  if (existsSync(SYNC_STATE_PATH)) {
-    try {
-      return JSON.parse(readFileSync(SYNC_STATE_PATH, "utf-8"));
-    } catch {
-      // Corrupted, start fresh
-    }
-  }
-  return { lastSyncedAt: "", conversations: {} };
-}
-
-function saveSyncState(state: SyncState): void {
-  writeFileSync(SYNC_STATE_PATH, JSON.stringify(state, null, 2));
-}
-
-function needsPull(
-  state: SyncState,
-  conv: ConversationSummary,
-): boolean {
-  const existing = state.conversations[conv.uuid];
-  if (!existing) return true;
-  // Re-pull if conversation was updated since last pull
-  return conv.updated_at !== existing.updatedAt;
-}
-
-// ============================================================================
-// Main
-// ============================================================================
-
-function sleep(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-
-async function main() {
-  const { values } = parseArgs({
-    args: process.argv.slice(2),
-    options: {
-      "session-key": { type: "string" },
-      limit: { type: "string" },
-      since: { type: "string" },
-      verbose: { type: "boolean", default: false },
-    },
-    strict: false,
-  });
-
-  const sessionKey =
-    values["session-key"] ||
-    process.env.CLAUDE_SESSION_KEY ||
-    process.env.CLAUDE_DESKTOP_SESSION_KEY;
-
-  if (!sessionKey) {
-    console.error(
-      "❌ 需要 sessionKey。用法：\n" +
-        "   CLAUDE_SESSION_KEY='...' bun scripts/pull-claude-desktop.ts\n\n" +
-        "获取方式：浏览器登录 claude.ai → DevTools → Application → Cookies → sessionKey",
-    );
-    process.exit(1);
-  }
-
-  const limit = values.limit ? parseInt(values.limit, 10) : Infinity;
-  const sinceDate = values.since ? new Date(values.since) : null;
-  const verbose = values.verbose ?? false;
-
-  // Ensure output directory
-  mkdirSync(OUTPUT_DIR, { recursive: true });
-
-  const client = new ClaudeApiClient(sessionKey);
-  const state = loadSyncState();
-
-  // Step 1: Verify auth
-  console.log("🔑 验证 sessionKey...");
-  try {
-    const orgId = await client.getOrgId();
-    console.log(`  ✅ 认证成功 (org: ${orgId.slice(0, 8)}...)`);
-  } catch (err: any) {
-    console.error(`  ❌ ${err.message}`);
-    process.exit(1);
-  }
-
-  // Step 2: List all conversations (paginated)
-  console.log("\n📋 获取对话列表...");
-  const allConversations: ConversationSummary[] = [];
-  let offset = 0;
-  let pageCount = 0;
-
-  while (true) {
-    const page = await client.listConversations(offset);
-    if (!Array.isArray(page) || page.length === 0) break;
-
-    // Filter by --since if specified
-    for (const conv of page) {
-      if (sinceDate && new Date(conv.updated_at) < sinceDate) {
-        // API returns newest first; once we hit older, stop
-        console.log(
-          `  ⏭️  跳过 ${conv.updated_at} 之前的对话 (--since ${values.since})`,
-        );
-        break;
-      }
-      allConversations.push(conv);
-    }
-
-    pageCount++;
-    process.stdout.write(
-      `  已扫描 ${allConversations.length} 条对话 (第 ${pageCount} 页)...\r`,
-    );
-
-    // Stop if we hit the since boundary or got a short page
-    if (sinceDate) {
-      const lastOnPage = page[page.length - 1];
-      if (new Date(lastOnPage.updated_at) < sinceDate) break;
-    }
-    if (page.length < PAGE_SIZE) break;
-
-    offset += PAGE_SIZE;
-    await client.throttle();
-  }
-
-  console.log(`  ✅ 共发现 ${allConversations.length} 条对话`);
-
-  // Step 3: Filter — skip already-synced
-  const toPull = allConversations.filter((c) => needsPull(state, c));
-  const toSkip = allConversations.length - toPull.length;
-
-  if (toSkip > 0) {
-    console.log(`  ⏭️  ${toSkip} 条已同步，跳过`);
-  }
-
-  // Apply --limit
-  const batch = toPull.slice(0, limit);
-  if (batch.length === 0) {
-    console.log("\n✅ 没有需要拉取的新对话。");
-    return;
-  }
-
-  console.log(`\n🔄 开始拉取 ${batch.length} 条对话...\n`);
-
-  // Step 4: Pull each conversation
-  let pulled = 0;
-  let skipped = 0;
-  let failed = 0;
-  const errors: string[] = [];
-
-  for (let i = 0; i < batch.length; i++) {
-    const conv = batch[i];
-    const label = conv.name?.slice(0, 40) || conv.uuid.slice(0, 8);
-
-    try {
-      process.stdout.write(
-        `  [${i + 1}/${batch.length}] ${label}...`,
-      );
-
-      const full = await client.getConversation(conv.uuid);
-      const lines = convertToCCTranscript(full);
-
-      if (lines.length === 0) {
-        console.log(` ⏭️  空对话`);
-        skipped++;
-      } else {
-        const outPath = join(OUTPUT_DIR, `${conv.uuid}.jsonl`);
-        writeFileSync(outPath, lines.join("\n") + "\n");
-        console.log(` ✅ ${lines.length} 条消息`);
-        pulled++;
-      }
-
-      // Update sync state
-      state.conversations[conv.uuid] = {
-        messageCount: lines.length,
-        updatedAt: conv.updated_at,
-        pulledAt: new Date().toISOString(),
-        name: conv.name || "",
-      };
-
-      // Save state periodically (every 10 conversations)
-      if ((i + 1) % 10 === 0) {
-        state.lastSyncedAt = new Date().toISOString();
-        saveSyncState(state);
-      }
-
-      await client.throttle();
-    } catch (err: any) {
-      console.log(` ❌ ${err.message}`);
-      errors.push(`${label}: ${err.message}`);
-      failed++;
-
-      // Don't abort on single conversation failure
-      await client.throttle();
-    }
-  }
-
-  // Final save
-  state.lastSyncedAt = new Date().toISOString();
-  saveSyncState(state);
-
-  // Step 5: Summary
-  console.log("\n📊 拉取汇总:");
-  console.log(`  总对话数: ${allConversations.length}`);
-  console.log(`  已同步跳过: ${toSkip}`);
-  console.log(`  本次拉取: ${pulled}`);
-  console.log(`  空对话跳过: ${skipped}`);
-  console.log(`  失败: ${failed}`);
-  console.log(`  输出目录: ${OUTPUT_DIR}`);
-
-  if (errors.length > 0) {
-    console.log("\n⚠️  失败详情:");
-    for (const e of errors.slice(0, 10)) {
-      console.log(`  - ${e}`);
-    }
-  }
-
-  if (pulled > 0) {
-    console.log(
-      "\n🎯 下一步：运行以下命令将对话导入 RecallNest：",
-    );
-    console.log("   bun run src/cli.ts ingest --source desktop");
-  }
-}
-
-main().catch((err) => {
-  console.error(`\n❌ 致命错误: ${err.message}`);
-  if (err.stack) console.error(err.stack);
-  process.exit(1);
-});

package/scripts/re-ingest-cc.sh

@@ -1,66 +0,0 @@
-#!/bin/bash
-# re-ingest-cc.sh — 全量重建 RecallNest 索引
-#
-# 背景：旧 dedup 阈值 (hard=0.80, soft=0.68) 导致 72% 内容被误杀，
-#       且旧数据中可能有噪音。最干净的做法是清空后用新阈值从头导入。
-#
-# 用法：
-#   cd ~/recallnest && bash scripts/re-ingest-cc.sh
-#
-# 步骤：
-#   1. 备份 LanceDB + ingested-files.json + distill-progress.json
-#   2. reset（清空 LanceDB）
-#   3. 清空 ingest 和 distill 进度记录
-#   4. 全量 ingest（所有源）
-#   5. 完成后提示手动跑 distill-facts
-#
-# 预计耗时：3-5 小时（后台跑）
-# 预计 API 成本：~$3-5
-
-set -euo pipefail
-cd "$(dirname "$0")/.."
-
-TIMESTAMP=$(date +%Y%m%d-%H%M%S)
-BACKUP_DIR="data/backup-$TIMESTAMP"
-LOG_FILE="logs/re-ingest-$TIMESTAMP.log"
-
-echo "=== RecallNest 全量重建 ===" | tee "$LOG_FILE"
-echo "开始时间: $(date)" | tee -a "$LOG_FILE"
-
-# Step 1: 备份
-echo "[1/4] 备份现有数据..." | tee -a "$LOG_FILE"
-mkdir -p "$BACKUP_DIR"
-cp data/ingested-files.json "$BACKUP_DIR/" 2>/dev/null || true
-cp data/distill-progress.json "$BACKUP_DIR/" 2>/dev/null || true
-# LanceDB 太大不整体备份，只记录 stats
-echo "  LanceDB 当前大小: $(du -sh data/lancedb 2>/dev/null | cut -f1)" | tee -a "$LOG_FILE"
-echo "  备份到: $BACKUP_DIR/" | tee -a "$LOG_FILE"
-
-# Step 2: 清空 LanceDB
-echo "[2/4] 清空 LanceDB 索引..." | tee -a "$LOG_FILE"
-bun run src/cli.ts reset --yes 2>&1 | tee -a "$LOG_FILE"
-
-# Step 3: 清空进度记录
-echo "[3/4] 清空 ingest/distill 进度记录..." | tee -a "$LOG_FILE"
-# Reset ingested-files.json — 让 ingest 认为所有文件都没处理过
-echo '{"files":{}}' > data/ingested-files.json
-# Reset distill progress
-echo '{"startedAt":"","lastUpdated":"","completedScopes":[],"stats":{}}' > data/distill-progress.json
-echo "  进度已重置" | tee -a "$LOG_FILE"
-
-# Step 4: 全量 ingest
-echo "[4/4] 开始全量导入（这一步需要 3-5 小时）..." | tee -a "$LOG_FILE"
-echo "  日志实时写入: $LOG_FILE" | tee -a "$LOG_FILE"
-echo "  查看进度: tail -f $LOG_FILE" | tee -a "$LOG_FILE"
-echo "" | tee -a "$LOG_FILE"
-
-bun run src/cli.ts ingest 2>&1 | tee -a "$LOG_FILE"
-
-echo "" | tee -a "$LOG_FILE"
-echo "=== 全量导入完成 ===" | tee -a "$LOG_FILE"
-echo "结束时间: $(date)" | tee -a "$LOG_FILE"
-echo "" | tee -a "$LOG_FILE"
-echo "下一步：" | tee -a "$LOG_FILE"
-echo "  1. 检查日志确认入库率（目标 50-65%，旧阈值是 27%）" | tee -a "$LOG_FILE"
-echo "  2. 跑 distill: bun scripts/distill-facts.ts" | tee -a "$LOG_FILE"
-echo "  3. 旧数据备份在: $BACKUP_DIR/" | tee -a "$LOG_FILE"