recallnest 2.2.0 → 2.3.0

package/scripts/diagnose-vector-search.ts

@@ -1,132 +0,0 @@
-/**
- * 诊断脚本：验证跨进程 vectorSearch 的 _distance 值
- *
- * 用法: bun run scripts/diagnose-vector-search.ts
- *
- * 做三件事:
- * 1. list() 确认数据存在
- * 2. 取第一条记录的 vector，用它自己做 vectorSearch（理论上 distance=0, score=1.0）
- * 3. 打印 raw _distance 值，看是不是 NaN 或异常大
- */
-
-import lancedb from "@lancedb/lancedb";
-import { resolve } from "node:path";
-
-const DB_PATH = resolve(import.meta.dir, "../data/lancedb");
-const TABLE_NAME = "memories";
-
-async function main() {
-  console.log(`[1] Opening LanceDB at: ${DB_PATH}`);
-  const db = await lancedb.connect(DB_PATH);
-
-  let table: lancedb.Table;
-  try {
-    table = await db.openTable(TABLE_NAME);
-  } catch (err) {
-    console.error(`Failed to open table "${TABLE_NAME}":`, err);
-    process.exit(1);
-  }
-
-  // Step 1: list a few entries to confirm data exists
-  console.log(`\n[2] Listing first 3 entries (SQL query, no vector)...`);
-  const sample = await table.query().limit(3).toArray();
-  console.log(`  Found ${sample.length} entries`);
-
-  if (sample.length === 0) {
-    console.log("  ❌ No data in table. Nothing to diagnose.");
-    process.exit(0);
-  }
-
-  for (const row of sample) {
-    const vec = row.vector;
-    const vecType = Object.prototype.toString.call(vec);
-    const isArray = Array.isArray(vec);
-    const len = vec?.length ?? "N/A";
-    const first3 = vec ? Array.from(vec).slice(0, 3) : "N/A";
-    console.log(`  id=${(row.id as string).slice(0, 8)}  vector.type=${vecType}  isArray=${isArray}  dim=${len}  first3=${JSON.stringify(first3)}`);
-  }
-
-  // Step 2: use the first entry's vector as the query (self-search, should be distance≈0)
-  const firstVec = sample[0].vector;
-  if (!firstVec || !firstVec.length) {
-    console.log("  ❌ First entry has no vector.");
-    process.exit(1);
-  }
-
-  // Convert to plain number[] in case it's a Float32Array or Arrow vector
-  const queryVec: number[] = Array.from(firstVec);
-  console.log(`\n[3] Self-search: using first entry's vector (dim=${queryVec.length}) as query...`);
-  console.log(`  Query vector first 5: ${queryVec.slice(0, 5)}`);
-
-  // Test 3a: vectorSearch WITH distanceType('cosine')
-  console.log(`\n[3a] vectorSearch with distanceType('cosine')...`);
-  try {
-    const results = await table.vectorSearch(queryVec).distanceType('cosine').limit(5).toArray();
-    console.log(`  Returned ${results.length} results`);
-    for (const r of results) {
-      const dist = r._distance;
-      const distNum = Number(dist);
-      const score = 1 / (1 + distNum);
-      const vecLen = r.vector?.length ?? "N/A";
-      const vecType = Object.prototype.toString.call(r.vector);
-      console.log(`    id=${(r.id as string).slice(0, 8)}  _distance=${dist} (Number: ${distNum})  score=${score.toFixed(4)}  vec.type=${vecType}  vec.dim=${vecLen}`);
-    }
-  } catch (err) {
-    console.error(`  ❌ vectorSearch(cosine) failed:`, err);
-  }
-
-  // Test 3b: vectorSearch WITHOUT distanceType (default = L2)
-  console.log(`\n[3b] vectorSearch without distanceType (default L2)...`);
-  try {
-    const results = await table.vectorSearch(queryVec).limit(5).toArray();
-    console.log(`  Returned ${results.length} results`);
-    for (const r of results) {
-      const dist = r._distance;
-      const distNum = Number(dist);
-      console.log(`    id=${(r.id as string).slice(0, 8)}  _distance=${dist} (Number: ${distNum})`);
-    }
-  } catch (err) {
-    console.error(`  ❌ vectorSearch(L2) failed:`, err);
-  }
-
-  // Test 3c: check if the vector round-trips correctly
-  console.log(`\n[4] Vector round-trip check...`);
-  const refetched = await table.query().where(`id = '${sample[0].id}'`).limit(1).toArray();
-  if (refetched.length > 0) {
-    const storedVec = Array.from(refetched[0].vector);
-    const match = queryVec.every((v, i) => Math.abs(v - storedVec[i]) < 1e-6);
-    console.log(`  Vectors match: ${match}`);
-    if (!match) {
-      console.log(`  First 5 query:  ${queryVec.slice(0, 5)}`);
-      console.log(`  First 5 stored: ${storedVec.slice(0, 5)}`);
-      const diffs = queryVec.slice(0, 5).map((v, i) => Math.abs(v - storedVec[i]));
-      console.log(`  Diffs: ${diffs}`);
-    }
-  }
-
-  // Test 3d: manual cosine similarity computation
-  console.log(`\n[5] Manual cosine similarity (sanity check)...`);
-  if (sample.length >= 2) {
-    const vec1: number[] = Array.from(sample[0].vector);
-    const vec2: number[] = Array.from(sample[1].vector);
-
-    let dot = 0, norm1 = 0, norm2 = 0;
-    for (let i = 0; i < vec1.length; i++) {
-      dot += vec1[i] * vec2[i];
-      norm1 += vec1[i] * vec1[i];
-      norm2 += vec2[i] * vec2[i];
-    }
-    const cosSim = dot / (Math.sqrt(norm1) * Math.sqrt(norm2));
-    const cosDist = 1 - cosSim;
-    const score = 1 / (1 + cosDist);
-    console.log(`  Manual cosine(entry0, entry1): sim=${cosSim.toFixed(6)}  dist=${cosDist.toFixed(6)}  score=${score.toFixed(4)}`);
-    console.log(`  vec1 norm=${Math.sqrt(norm1).toFixed(4)}  vec2 norm=${Math.sqrt(norm2).toFixed(4)}`);
-  }
-
-  console.log(`\n✅ Diagnosis complete.`);
-}
-
-main().catch(err => {
-  console.error("Fatal:", err);
-  process.exit(1);
-});

package/scripts/distill-facts.ts

@@ -1,305 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Bulk distillation: Convert raw "fact" conversation fragments into
- * structured knowledge using smartExtractBatch().
- *
- * Usage:
- *   bun scripts/distill-facts.ts --dry-run          # Process 1 session, print results
- *   bun scripts/distill-facts.ts                     # Batch all sessions with checkpoint
- *   bun scripts/distill-facts.ts --scope cc:09541b9c # Process specific session only
- */
-
-import path from "path";
-import fs from "fs";
-import { loadConfig, createComponents } from "../src/runtime-config.js";
-import { buildDistillationBoundary } from "../src/memory-boundaries.js";
-import type { MemoryEntry } from "../src/store.js";
-import type { SmartExtraction, LLMClient } from "../src/llm-client.js";
-import type { MemoryStore } from "../src/store.js";
-import type { Embedder } from "../src/embedder.js";
-
-// ─── Config ─────────────────────────────────────────────────────────────────
-
-const BATCH_SIZE = 20;
-const PROGRESS_FILE = path.join(import.meta.dir, "..", "data", "distill-progress.json");
-
-interface DistillProgress {
-  startedAt: string;
-  lastUpdated: string;
-  completedScopes: string[];
-  stats: {
-    totalFacts: number;
-    processedFacts: number;
-    newRecords: number;
-    dedupedSkips: number;
-    errors: number;
-  };
-}
-
-function loadProgress(): DistillProgress | null {
-  try {
-    return JSON.parse(fs.readFileSync(PROGRESS_FILE, "utf-8"));
-  } catch {
-    return null;
-  }
-}
-
-function saveProgress(progress: DistillProgress): void {
-  fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
-}
-
-// ─── Core Logic ─────────────────────────────────────────────────────────────
-
-async function distillBatch(
-  texts: string[],
-  originals: MemoryEntry[],
-  store: MemoryStore,
-  embedder: Embedder,
-  llm: LLMClient,
-  dryRun = false,
-): Promise<{ newCount: number; dedupCount: number; errors: number }> {
-  let newCount = 0;
-  let dedupCount = 0;
-  let errors = 0;
-
-  // Step 1: Extract knowledge via LLM
-  // smartExtractBatch returns (SmartExtraction | null)[] with same length as input
-  let rawExtractions: (SmartExtraction | null)[];
-  try {
-    rawExtractions = await llm.smartExtractBatch(texts);
-  } catch (e) {
-    console.error(`    LLM extraction failed: ${e}`);
-    return { newCount: 0, dedupCount: 0, errors: texts.length };
-  }
-
-  const validIndices: number[] = [];
-  for (let i = 0; i < rawExtractions.length; i++) {
-    const ext = rawExtractions[i];
-    if (ext && (ext.l0.length > 10 || ext.l1.length > 10)) {
-      validIndices.push(i);
-    }
-  }
-
-  if (validIndices.length === 0) {
-    return { newCount: 0, dedupCount: texts.length, errors: 0 };
-  }
-
-  // Step 3: Generate embeddings
-  const textsToEmbed = validIndices.map(i => {
-    const ext = rawExtractions[i]!;
-    return ext.l1 || ext.l0;
-  });
-
-  let vectors: number[][];
-  try {
-    vectors = await embedder.embedBatchPassage(textsToEmbed);
-  } catch (e) {
-    console.error(`    Embedding failed: ${e}`);
-    return { newCount: 0, dedupCount: 0, errors: validIndices.length };
-  }
-
-  // Step 4: Dedup + store each extracted record
-  const idMapping = new Map<number, string>();
-  for (let vi = 0; vi < validIndices.length; vi++) {
-    const idx = validIndices[vi];
-    const extraction = rawExtractions[idx]!;
-    const vector = vectors[vi];
-    const original = originals[idx];
-
-    // Skip per-record dedup — distilled L0/L1 text is unlikely to duplicate existing records,
-    // and dedupCheck was the main bottleneck (vector search per record). Post-distillation
-    // health check can detect duplicates if needed.
-
-    if (dryRun) {
-      console.log(`\n    [${extraction.category}] importance=${extraction.importance}`);
-      console.log(`       src: ${original.text.slice(0, 80)}...`);
-      console.log(`       L0: ${extraction.l0}`);
-      console.log(`       L1: ${(extraction.l1 || "").slice(0, 120)}`);
-      newCount++;
-      continue;
-    }
-
-    // Build new record
-    const boundary = buildDistillationBoundary(extraction.category as any);
-    const metadata = JSON.stringify({
-      source: "distillation",
-      distilledFrom: original.id,
-      originalScope: original.scope,
-      l0: extraction.l0,
-      l1: extraction.l1,
-      tier: extraction.importance >= 0.8 ? "working" : "peripheral",
-      boundary,
-    });
-
-    try {
-      const entry = await store.store({
-        text: extraction.l1 || extraction.l0,
-        vector,
-        category: extraction.category as any,
-        scope: original.scope,
-        importance: extraction.importance,
-        metadata,
-      });
-      idMapping.set(idx, entry.id);
-      newCount++;
-    } catch (e) {
-      console.error(`    Store failed: ${e}`);
-      errors++;
-    }
-  }
-
-  // Step 5: Archive originals (skip in dry-run)
-  if (dryRun) return { newCount, dedupCount, errors };
-  for (let vi = 0; vi < validIndices.length; vi++) {
-    const idx = validIndices[vi];
-    const original = originals[idx];
-    const distilledId = idMapping.get(idx);
-    if (!distilledId) continue; // Was deduped or failed
-    try {
-      const existingMeta = original.metadata ? JSON.parse(original.metadata) : {};
-      const updatedMeta = JSON.stringify({
-        ...existingMeta,
-        archived: true,
-        archivedAt: new Date().toISOString(),
-        distilledTo: [distilledId],
-        originalCategory: "fact",
-      });
-      await store.update(original.id, { metadata: updatedMeta });
-    } catch (e) {
-      console.error(`    Archive mark failed for ${original.id}: ${e}`);
-    }
-  }
-
-  return { newCount, dedupCount, errors };
-}
-
-// ─── Main ───────────────────────────────────────────────────────────────────
-
-async function main() {
-  const args = process.argv.slice(2);
-  const dryRun = args.includes("--dry-run");
-  const scopeIdx = args.indexOf("--scope");
-  const scopeArg = args.find(a => a.startsWith("--scope="))?.split("=")[1]
-    || (scopeIdx >= 0 ? args[scopeIdx + 1] : undefined);
-
-  const config = loadConfig();
-  const { store, embedder, llm } = createComponents(config);
-
-  if (!llm) {
-    console.error("LLM not configured. Check config.json llm section.");
-    process.exit(1);
-  }
-
-  const llmTest = await llm.test();
-  if (!llmTest.success) {
-    console.error(`LLM connection failed: ${llmTest.error}`);
-    process.exit(1);
-  }
-  console.log(`LLM: ${config.llm?.model}`);
-
-  // Fetch all fact records
-  console.log("\nReading fact records...");
-  const allFacts: MemoryEntry[] = await store.list(undefined, "fact", 50000, 0);
-  console.log(`Total: ${allFacts.length} fact records\n`);
-
-  // Filter out already-archived facts
-  const unarchived = allFacts.filter(f => {
-    if (!f.metadata) return true;
-    try { return !JSON.parse(f.metadata).archived; } catch { return true; }
-  });
-  console.log(`Unarchived: ${unarchived.length} (already archived: ${allFacts.length - unarchived.length})\n`);
-
-  // Group by scope
-  const byScope = new Map<string, MemoryEntry[]>();
-  for (const f of unarchived) {
-    const arr = byScope.get(f.scope) || [];
-    arr.push(f);
-    byScope.set(f.scope, arr);
-  }
-  const scopes = [...byScope.keys()].sort((a, b) => byScope.get(b)!.length - byScope.get(a)!.length);
-  console.log(`${scopes.length} scopes\n`);
-
-  // Load or create progress
-  let progress = loadProgress();
-  if (!progress) {
-    progress = {
-      startedAt: new Date().toISOString(),
-      lastUpdated: new Date().toISOString(),
-      completedScopes: [],
-      stats: { totalFacts: unarchived.length, processedFacts: 0, newRecords: 0, dedupedSkips: 0, errors: 0 },
-    };
-  }
-
-  // Filter scopes
-  let targetScopes = scopes;
-  if (scopeArg) {
-    targetScopes = scopes.filter(s => s === scopeArg);
-    if (targetScopes.length === 0) {
-      console.error(`Scope "${scopeArg}" not found`);
-      process.exit(1);
-    }
-  }
-  if (dryRun) {
-    targetScopes = [targetScopes[0]];
-    console.log(`DRY RUN: processing scope "${targetScopes[0]}" (${byScope.get(targetScopes[0])!.length} records)\n`);
-  }
-
-  // Process scopes — worker pool with immediate checkpoint per scope
-  const CONCURRENCY = 5;
-
-  // Filter to pending scopes, sort small-first for faster feedback
-  const pendingScopes = targetScopes.filter(s => !progress.completedScopes.includes(s));
-  pendingScopes.sort((a, b) => byScope.get(a)!.length - byScope.get(b)!.length);
-  const skipped = targetScopes.length - pendingScopes.length;
-  if (skipped > 0) console.log(`Skipping ${skipped} already-completed scopes`);
-  console.log(`Processing ${pendingScopes.length} scopes (small-first, concurrency=${CONCURRENCY})\n`);
-
-  let completed = 0;
-  const totalPending = pendingScopes.length;
-  let scopeQueue = [...pendingScopes];
-
-  async function worker() {
-    while (scopeQueue.length > 0) {
-      const scope = scopeQueue.shift()!;
-      const facts = byScope.get(scope)!;
-      let scopeNew = 0, scopeErrors = 0;
-
-      for (let i = 0; i < facts.length; i += BATCH_SIZE) {
-        const batch = facts.slice(i, i + BATCH_SIZE);
-        const texts = batch.map(f => f.text);
-        const result = await distillBatch(texts, batch, store, embedder, llm, dryRun);
-        scopeNew += result.newCount;
-        scopeErrors += result.errors;
-      }
-
-      // Checkpoint immediately
-      progress.completedScopes.push(scope);
-      progress.stats.processedFacts += facts.length;
-      progress.stats.newRecords += scopeNew;
-      progress.stats.errors += scopeErrors;
-      progress.lastUpdated = new Date().toISOString();
-      completed++;
-
-      if (!dryRun) saveProgress(progress);
-      console.log(`[${completed}/${totalPending}] ${scope} (${facts.length} facts, +${scopeNew} new, ${scopeErrors} err)`);
-    }
-  }
-
-  // Launch workers
-  await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
-
-  // Final summary
-  console.log("\n" + "=".repeat(60));
-  console.log("Distillation summary:");
-  console.log(`  Total facts: ${progress.stats.totalFacts}`);
-  console.log(`  Processed: ${progress.stats.processedFacts}`);
-  console.log(`  New knowledge: ${progress.stats.newRecords}`);
-  console.log(`  Dedup skipped: ${progress.stats.dedupedSkips}`);
-  console.log(`  Errors: ${progress.stats.errors}`);
-  console.log(`  Compression: ${(progress.stats.processedFacts / Math.max(progress.stats.newRecords, 1)).toFixed(1)}:1`);
-  if (dryRun) {
-    console.log("\nDRY RUN — no data was written. Remove --dry-run to run for real.");
-  }
-}
-
-main().catch(console.error);