recallnest 1.4.1 → 2.2.0

package/src/store.ts

@@ -9,6 +9,7 @@ import { dirname } from "node:path";
 import { logWarn } from "./stderr-log.js";
 import type { DurableMemoryCategory } from "./memory-schema.js";
 import { matchesScopeFilter } from "./scope-policy.js";
+import { detectEmotionIfEnabled } from "./emotion-detector.js";
 
 // ============================================================================
 // Types
@@ -42,6 +43,8 @@ export interface MemoryEntry {
   importance: number;
   timestamp: number;
   metadata?: string; // JSON string for extensible metadata — includes l0_abstract/l1_overview/l2_content/tier
+  language?: string;   // ISO 639-1: "zh"|"ja"|"ko"|"en"
+  fts_text?: string;   // Pre-tokenized text for FTS indexing
 }
 
 export interface MemorySearchResult {
@@ -258,14 +261,24 @@ export class MemoryStore {
     try {
       table = await db.openTable(TABLE_NAME);
 
-      // Check if we need to add scope column for backward compatibility
+      // Backward compatibility: add missing columns to existing tables
       try {
         const sample = await table.query().limit(1).toArray();
-        if (sample.length > 0 && !("scope" in sample[0])) {
-          logWarn("Adding scope column for backward compatibility with existing data");
+        if (sample.length > 0) {
+          if (!("scope" in sample[0])) {
+            logWarn("Adding scope column for backward compatibility with existing data");
+          }
+          if (!("language" in sample[0])) {
+            logWarn("Adding language column for backward compatibility");
+            await table.addColumns([{ name: "language", valueSql: "'en'" }]);
+          }
+          if (!("fts_text" in sample[0])) {
+            logWarn("Adding fts_text column for backward compatibility");
+            await table.addColumns([{ name: "fts_text", valueSql: "text" }]);
+          }
         }
       } catch (err) {
-        logWarn("Could not check table schema:", err);
+        logWarn("Could not check/migrate table schema:", err);
       }
     } catch (_openErr) {
       // Table doesn't exist yet — create it
@@ -278,6 +291,8 @@ export class MemoryStore {
         importance: 0,
         timestamp: 0,
         metadata: "{}",
+        language: "en",
+        fts_text: "__schema__",
       };
 
       try {
@@ -325,13 +340,13 @@ export class MemoryStore {
       // Check if FTS index already exists
       const indices = await table.listIndices();
       const hasFtsIndex = indices?.some((idx: any) =>
-        idx.indexType === "FTS" || idx.columns?.includes("text")
+        idx.indexType === "FTS" || idx.columns?.includes("fts_text")
       );
 
       if (!hasFtsIndex) {
         // LanceDB @lancedb/lancedb >=0.26: use Index.fts() config
         const lancedb = await loadLanceDB();
-        await table.createIndex("text", {
+        await table.createIndex("fts_text", {
           config: (lancedb as any).Index.fts(),
         });
       }
@@ -348,8 +363,17 @@ export class MemoryStore {
       id: entry.id || deterministicId(entry.scope, entry.text),
       timestamp: Date.now(),
       metadata: entry.metadata || "{}",
+      language: entry.language || "en",
+      fts_text: entry.fts_text || entry.text,
     };
 
+    const emotionResult = detectEmotionIfEnabled(fullEntry.text);
+    if (emotionResult) {
+      const existingMeta = JSON.parse(fullEntry.metadata || "{}");
+      existingMeta.emotion = emotionResult;
+      fullEntry.metadata = JSON.stringify(existingMeta);
+    }
+
     try {
       await this.table!.add([fullEntry]);
     } catch (err: any) {
@@ -375,10 +399,22 @@ export class MemoryStore {
       id: deterministicId(entry.scope, entry.text),
       timestamp: Date.now(),
       metadata: entry.metadata || "{}",
+      language: entry.language || "en",
+      fts_text: entry.fts_text || entry.text,
     }));
 
+    const enrichedEntries = fullEntries.map(e => {
+      const emotionResult = detectEmotionIfEnabled(e.text);
+      if (emotionResult) {
+        const meta = JSON.parse(e.metadata || "{}");
+        meta.emotion = emotionResult;
+        return { ...e, metadata: JSON.stringify(meta) };
+      }
+      return e;
+    });
+
     try {
-      await this.table!.add(fullEntries);
+      await this.table!.add(enrichedEntries);
     } catch (err: any) {
       const code = err.code || "";
       const message = err.message || String(err);
@@ -414,8 +450,22 @@ export class MemoryStore {
       importance: Number.isFinite(entry.importance) ? entry.importance : 0.5,
       timestamp: Number.isFinite(entry.timestamp) ? entry.timestamp : Date.now(),
       metadata: entry.metadata || "{}",
+      language: entry.language || "en",
+      fts_text: entry.fts_text || entry.text,
     };
 
+    // HP-emo: Backfill emotion if absent during import
+    try {
+      const metaParsed = JSON.parse(full.metadata || "{}");
+      if (!metaParsed.emotion) {
+        const emotionResult = detectEmotionIfEnabled(full.text);
+        if (emotionResult) {
+          metaParsed.emotion = emotionResult;
+          full.metadata = JSON.stringify(metaParsed);
+        }
+      }
+    } catch { /* malformed metadata — skip backfill */ }
+
     await this.table!.add([full]);
     return full;
   }
@@ -716,6 +766,30 @@ export class MemoryStore {
     };
   }
 
+  /**
+   * Batch-fetch vectors for a list of entry IDs.
+   * Used by graph-export for cross-scope semantic bridge computation.
+   * Single LanceDB query — much faster than N individual getById calls.
+   */
+  async getVectors(ids: string[]): Promise<Map<string, number[]>> {
+    await this.ensureInitialized();
+    if (ids.length === 0) return new Map();
+
+    const conditions = ids.map(id => `id = '${escapeSqlLiteral(id)}'`).join(" OR ");
+    const rows = await this.table!.query()
+      .select(["id", "vector"])
+      .where(conditions)
+      .limit(ids.length)
+      .toArray();
+
+    const result = new Map<string, number[]>();
+    for (const row of rows) {
+      const vec = Array.from(row.vector as Iterable<number>);
+      if (vec.length > 0) result.set(row.id as string, vec);
+    }
+    return result;
+  }
+
   async stats(scopeFilter?: string[]): Promise<{
     totalCount: number;
     scopeCounts: Record<string, number>;
@@ -800,7 +874,7 @@ export class MemoryStore {
 
   async update(
     id: string,
-    updates: { text?: string; vector?: number[]; importance?: number; category?: MemoryEntry["category"]; metadata?: string; timestamp?: number },
+    updates: { text?: string; vector?: number[]; importance?: number; category?: MemoryEntry["category"]; metadata?: string; timestamp?: number; language?: string; fts_text?: string },
     scopeFilter?: string[]
   ): Promise<MemoryEntry | null> {
     await this.ensureInitialized();
@@ -821,7 +895,7 @@ export class MemoryStore {
       rows = await this.table!.query().where(`id = '${safeId}'`).limit(1).toArray();
     } else {
       // Prefix match
-      const all = await this.table!.query().select(["id", "text", "vector", "category", "scope", "importance", "timestamp", "metadata"]).toArray();
+      const all = await this.table!.query().select(["id", "text", "vector", "category", "scope", "importance", "timestamp", "metadata", "language", "fts_text"]).toArray();
       rows = all.filter((r: any) => (r.id as string).startsWith(id));
       if (rows.length > 1) {
         throw new Error(`Ambiguous prefix "${id}" matches ${rows.length} memories. Use a longer prefix or full ID.`);
@@ -838,7 +912,7 @@ export class MemoryStore {
       throw new Error(`Memory ${id} is outside accessible scopes`);
     }
 
-    // Build updated entry, preserving original timestamp
+    // Build updated entry, preserving original timestamp and language fields
     const updated: MemoryEntry = {
       id: row.id as string,
       text: updates.text ?? (row.text as string),
@@ -848,8 +922,20 @@ export class MemoryStore {
       importance: updates.importance ?? Number(row.importance),
       timestamp: updates.timestamp ?? Number(row.timestamp),
       metadata: updates.metadata ?? ((row.metadata as string) || "{}"),
+      language: updates.language ?? ((row.language as string) || "en"),
+      fts_text: updates.fts_text ?? ((row.fts_text as string) || (updates.text ?? (row.text as string))),
     };
 
+    // HP-emo: Re-detect emotion when text changes
+    if (updates.text) {
+      const emotionResult = detectEmotionIfEnabled(updates.text);
+      if (emotionResult) {
+        const meta = JSON.parse(updated.metadata || "{}");
+        meta.emotion = emotionResult;
+        updated.metadata = JSON.stringify(meta);
+      }
+    }
+
     // LanceDB doesn't support in-place update; delete + re-add
     const resolvedId = escapeSqlLiteral(row.id as string);
     await this.table!.delete(`id = '${resolvedId}'`);

package/src/topic-tag.ts

@@ -0,0 +1,66 @@
+/**
+ * MP-1: Topic Tag — scope 内主题分区
+ *
+ * Keyword-based topic detection for memory entries.
+ * Tags are stored in metadata and used as a pre-filter during retrieval.
+ */
+
+const TOPIC_PATTERNS: Array<{ tag: string; patterns: RegExp[] }> = [
+  { tag: "auth", patterns: [/\b(auth|login|oauth|jwt|session.?token|credentials?|password|sso|saml)\b/i] },
+  { tag: "deploy", patterns: [/\b(deploy|ci[\s/]?cd|pipeline|release|rollout|staging|production|rollback)\b/i] },
+  { tag: "infra", patterns: [/\b(docker|kubernetes|k8s|terraform|nginx|server|cluster|load.?balanc|cdn|dns)\b/i] },
+  { tag: "testing", patterns: [/\b(test|spec|jest|vitest|bun.?test|coverage|assertion|mock|fixture|e2e)\b/i] },
+  { tag: "database", patterns: [/\b(database|sql|postgres|mysql|sqlite|lancedb|mongodb|migration|schema|index)\b/i] },
+  { tag: "api", patterns: [/\b(api|endpoint|rest|graphql|grpc|webhook|http|request|response|route)\b/i] },
+  { tag: "ui", patterns: [/\b(ui|frontend|react|vue|svelte|css|tailwind|component|layout|responsive)\b/i] },
+  { tag: "perf", patterns: [/\b(performance|latency|throughput|cache|optimization|bottleneck|profil|benchmark)\b/i] },
+  { tag: "security", patterns: [/\b(security|vulnerability|xss|csrf|injection|encrypt|certificate|tls|ssl)\b/i] },
+  { tag: "docs", patterns: [/\b(documentation|readme|changelog|tutorial|guide|specification|api.?doc)\b/i] },
+  { tag: "memory", patterns: [/\b(memory|recall|retriev|embed|vector|checkpoint|distill|ingest|lancedb)\b/i] },
+  { tag: "mcp", patterns: [/\b(mcp|model.?context.?protocol|tool.?registr|mcp.?server)\b/i] },
+  { tag: "llm", patterns: [/\b(llm|language.?model|prompt|token|context.?window|embedding|openai|anthropic|claude)\b/i] },
+  { tag: "config", patterns: [/\b(config|settings?|environment|env.?var|dotenv|feature.?flag)\b/i] },
+  { tag: "data", patterns: [/\b(data|dataset|csv|json|parsing|transform|etl|pipeline|ingest)\b/i] },
+];
+
+const MAX_SCAN_LENGTH = 2000;
+
+export function detectTopicTag(text: string): string | undefined {
+  const sample = text.slice(0, MAX_SCAN_LENGTH).toLowerCase();
+  let bestTag: string | undefined;
+  let bestCount = 0;
+
+  for (const { tag, patterns } of TOPIC_PATTERNS) {
+    let count = 0;
+    for (const pattern of patterns) {
+      const matches = sample.match(new RegExp(pattern.source, "gi"));
+      if (matches) count += matches.length;
+    }
+    if (count > bestCount) {
+      bestCount = count;
+      bestTag = tag;
+    }
+  }
+
+  return bestCount >= 1 ? bestTag : undefined;
+}
+
+export function extractTopicTag(metadata?: string): string | undefined {
+  if (!metadata) return undefined;
+  try {
+    const parsed = JSON.parse(metadata);
+    return typeof parsed.topicTag === "string" ? parsed.topicTag : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export function injectTopicTag(metadata: string, topicTag: string): string {
+  try {
+    const parsed = JSON.parse(metadata);
+    parsed.topicTag = topicTag;
+    return JSON.stringify(parsed);
+  } catch {
+    return metadata;
+  }
+}

package/src/tracker.ts

@@ -5,7 +5,9 @@
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
 import { resolve } from "node:path";
 
-const TRACKER_PATH = resolve(import.meta.dir, "../data/ingested-files.json");
+import { metaDir } from "./compat.js";
+
+const TRACKER_PATH = resolve(metaDir(import.meta), "../data/ingested-files.json");
 
 interface TrackerData {
   files: Record<string, { ingestedAt: string; size: number; chunks: number; mtimeMs?: number }>;

package/src/ui-server.ts

@@ -3,6 +3,7 @@
 import { existsSync, realpathSync } from "node:fs";
 import { join, resolve, sep } from "node:path";
 
+import { metaDir } from "./compat.js";
 import type { RetrievalResult } from "./retriever.js";
 import type { MemoryStore } from "./store.js";
 import { distillResults, formatExplainResults, formatSearchResults, selectBriefSeedResults, summarizeResults } from "./memory-output.js";
@@ -83,8 +84,8 @@ function errorResponse(error: unknown): Response {
 function isAllowedArtifactPath(targetPath: string): boolean {
   try {
     const resolved = realpathSync(targetPath);
-    const exportsDir = resolve(import.meta.dir, "../data/exports");
-    const pinsDir = resolve(import.meta.dir, "../data/pins");
+    const exportsDir = resolve(metaDir(import.meta), "../data/exports");
+    const pinsDir = resolve(metaDir(import.meta), "../data/pins");
     return (
       resolved === exportsDir ||
       resolved.startsWith(`${exportsDir}${sep}`) ||
@@ -141,7 +142,7 @@ function serializeResults(results: RetrievalResult[]) {
 }
 
 function serveStatic(pathname: string): Response | null {
-  const uiDir = resolve(import.meta.dir, "../assets/ui");
+  const uiDir = resolve(metaDir(import.meta), "../assets/ui");
   const relativePath = pathname === "/"
     ? join(uiDir, "index.html")
     : join(uiDir, pathname.replace(/^\/ui\//, ""));
@@ -157,14 +158,17 @@ function serveStatic(pathname: string): Response | null {
 async function handleSearch(mode: "search" | "explain" | "distill", body: Record<string, any>) {
   const query = requireString(body.query, "query");
   const { retriever, profile } = getComponents(readOptionalString(body.profile));
+  const explicitScope = readOptionalString(body.scope);
   const results = await retriever.retrieve(buildRetrievalContext({
     query,
     limit: readLimit(body.limit, 5, 1, 20),
-    scope: readOptionalString(body.scope),
+    scope: explicitScope,
     sessionId: readOptionalString(body.sessionId),
-    allScopes: body.allScopes === true,
+    allScopes: explicitScope ? false : true,
+    topicTag: readOptionalString(body.topicTag),
   }, {
     operation: `ui:${mode}`,
+    allowUnscoped: true,
   }));
   const context = { query, profile: profile.name };
   const output = mode === "explain"
@@ -260,6 +264,28 @@ const server = Bun.serve({
         });
       }
 
+      if (request.method === "GET" && url.pathname === "/api/skills") {
+        const { store, embedder } = getComponents();
+        const { retrieveSkills } = await import("./skill-engine.js");
+        const scope = url.searchParams.get("scope") || undefined;
+        const skills = await retrieveSkills(store, embedder, "all skills", scope, 20);
+        return Response.json({
+          items: skills.map(({ skill, score }) => ({
+            shortId: skill.id.slice(0, 8),
+            name: skill.name,
+            description: skill.description,
+            trigger: skill.triggerPattern,
+            type: skill.implementationType,
+            implementation: skill.implementation,
+            verification: skill.verification,
+            scope: skill.scope,
+            tags: skill.tags,
+            score: Math.round(score * 100),
+            storedAt: skill.storedAt,
+          })),
+        });
+      }
+
       if (request.method === "GET" && url.pathname === "/api/stats") {
         const { store } = getComponents();
         const stats = await store.stats();
@@ -296,7 +322,7 @@ const server = Bun.serve({
         const scopeSelection = resolveScopeSelection({
           scope: readOptionalString(body.scope),
           sessionId: readOptionalString(body.sessionId),
-          allScopes: body.allScopes === true,
+          allScopes: readOptionalString(body.scope) ? false : true,
           operation: "ui:pin",
         });
         const entry = await store.get(memoryId, scopeSelection.scopeFilter);
@@ -327,7 +353,7 @@ const server = Bun.serve({
           limit: readLimit(body.limit, 8, 1, 20),
           scope: readOptionalString(body.scope),
           sessionId: readOptionalString(body.sessionId),
-          allScopes: body.allScopes === true,
+          allScopes: readOptionalString(body.scope) ? false : true,
         }, {
           operation: "ui:brief",
         }));
@@ -377,7 +403,7 @@ const server = Bun.serve({
           limit: readLimit(body.limit, 8, 1, 20),
           scope: readOptionalString(body.scope),
           sessionId: readOptionalString(body.sessionId),
-          allScopes: body.allScopes === true,
+          allScopes: readOptionalString(body.scope) ? false : true,
         }, {
           operation: "ui:export",
         }));
@@ -420,6 +446,52 @@ const server = Bun.serve({
         });
       }
 
+      // ----- Dashboard API -----
+      if (request.method === "GET" && url.pathname === "/api/dashboard-stats") {
+        const { store } = getComponents();
+        const stats = await store.stats();
+        const allEntries = await store.list(undefined, undefined, 10000, 0);
+
+        const now = Date.now();
+        const weekAgo = now - 7 * 24 * 60 * 60 * 1000;
+        const monthAgo = now - 30 * 24 * 60 * 60 * 1000;
+        const thisWeek = allEntries.filter(e => e.timestamp > weekAgo).length;
+        const thisMonth = allEntries.filter(e => e.timestamp > monthAgo).length;
+
+        return Response.json({
+          totalCount: stats.totalCount,
+          categoryCounts: stats.categoryCounts,
+          scopeCounts: stats.scopeCounts,
+          growth: { thisWeek, thisMonth },
+        });
+      }
+
+      if (request.method === "GET" && url.pathname === "/api/stale-memories") {
+        const { store } = getComponents();
+        const { runMemoryLint } = await import("./memory-lint.js");
+        const report = await runMemoryLint({ store });
+        const staleFindings = report.findings.filter(f => f.check === "stale");
+        return Response.json({
+          count: staleFindings.length,
+          items: staleFindings.slice(0, 20).map(f => ({
+            detail: f.detail,
+            memoryIds: f.memoryIds,
+          })),
+        });
+      }
+
+      if (request.method === "GET" && url.pathname === "/api/lint-summary") {
+        const { store } = getComponents();
+        const { runMemoryLint } = await import("./memory-lint.js");
+        const report = await runMemoryLint({ store });
+        return Response.json({
+          healthScore: report.healthScore,
+          totalScanned: report.totalScanned,
+          summary: report.summary,
+          timestamp: report.timestamp,
+        });
+      }
+
       return new Response("Not Found", { status: 404 });
     } catch (error) {
       return errorResponse(error);

package/src/workflow-observation-scope-review.ts

@@ -1,6 +1,7 @@
 import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs";
 import { basename, join, resolve } from "node:path";
 
+import { metaDir } from "./compat.js";
 import { buildLegacyScopeKeepReview, suppressesLegacyScopeIssue } from "./legacy-scope-review.js";
 import { classifyLegacyScope } from "./store.js";
 
@@ -66,7 +67,7 @@ export interface WorkflowObservationScopeReviewOptions {
 }
 
 function defaultDir(): string {
-  return resolve(import.meta.dir, "../data/workflow-observations");
+  return resolve(metaDir(import.meta), "../data/workflow-observations");
 }
 
 function clip(text: string, maxLen = 72): string {

package/src/workflow-observation-store.ts

@@ -1,6 +1,7 @@
 import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join, resolve } from "node:path";
 
+import { metaDir } from "./compat.js";
 import type { WorkflowObservationOutcome, WorkflowObservationRecord } from "./workflow-observation-schema.js";
 import { WorkflowObservationRecordSchema } from "./workflow-observation-schema.js";
 
@@ -26,7 +27,7 @@ function sortNewestFirst(records: WorkflowObservationRecord[]): WorkflowObservat
 }
 
 export class WorkflowObservationStore {
-  constructor(private readonly dir = resolve(import.meta.dir, "../data/workflow-observations")) {}
+  constructor(private readonly dir = resolve(metaDir(import.meta), "../data/workflow-observations")) {}
 
   get dataDir(): string {
     return ensureDir(this.dir);

package/src/write-verifier.ts

@@ -0,0 +1,126 @@
+/**
+ * HP-2: Formation Review Pass — post-write verification.
+ * After storing a memory, verify: embedding exists, metadata complete, scope set.
+ * Non-blocking: failures are logged but never block the write path.
+ */
+
+import type { MemoryStore, MemoryEntry } from "./store.js";
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export interface WriteVerifierConfig {
+  /** Enable/disable verification (default: true) */
+  enabled: boolean;
+  /** Maximum time to wait for verification in ms (default: 2000) */
+  timeoutMs: number;
+}
+
+export const DEFAULT_VERIFIER_CONFIG: WriteVerifierConfig = {
+  enabled: true,
+  timeoutMs: 2000,
+};
+
+// ---------------------------------------------------------------------------
+// Verification result
+// ---------------------------------------------------------------------------
+
+export type VerificationIssue =
+  | "missing_entry"
+  | "missing_vector"
+  | "missing_scope"
+  | "missing_importance"
+  | "empty_text";
+
+export interface VerificationResult {
+  ok: boolean;
+  entryId: string;
+  issues: VerificationIssue[];
+  durationMs: number;
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Verify a just-written memory entry by reading it back.
+ * Returns issues found (empty = all good).
+ */
+export async function verifyWrite(
+  store: Pick<MemoryStore, "get">,
+  entryId: string,
+  cfg?: Partial<WriteVerifierConfig>,
+): Promise<VerificationResult> {
+  const config = { ...DEFAULT_VERIFIER_CONFIG, ...cfg };
+  const start = Date.now();
+
+  if (!config.enabled) {
+    return { ok: true, entryId, issues: [], durationMs: 0 };
+  }
+
+  try {
+    const entry = await Promise.race([
+      store.get(entryId),
+      new Promise<null>((_, reject) =>
+        setTimeout(() => reject(new Error("timeout")), config.timeoutMs),
+      ),
+    ]) as MemoryEntry | null;
+
+    const issues = checkEntry(entry, entryId);
+    return {
+      ok: issues.length === 0,
+      entryId,
+      issues,
+      durationMs: Date.now() - start,
+    };
+  } catch {
+    return {
+      ok: false,
+      entryId,
+      issues: ["missing_entry"],
+      durationMs: Date.now() - start,
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Internal checks
+// ---------------------------------------------------------------------------
+
+function checkEntry(entry: MemoryEntry | null, id: string): VerificationIssue[] {
+  const issues: VerificationIssue[] = [];
+
+  if (!entry) {
+    issues.push("missing_entry");
+    return issues;
+  }
+
+  // Vector should exist (embedding succeeded)
+  if (!entry.vector || (Array.isArray(entry.vector) && entry.vector.length === 0)) {
+    issues.push("missing_vector");
+  }
+
+  // Text should not be empty
+  if (!entry.text || entry.text.trim().length === 0) {
+    issues.push("empty_text");
+  }
+
+  // Metadata should contain scope
+  if (entry.metadata) {
+    try {
+      const meta = JSON.parse(entry.metadata);
+      if (!meta.scope) issues.push("missing_scope");
+      if (typeof meta.importance !== "number") issues.push("missing_importance");
+    } catch {
+      issues.push("missing_scope");
+      issues.push("missing_importance");
+    }
+  } else {
+    issues.push("missing_scope");
+    issues.push("missing_importance");
+  }
+
+  return issues;
+}