tinker-agent 2.0.0 → 2.2.0

package/src/memory/memory-store.ts

@@ -5,14 +5,16 @@ import path from "node:path";
 import { Database } from "bun:sqlite";
 import { createUuidV7, isCanonicalUuidV7 } from "../ids/uuid-v7";
 import {
-  MAX_MEMORIES_PER_TURN,
+  MAX_MEMORY_SUMMARY_BYTES,
   MAX_MEMORY_TEXT_BYTES,
   MEMORY_SCHEMA_VERSION,
   MEMORY_SEARCH_LIMIT,
   MemoryError,
   type MemoryEmbeddingIdentity,
+  type MemoryFtsMatch,
   type MemoryPaths,
   type MemorySearchMatch,
+  type StoredMemoryRecord,
   type StoredMemorySummary,
   type MemoryWriteBatch,
   type MemoryWriteResult,
@@ -34,6 +36,7 @@ const CREATE_MEMORY_META_SQL = `CREATE TABLE memory_meta (
 const CREATE_MEMORIES_SQL = `CREATE TABLE memories (
   memory_id TEXT PRIMARY KEY,
   text TEXT NOT NULL,
+  summary TEXT NOT NULL DEFAULT '',
   text_sha256 TEXT NOT NULL UNIQUE,
   embedding BLOB NOT NULL,
   source_workspace TEXT NOT NULL,
@@ -45,6 +48,16 @@ const CREATE_MEMORIES_SQL = `CREATE TABLE memories (
 const CREATE_MEMORIES_INDEX_SQL = `CREATE INDEX memories_created_at
 ON memories(created_at DESC)`;
 
+const MEMORIES_FTS_TABLE = "memories_fts";
+
+const CREATE_MEMORIES_FTS_SQL = `CREATE VIRTUAL TABLE memories_fts USING fts5(
+  text,
+  summary,
+  content='memories',
+  content_rowid='rowid',
+  tokenize='trigram'
+)`;
+
 const EXPECTED_SCHEMA = new Map([
   ["index:memories_created_at", CREATE_MEMORIES_INDEX_SQL],
   ["table:memories", CREATE_MEMORIES_SQL],
@@ -72,6 +85,7 @@ export function resolveMemoryPaths(homeRoot = os.homedir()): MemoryPaths {
 export class MemoryStore {
   readonly paths: MemoryPaths;
   readonly dimensions: number;
+  readonly ftsAvailable: boolean;
   private closed = false;
 
   private constructor(
@@ -79,10 +93,12 @@ export class MemoryStore {
     input: Required<Pick<OpenMemoryStoreInput, "clock" | "createMemoryId">> & {
       readonly paths: MemoryPaths;
       readonly embedding: MemoryEmbeddingIdentity;
+      readonly ftsAvailable: boolean;
     },
   ) {
     this.paths = input.paths;
     this.dimensions = input.embedding.dimensions;
+    this.ftsAvailable = input.ftsAvailable;
     this.clock = input.clock;
     this.createMemoryId = input.createMemoryId;
   }
@@ -120,7 +136,13 @@ export class MemoryStore {
         safeIntegers: true,
       });
       configureDatabase(database, busyTimeoutMs);
-      initializeOrVerifySchema(database, input.embedding);
+      initializeOrVerifySchema(database, input.embedding, paths.database);
+      let ftsAvailable = true;
+      try {
+        ensureFtsIndex(database);
+      } catch {
+        ftsAvailable = false;
+      }
       await secureCreatedAuxiliaryFile(`${paths.database}-wal`, walExisted);
       await secureCreatedAuxiliaryFile(`${paths.database}-shm`, shmExisted);
       await validatePrivateFile(paths.database);
@@ -130,6 +152,7 @@ export class MemoryStore {
         embedding: Object.freeze({ ...input.embedding }),
         clock: input.clock ?? (() => new Date().toISOString()),
         createMemoryId: input.createMemoryId ?? createUuidV7,
+        ftsAvailable,
       });
     } catch (error) {
       database?.close();
@@ -163,11 +186,16 @@ export class MemoryStore {
     runImmediateTransaction(this.database, () => {
       const insert = this.database.query(
         `INSERT INTO memories (
-           memory_id, text, text_sha256, embedding, source_workspace,
+           memory_id, text, summary, text_sha256, embedding, source_workspace,
            source_session_id, source_turn_id, created_at
-         ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
          ON CONFLICT(text_sha256) DO NOTHING`,
       );
+      const ftsInsert = this.ftsAvailable
+        ? this.database.query(
+            `INSERT INTO ${MEMORIES_FTS_TABLE}(rowid, text, summary) VALUES (?, ?, ?)`,
+          )
+        : undefined;
       for (const candidate of input.candidates) {
         const memoryId = this.createMemoryId();
         if (!isCanonicalUuidV7(memoryId)) {
@@ -179,6 +207,7 @@ export class MemoryStore {
         const result = insert.run(
           memoryId,
           candidate.text,
+          candidate.summary,
           sha256(candidate.text),
           encodeEmbedding(candidate.embedding),
           input.workspaceRoot,
@@ -194,6 +223,7 @@ export class MemoryStore {
           );
         }
         if (changes === 1) {
+          ftsInsert?.run(result.lastInsertRowid, candidate.text, candidate.summary);
           inserted.push(
             Object.freeze({
               memoryId,
@@ -236,7 +266,8 @@ export class MemoryStore {
     const matches: MemorySearchMatch[] = [];
     const rows = this.database
       .query(
-        `SELECT memory_id, text, embedding, source_workspace, created_at
+        `SELECT memory_id, text, summary, embedding, source_workspace,
+                source_session_id, created_at
          FROM memories`,
       )
       .iterate();
@@ -247,8 +278,10 @@ export class MemoryStore {
         Object.freeze({
           memoryId: sqlString(row.memory_id, "memory_id"),
           text: sqlString(row.text, "memory text"),
+          summary: sqlSummary(row.summary, "memory summary"),
           score: cosineFromNormalized(queryEmbedding, embedding),
           sourceWorkspace: sqlString(row.source_workspace, "memory source_workspace"),
+          sourceSessionId: sqlString(row.source_session_id, "memory source_session_id"),
           createdAt: requireUtcTimestamp(
             sqlString(row.created_at, "memory created_at"),
             "memory created_at",
@@ -265,12 +298,74 @@ export class MemoryStore {
     return Object.freeze(matches.slice(0, limit));
   }
 
+  searchFts(
+    keywords: readonly string[],
+    limit = MEMORY_SEARCH_LIMIT,
+  ): readonly MemoryFtsMatch[] {
+    this.requireOpen();
+    if (!this.ftsAvailable) {
+      throw new MemoryError(
+        "memory_fts_unavailable",
+        "Global memory keyword index is unavailable.",
+      );
+    }
+    if (!Number.isSafeInteger(limit) || limit < 1) {
+      throw new MemoryError(
+        "memory_search_limit_invalid",
+        "Memory search limit must be a positive safe integer.",
+      );
+    }
+    const matchExpression = buildFtsMatchExpression(keywords);
+    if (matchExpression === null) {
+      return Object.freeze([]);
+    }
+    const matches: MemoryFtsMatch[] = [];
+    const rows = this.database
+      .query(
+        `SELECT m.memory_id, m.text, m.summary, m.source_workspace,
+                m.source_session_id, m.created_at,
+                bm25(${MEMORIES_FTS_TABLE}, 10.0, 1.0) AS bm25
+         FROM ${MEMORIES_FTS_TABLE}
+         JOIN memories AS m ON m.rowid = ${MEMORIES_FTS_TABLE}.rowid
+         WHERE ${MEMORIES_FTS_TABLE} MATCH ?
+         ORDER BY bm25 ASC, m.created_at DESC, m.memory_id ASC
+         LIMIT ?`,
+      )
+      .iterate(matchExpression, limit);
+    for (const rowValue of rows) {
+      const row = sqlRecord(rowValue, "memory fts row");
+      const bm25 = Number(row.bm25);
+      if (!Number.isFinite(bm25)) {
+        throw new MemoryError(
+          "memory_store_read_failed",
+          "Memory FTS bm25 score must be finite.",
+        );
+      }
+      matches.push(
+        Object.freeze({
+          memoryId: sqlString(row.memory_id, "memory_id"),
+          text: sqlString(row.text, "memory text"),
+          summary: sqlSummary(row.summary, "memory summary"),
+          bm25,
+          sourceWorkspace: sqlString(row.source_workspace, "memory source_workspace"),
+          sourceSessionId: sqlString(row.source_session_id, "memory source_session_id"),
+          createdAt: requireUtcTimestamp(
+            sqlString(row.created_at, "memory created_at"),
+            "memory created_at",
+          ),
+        }),
+      );
+    }
+    return Object.freeze(matches);
+  }
+
   listStoredMemories(): readonly StoredMemorySummary[] {
     this.requireOpen();
     const memories: StoredMemorySummary[] = [];
     const rows = this.database
       .query(
-        `SELECT memory_id, text, source_workspace, created_at
+        `SELECT memory_id, text, summary, source_workspace, source_session_id,
+                created_at
          FROM memories
          ORDER BY created_at DESC, memory_id DESC`,
       )
@@ -281,7 +376,9 @@ export class MemoryStore {
         Object.freeze({
           memoryId: sqlString(row.memory_id, "memory_id"),
           text: sqlString(row.text, "memory text"),
+          summary: sqlSummary(row.summary, "memory summary"),
           sourceWorkspace: sqlString(row.source_workspace, "memory source_workspace"),
+          sourceSessionId: sqlString(row.source_session_id, "memory source_session_id"),
           createdAt: requireUtcTimestamp(
             sqlString(row.created_at, "memory created_at"),
             "memory created_at",
@@ -292,6 +389,34 @@ export class MemoryStore {
     return Object.freeze(memories);
   }
 
+  getById(memoryId: string): StoredMemoryRecord | undefined {
+    this.requireOpen();
+    const rowValue = this.database
+      .query(
+        `SELECT memory_id, text, summary, source_workspace, source_session_id,
+                source_turn_id, created_at
+         FROM memories
+         WHERE memory_id = ?`,
+      )
+      .get(memoryId);
+    if (rowValue === null) {
+      return undefined;
+    }
+    const row = sqlRecord(rowValue, "memory row");
+    return Object.freeze({
+      memoryId: sqlString(row.memory_id, "memory_id"),
+      text: sqlString(row.text, "memory text"),
+      summary: sqlSummary(row.summary, "memory summary"),
+      sourceWorkspace: sqlString(row.source_workspace, "memory source_workspace"),
+      sourceSessionId: sqlString(row.source_session_id, "memory source_session_id"),
+      sourceTurnId: sqlString(row.source_turn_id, "memory source_turn_id"),
+      createdAt: requireUtcTimestamp(
+        sqlString(row.created_at, "memory created_at"),
+        "memory created_at",
+      ),
+    });
+  }
+
   count(): number {
     this.requireOpen();
     const row = this.database.query("SELECT COUNT(*) AS count FROM memories").get();
@@ -350,6 +475,7 @@ function configureDatabase(database: Database, busyTimeoutMs: number): void {
 function initializeOrVerifySchema(
   database: Database,
   embedding: MemoryEmbeddingIdentity,
+  databasePath: string,
 ): void {
   let started = false;
   try {
@@ -367,7 +493,7 @@ function initializeOrVerifySchema(
         insert.run(key, value);
       }
     }
-    verifySchema(database, embedding);
+    verifySchema(database, embedding, databasePath);
     database.exec("COMMIT");
     started = false;
   } catch (error) {
@@ -391,11 +517,15 @@ function initializeOrVerifySchema(
   }
 }
 
-function verifySchema(database: Database, embedding: MemoryEmbeddingIdentity): void {
+function verifySchema(
+  database: Database,
+  embedding: MemoryEmbeddingIdentity,
+  databasePath: string,
+): void {
   const objects = applicationSchemaObjects(database);
   if (objects.length !== EXPECTED_SCHEMA.size) {
-    throw new MemoryError(
-      "memory_schema_invalid",
+    throw unsupportedSchemaError(
+      databasePath,
       "Global memory schema has unexpected objects.",
     );
   }
@@ -403,8 +533,8 @@ function verifySchema(database: Database, embedding: MemoryEmbeddingIdentity): v
     const key = `${row.type}:${row.name}`;
     const expected = EXPECTED_SCHEMA.get(key);
     if (expected === undefined || normalizeSql(row.sql) !== normalizeSql(expected)) {
-      throw new MemoryError(
-        "memory_schema_invalid",
+      throw unsupportedSchemaError(
+        databasePath,
         `Global memory schema object ${key} is incompatible.`,
       );
     }
@@ -434,11 +564,16 @@ function verifySchema(database: Database, embedding: MemoryEmbeddingIdentity): v
     const identityMismatch = identityKeys.some(
       (key) => actual.has(key) && actual.get(key) !== expected.get(key),
     );
-    throw new MemoryError(
-      identityMismatch ? "memory_embedding_identity_mismatch" : "memory_schema_invalid",
-      identityMismatch
-        ? "Configured embedding profile does not match the existing global memory database."
-        : "Global memory metadata is invalid.",
+    if (identityMismatch) {
+      throw new MemoryError(
+        "memory_embedding_identity_mismatch",
+        "Configured embedding profile does not match the existing global memory database.",
+      );
+    }
+    throw unsupportedSchemaError(
+      databasePath,
+      "Global memory metadata does not match schema version " +
+        `${MEMORY_SCHEMA_VERSION}.`,
     );
   }
 
@@ -446,6 +581,15 @@ function verifySchema(database: Database, embedding: MemoryEmbeddingIdentity): v
   expectedEmbeddingBlobBytes(storedDimensions);
 }
 
+function unsupportedSchemaError(databasePath: string, detail: string): MemoryError {
+  return new MemoryError(
+    "memory_schema_unsupported",
+    `${detail} Delete ${databasePath} (including ${databasePath}-wal and ` +
+      `${databasePath}-shm) and restart to finish the memory schema v` +
+      `${MEMORY_SCHEMA_VERSION} upgrade.`,
+  );
+}
+
 function applicationSchemaObjects(
   database: Database,
 ): Array<{ readonly type: string; readonly name: string; readonly sql: string }> {
@@ -454,6 +598,7 @@ function applicationSchemaObjects(
       `SELECT type, name, sql
        FROM sqlite_schema
        WHERE type IN ('table', 'index') AND name NOT LIKE 'sqlite_%'
+         AND name <> '${MEMORIES_FTS_TABLE}' AND name NOT LIKE '${MEMORIES_FTS_TABLE}\\_%' ESCAPE '\\'
        ORDER BY type, name`,
     )
     .all()
@@ -479,6 +624,60 @@ function metadataEntries(
   ]);
 }
 
+function ensureFtsIndex(database: Database): void {
+  runImmediateTransaction(database, () => {
+    const rowValue = database
+      .query(
+        `SELECT sql FROM sqlite_schema
+         WHERE type = 'table' AND name = '${MEMORIES_FTS_TABLE}'`,
+      )
+      .get();
+    if (rowValue === null) {
+      database.exec(CREATE_MEMORIES_FTS_SQL);
+      backfillFtsIndex(database);
+      return;
+    }
+    const row = sqlRecord(rowValue, "memories_fts schema");
+    const sql = sqlString(row.sql, "memories_fts SQL");
+    if (normalizeSql(sql) !== normalizeSql(CREATE_MEMORIES_FTS_SQL)) {
+      database.exec(`DROP TABLE ${MEMORIES_FTS_TABLE}`);
+      database.exec(CREATE_MEMORIES_FTS_SQL);
+      backfillFtsIndex(database);
+      return;
+    }
+    const memoryCount = countRows(database, "memories");
+    const ftsCount = countRows(database, `${MEMORIES_FTS_TABLE}_docsize`);
+    if (memoryCount !== ftsCount) {
+      database.exec(
+        `INSERT INTO ${MEMORIES_FTS_TABLE}(${MEMORIES_FTS_TABLE}) VALUES('rebuild')`,
+      );
+    }
+  });
+}
+
+function backfillFtsIndex(database: Database): void {
+  database.exec(
+    `INSERT INTO ${MEMORIES_FTS_TABLE}(rowid, text, summary)
+     SELECT rowid, text, summary FROM memories`,
+  );
+}
+
+function countRows(database: Database, table: string): number {
+  const row = sqlRecord(
+    database.query(`SELECT COUNT(*) AS count FROM ${table}`).get(),
+    `${table} count`,
+  );
+  return Number(row.count);
+}
+
+export function buildFtsMatchExpression(keywords: readonly string[]): string | null {
+  const phrases = keywords
+    .map((keyword) => keyword.trim())
+    .filter((keyword) => [...keyword].length >= 3)
+    .map((keyword) => `"${keyword.replaceAll('"', '""')}"`);
+  return phrases.length === 0 ? null : phrases.join(" OR ");
+}
+
 function runImmediateTransaction(database: Database, operation: () => void): void {
   let started = false;
   try {
@@ -519,10 +718,10 @@ function validateWriteBatch(input: MemoryWriteBatch, dimensions: number): void {
       "Memory source session and turn IDs must not be empty.",
     );
   }
-  if (input.candidates.length > MAX_MEMORIES_PER_TURN) {
+  if (input.candidates.length > 1) {
     throw new MemoryError(
       "memory_write_invalid",
-      `A memory batch may contain at most ${MAX_MEMORIES_PER_TURN} candidates.`,
+      "A memory batch may contain at most 1 candidate.",
     );
   }
   for (const candidate of input.candidates) {
@@ -530,11 +729,12 @@ function validateWriteBatch(input: MemoryWriteBatch, dimensions: number): void {
       candidate.text.trim() !== candidate.text ||
       Buffer.byteLength(candidate.text, "utf8") < 1 ||
       Buffer.byteLength(candidate.text, "utf8") > MAX_MEMORY_TEXT_BYTES ||
+      Buffer.byteLength(candidate.summary, "utf8") > MAX_MEMORY_SUMMARY_BYTES ||
       candidate.embedding.length !== dimensions
     ) {
       throw new MemoryError(
         "memory_write_invalid",
-        "Memory candidate text or embedding is invalid.",
+        "Memory candidate text, summary, or embedding is invalid.",
       );
     }
   }
@@ -664,6 +864,13 @@ function sqlString(value: unknown, name: string): string {
   return value;
 }
 
+function sqlSummary(value: unknown, name: string): string {
+  if (typeof value !== "string") {
+    throw new MemoryError("memory_store_read_failed", `${name} must be a string.`);
+  }
+  return value;
+}
+
 function requireUtcTimestamp(value: string, name: string): string {
   if (!value.endsWith("Z") || Number.isNaN(Date.parse(value))) {
     throw new MemoryError(