tinker-agent 2.0.0 → 2.1.0

package/src/memory/contracts.ts

@@ -1,11 +1,19 @@
 import type { SessionId, TurnId } from "../ids/runtime-id";
 
 export const MEMORY_SEARCH_TOOL_NAME = "MemorySearch" as const;
-export const MEMORY_SCHEMA_VERSION = 1 as const;
-export const MAX_MEMORIES_PER_TURN = 4;
+export const MEMORY_GET_TOOL_NAME = "MemoryGet" as const;
+export const MEMORY_SCHEMA_VERSION = 2 as const;
 export const MAX_MEMORY_TEXT_BYTES = 512;
+export const MAX_MEMORY_SUMMARY_BYTES = 4_096;
+export const MAX_SEARCH_RESULT_SUMMARY_BYTES = 1_536;
 export const MAX_MEMORY_QUERY_BYTES = 1_024;
+export const MAX_MEMORY_ID_BYTES = 64;
 export const MEMORY_SEARCH_LIMIT = 5;
+export const MEMORY_RECALL_CANDIDATE_LIMIT = 20;
+export const MAX_MEMORY_KEYWORDS = 8;
+export const MAX_MEMORY_KEYWORD_BYTES = 128;
+export const MEMORY_RRF_K = 60;
+export const MEMORY_EXTRACTION_QUEUE_CAPACITY = 64;
 
 export type MemoryEmbeddingKind = "openai-compatible";
 
@@ -32,6 +40,7 @@ export type MemoryPaths = {
 
 export type MemoryWriteCandidate = {
   readonly text: string;
+  readonly summary: string;
   readonly embedding: Float32Array;
 };
 
@@ -57,18 +66,51 @@ export type MemoryInsertedRecord = {
 export type MemorySearchMatch = {
   readonly memoryId: string;
   readonly text: string;
+  readonly summary: string;
   readonly score: number;
   readonly sourceWorkspace: string;
+  readonly sourceSessionId: string;
   readonly createdAt: string;
 };
 
+export type MemoryRecallPath = "vector" | "fts";
+
+export type MemoryFtsMatch = {
+  readonly memoryId: string;
+  readonly text: string;
+  readonly summary: string;
+  readonly bm25: number;
+  readonly sourceWorkspace: string;
+  readonly sourceSessionId: string;
+  readonly createdAt: string;
+};
+
+export type MemoryHybridMatch = {
+  readonly memoryId: string;
+  readonly text: string;
+  readonly summary: string;
+  readonly score: number;
+  readonly via: readonly MemoryRecallPath[];
+  readonly sourceWorkspace: string;
+  readonly sourceSessionId: string;
+  readonly createdAt: string;
+};
+
+export type MemoryRecallDegraded = "vector" | "fts";
+
 export type StoredMemorySummary = {
   readonly memoryId: string;
   readonly text: string;
+  readonly summary: string;
   readonly sourceWorkspace: string;
+  readonly sourceSessionId: string;
   readonly createdAt: string;
 };
 
+export type StoredMemoryRecord = StoredMemorySummary & {
+  readonly sourceTurnId: string;
+};
+
 export type MemoryExtractionRejectedCounts = {
   readonly duplicate: number;
   readonly secret: number;
@@ -98,7 +140,11 @@ export type MemorySearchDiagnostic = {
   readonly workspace: string;
   readonly sessionId: string;
   readonly queryBytes: number;
+  readonly keywordCount: number;
   readonly returned: number;
+  readonly vectorReturned: number;
+  readonly ftsReturned: number;
+  readonly degraded: MemoryRecallDegraded | null;
   readonly scores: readonly number[];
   readonly ms: number;
 };
@@ -110,9 +156,21 @@ export type MemoryInitDiagnostic = {
   readonly reason: string;
 };
 
+export type MemoryGetDiagnostic = {
+  readonly at: string;
+  readonly kind: "get";
+  readonly outcome: "ok" | "failed";
+  readonly reason: string | null;
+  readonly workspace: string;
+  readonly sessionId: string;
+  readonly found: boolean;
+  readonly ms: number;
+};
+
 export type MemoryDiagnostic =
   | MemoryExtractionDiagnostic
   | MemorySearchDiagnostic
+  | MemoryGetDiagnostic
   | MemoryInitDiagnostic;
 
 export class MemoryError extends Error {
@@ -136,7 +194,7 @@ export function boundedMemoryError(error: unknown): string {
   return truncateUtf8(singleLine, 400);
 }
 
-function truncateUtf8(value: string, maxBytes: number): string {
+export function truncateUtf8(value: string, maxBytes: number): string {
   if (Buffer.byteLength(value, "utf8") <= maxBytes) {
     return value;
   }

package/src/memory/memory-coordinator.ts

@@ -8,17 +8,34 @@ import type { SessionId } from "../ids/runtime-id";
 import type { ModelContextBudget } from "../model/model-context-profile";
 import type { ModelClient } from "../model/model-client";
 import type { CompletedTurnSnapshot } from "../session/session-store";
-import type { MemorySearchRawResult, ToolExecutor } from "../tools/types";
+import type {
+  MemoryGetRawResult,
+  MemorySearchRawResult,
+  ToolExecutor,
+} from "../tools/types";
 import {
   boundedMemoryError,
+  MAX_SEARCH_RESULT_SUMMARY_BYTES,
+  MEMORY_EXTRACTION_QUEUE_CAPACITY,
   memoryErrorCode,
+  MEMORY_GET_TOOL_NAME,
+  MEMORY_RECALL_CANDIDATE_LIMIT,
+  MEMORY_RRF_K,
+  MEMORY_SEARCH_LIMIT,
   MEMORY_SEARCH_TOOL_NAME,
   MemoryError,
+  truncateUtf8,
   type MemoryEmbeddingConfig,
   type MemoryExtractionDiagnostic,
   type MemoryExtractionRejectedCounts,
+  type MemoryFtsMatch,
+  type MemoryGetDiagnostic,
+  type MemoryHybridMatch,
   type MemoryPaths,
+  type MemoryRecallDegraded,
+  type MemoryRecallPath,
   type MemorySearchDiagnostic,
+  type MemorySearchMatch,
   type StoredMemorySummary,
 } from "./contracts";
 import {
@@ -33,6 +50,7 @@ import {
   type MemoryExtractionResult,
 } from "./memory-extractor";
 import { ExtractedMemoryLog, MemoryLog } from "./memory-log";
+import { createMemoryGetToolExecutor } from "./memory-get-tool";
 import { createMemorySearchToolExecutor } from "./memory-search-tool";
 import { MemoryStore, resolveMemoryPaths } from "./memory-store";
 import { normalizeEmbedding } from "./vector";
@@ -62,7 +80,7 @@ export type CreateMemoryCoordinatorInput = {
 export class MemoryCoordinator implements CompletedTurnHook {
   private accepting = true;
   private active?: ActiveMemoryTask;
-  private pending?: MemoryWorkerTask;
+  private pending: MemoryWorkerTask[] = [];
 
   private constructor(
     private readonly store: MemoryStore,
@@ -124,7 +142,10 @@ export class MemoryCoordinator implements CompletedTurnHook {
       this.start(task);
       return;
     }
-    this.pending = task;
+    if (this.pending.length >= MEMORY_EXTRACTION_QUEUE_CAPACITY) {
+      this.pending.shift();
+    }
+    this.pending.push(task);
   }
 
   recordFailure(input: CompletedTurnHookFailure): void {
@@ -145,8 +166,18 @@ export class MemoryCoordinator implements CompletedTurnHook {
     readonly sessionId: SessionId;
   }): ToolExecutor {
     return createMemorySearchToolExecutor({
-      search: (query, signal) => this.search(query, signal, input),
-      recordInvalidCall: (queryBytes) => this.recordInvalidSearch(queryBytes, input),
+      search: (query, keywords, signal) => this.search(query, keywords, signal, input),
+      recordInvalidCall: (invalid) => this.recordInvalidSearch(invalid, input),
+    });
+  }
+
+  createGetToolExecutor(input: {
+    readonly workspaceRoot: string;
+    readonly sessionId: SessionId;
+  }): ToolExecutor {
+    return createMemoryGetToolExecutor({
+      get: (memoryId, signal) => this.get(memoryId, signal, input),
+      recordInvalidCall: () => this.recordInvalidGet(input),
     });
   }
 
@@ -159,7 +190,7 @@ export class MemoryCoordinator implements CompletedTurnHook {
       return;
     }
     this.accepting = false;
-    this.pending = undefined;
+    this.pending = [];
     this.active?.controller.abort(
       new MemoryError(
         "memory_coordinator_disposed",
@@ -183,8 +214,7 @@ export class MemoryCoordinator implements CompletedTurnHook {
           return;
         }
         this.active = undefined;
-        const next = this.accepting ? this.pending : undefined;
-        this.pending = undefined;
+        const next = this.accepting ? this.pending.shift() : undefined;
         if (next !== undefined) {
           this.start(next);
         }
@@ -204,7 +234,7 @@ export class MemoryCoordinator implements CompletedTurnHook {
     try {
       extraction = await this.extractor.extract(task.extractionEvidenceText, signal);
       inputTokens = extraction.inputTokens;
-      returned = extraction.memories.length;
+      returned = extraction.memory === null ? 0 : 1;
     } catch (error) {
       if (error instanceof MemoryExtractionSkippedError) {
         inputTokens = error.inputTokens;
@@ -246,17 +276,28 @@ export class MemoryCoordinator implements CompletedTurnHook {
       return;
     }
 
-    const safeMemories = extraction.memories.filter((text) => {
-      if (!containsSensitiveMemory(text)) {
-        return true;
-      }
-      rejected = Object.freeze({
-        ...rejected,
-        secret: rejected.secret + 1,
-      });
-      return false;
-    });
-    if (safeMemories.length === 0) {
+    const candidate = extraction.memory;
+    if (candidate === null) {
+      await this.log.append(
+        extractionDiagnostic({
+          clock: this.clock,
+          outcome: "ok",
+          reason: null,
+          workspace: task.workspaceRoot,
+          turnId: task.turnId,
+          inputTokens,
+          returned,
+          ms: elapsedMs(startedAt),
+        }),
+      );
+      return;
+    }
+
+    if (
+      containsSensitiveMemory(candidate.text) ||
+      containsSensitiveMemory(candidate.summary)
+    ) {
+      rejected = Object.freeze({ ...rejected, secret: 1 });
       await this.log.append(
         extractionDiagnostic({
           clock: this.clock,
@@ -273,24 +314,20 @@ export class MemoryCoordinator implements CompletedTurnHook {
       return;
     }
 
-    let embeddings: readonly Float32Array[];
+    let embedding: Float32Array;
     try {
-      const rawEmbeddings = await this.embeddingClient.embed(safeMemories, signal);
-      if (rawEmbeddings.length !== safeMemories.length) {
+      const rawEmbeddings = await this.embeddingClient.embed([candidate.text], signal);
+      if (rawEmbeddings.length !== 1 || rawEmbeddings[0] === undefined) {
         throw new MemoryError(
           "memory_embedding_response_invalid",
           "Embedding response count does not match the memory candidate count.",
         );
       }
-      embeddings = Object.freeze(
-        rawEmbeddings.map((vector) =>
-          normalizeEmbedding(vector, this.embeddingDimensions),
-        ),
-      );
+      embedding = normalizeEmbedding(rawEmbeddings[0], this.embeddingDimensions);
     } catch (error) {
       rejected = Object.freeze({
         ...rejected,
-        embedding: safeMemories.length,
+        embedding: 1,
       });
       await this.log.append(
         extractionDiagnostic({
@@ -315,10 +352,13 @@ export class MemoryCoordinator implements CompletedTurnHook {
         workspaceRoot: task.workspaceRoot,
         sessionId: task.sessionId,
         turnId: task.turnId,
-        candidates: safeMemories.map((text, index) => ({
-          text,
-          embedding: embeddings[index],
-        })),
+        candidates: [
+          {
+            text: candidate.text,
+            summary: candidate.summary,
+            embedding,
+          },
+        ],
       });
       rejected = Object.freeze({
         ...rejected,
@@ -366,22 +406,56 @@ export class MemoryCoordinator implements CompletedTurnHook {
   }
 
   private async search(
-    query: string,
+    query: string | null,
+    keywords: readonly string[],
     signal: AbortSignal,
     source: { readonly workspaceRoot: string; readonly sessionId: SessionId },
   ): Promise<MemorySearchRawResult> {
     const startedAt = performance.now();
-    const queryBytes = Buffer.byteLength(query, "utf8");
+    const queryBytes = query === null ? 0 : Buffer.byteLength(query, "utf8");
+    let degraded: MemoryRecallDegraded | null = null;
+    let vectorMatches: readonly MemorySearchMatch[] = [];
+    let ftsMatches: readonly MemoryFtsMatch[] = [];
     try {
-      const raw = await this.embeddingClient.embed([query], signal);
-      if (raw.length !== 1 || raw[0] === undefined) {
-        throw new MemoryError(
-          "memory_embedding_response_invalid",
-          "Embedding response did not contain the query vector.",
-        );
+      if (query !== null) {
+        try {
+          const raw = await this.embeddingClient.embed([query], signal);
+          if (raw.length !== 1 || raw[0] === undefined) {
+            throw new MemoryError(
+              "memory_embedding_response_invalid",
+              "Embedding response did not contain the query vector.",
+            );
+          }
+          const queryEmbedding = normalizeEmbedding(raw[0], this.embeddingDimensions);
+          vectorMatches = this.store.search(
+            queryEmbedding,
+            MEMORY_RECALL_CANDIDATE_LIMIT,
+          );
+        } catch (error) {
+          if (signal.aborted || keywords.length === 0) {
+            throw error;
+          }
+          degraded = "vector";
+        }
+      }
+      if (keywords.length > 0) {
+        try {
+          ftsMatches = this.store.searchFts(keywords, MEMORY_RECALL_CANDIDATE_LIMIT);
+        } catch (error) {
+          if (signal.aborted) {
+            throw error;
+          }
+          const vectorUsable = query !== null && degraded === null;
+          if (!vectorUsable) {
+            throw error;
+          }
+          degraded = "fts";
+        }
       }
-      const queryEmbedding = normalizeEmbedding(raw[0], this.embeddingDimensions);
-      const matches = this.store.search(queryEmbedding);
+      const fused = fuseMemoryRecall({ vector: vectorMatches, fts: ftsMatches }).slice(
+        0,
+        MEMORY_SEARCH_LIMIT,
+      );
       await this.log.append(
         searchDiagnostic({
           clock: this.clock,
@@ -390,19 +464,28 @@ export class MemoryCoordinator implements CompletedTurnHook {
           workspace: source.workspaceRoot,
           sessionId: source.sessionId,
           queryBytes,
-          returned: matches.length,
-          scores: matches.map((match) => roundScore(match.score)),
+          keywordCount: keywords.length,
+          returned: fused.length,
+          vectorReturned: vectorMatches.length,
+          ftsReturned: ftsMatches.length,
+          degraded,
+          scores: fused.map((match) => roundScore(match.score)),
           ms: elapsedMs(startedAt),
         }),
       );
       return {
         ok: true,
+        degraded,
         matches: Object.freeze(
-          matches.map((match) =>
+          fused.map((match) =>
             Object.freeze({
+              memoryId: match.memoryId,
               text: match.text,
+              summary: truncateUtf8(match.summary, MAX_SEARCH_RESULT_SUMMARY_BYTES),
               score: match.score,
+              via: match.via,
               sourceWorkspace: match.sourceWorkspace,
+              sourceSessionId: match.sourceSessionId,
               createdAt: match.createdAt,
             }),
           ),
@@ -420,6 +503,7 @@ export class MemoryCoordinator implements CompletedTurnHook {
           workspace: source.workspaceRoot,
           sessionId: source.sessionId,
           queryBytes,
+          keywordCount: keywords.length,
           ms: elapsedMs(startedAt),
         }),
       );
@@ -434,7 +518,7 @@ export class MemoryCoordinator implements CompletedTurnHook {
   }
 
   private recordInvalidSearch(
-    queryBytes: number,
+    invalid: { readonly queryBytes: number; readonly keywordCount: number },
     source: { readonly workspaceRoot: string; readonly sessionId: SessionId },
   ): Promise<void> {
     return this.log.append(
@@ -444,7 +528,82 @@ export class MemoryCoordinator implements CompletedTurnHook {
         reason: "memory_search_args_invalid",
         workspace: source.workspaceRoot,
         sessionId: source.sessionId,
-        queryBytes,
+        queryBytes: invalid.queryBytes,
+        keywordCount: invalid.keywordCount,
+        ms: 0,
+      }),
+    );
+  }
+
+  private async get(
+    memoryId: string,
+    signal: AbortSignal,
+    source: { readonly workspaceRoot: string; readonly sessionId: SessionId },
+  ): Promise<MemoryGetRawResult> {
+    const startedAt = performance.now();
+    try {
+      signal.throwIfAborted();
+      const record = this.store.getById(memoryId);
+      await this.log.append(
+        getDiagnostic({
+          clock: this.clock,
+          outcome: "ok",
+          reason: null,
+          workspace: source.workspaceRoot,
+          sessionId: source.sessionId,
+          found: record !== undefined,
+          ms: elapsedMs(startedAt),
+        }),
+      );
+      return {
+        ok: true,
+        memory:
+          record === undefined
+            ? null
+            : Object.freeze({
+                memoryId: record.memoryId,
+                text: record.text,
+                summary: record.summary,
+                sourceWorkspace: record.sourceWorkspace,
+                sourceSessionId: record.sourceSessionId,
+                sourceTurnId: record.sourceTurnId,
+                createdAt: record.createdAt,
+              }),
+      };
+    } catch (error) {
+      if (signal.aborted) {
+        throw error;
+      }
+      await this.log.append(
+        getDiagnostic({
+          clock: this.clock,
+          outcome: "failed",
+          reason: memoryErrorCode(error, "memory_get_failed"),
+          workspace: source.workspaceRoot,
+          sessionId: source.sessionId,
+          found: false,
+          ms: elapsedMs(startedAt),
+        }),
+      );
+      return {
+        ok: false,
+        error: boundedMemoryError(error),
+      };
+    }
+  }
+
+  private recordInvalidGet(source: {
+    readonly workspaceRoot: string;
+    readonly sessionId: SessionId;
+  }): Promise<void> {
+    return this.log.append(
+      getDiagnostic({
+        clock: this.clock,
+        outcome: "failed",
+        reason: "memory_get_args_invalid",
+        workspace: source.workspaceRoot,
+        sessionId: source.sessionId,
+        found: false,
         ms: 0,
       }),
     );
@@ -463,7 +622,10 @@ export function buildExtractionEvidenceText(
   }
   const messages = snapshot.messages
     .filter(
-      (message) => message.role !== "tool" || message.name !== MEMORY_SEARCH_TOOL_NAME,
+      (message) =>
+        message.role !== "tool" ||
+        (message.name !== MEMORY_SEARCH_TOOL_NAME &&
+          message.name !== MEMORY_GET_TOOL_NAME),
     )
     .map((message) => ({ ...message }));
   return JSON.stringify(
@@ -520,7 +682,11 @@ function searchDiagnostic(input: {
   readonly workspace: string;
   readonly sessionId: string;
   readonly queryBytes: number;
+  readonly keywordCount: number;
   readonly returned?: number;
+  readonly vectorReturned?: number;
+  readonly ftsReturned?: number;
+  readonly degraded?: MemoryRecallDegraded | null;
   readonly scores?: readonly number[];
   readonly ms: number;
 }): MemorySearchDiagnostic {
@@ -532,12 +698,37 @@ function searchDiagnostic(input: {
     workspace: input.workspace,
     sessionId: input.sessionId,
     queryBytes: input.queryBytes,
+    keywordCount: input.keywordCount,
     returned: input.returned ?? 0,
+    vectorReturned: input.vectorReturned ?? 0,
+    ftsReturned: input.ftsReturned ?? 0,
+    degraded: input.degraded ?? null,
     scores: Object.freeze([...(input.scores ?? [])]),
     ms: input.ms,
   });
 }
 
+function getDiagnostic(input: {
+  readonly clock: () => string;
+  readonly outcome: MemoryGetDiagnostic["outcome"];
+  readonly reason: string | null;
+  readonly workspace: string;
+  readonly sessionId: string;
+  readonly found: boolean;
+  readonly ms: number;
+}): MemoryGetDiagnostic {
+  return Object.freeze({
+    at: input.clock(),
+    kind: "get",
+    outcome: input.outcome,
+    reason: input.reason,
+    workspace: input.workspace,
+    sessionId: input.sessionId,
+    found: input.found,
+    ms: input.ms,
+  });
+}
+
 function emptyRejectedCounts(): MemoryExtractionRejectedCounts {
   return Object.freeze({
     duplicate: 0,
@@ -554,3 +745,76 @@ function elapsedMs(startedAt: number): number {
 function roundScore(score: number): number {
   return Math.round(score * 1_000) / 1_000;
 }
+
+export function fuseMemoryRecall(input: {
+  readonly vector: readonly MemorySearchMatch[];
+  readonly fts: readonly MemoryFtsMatch[];
+}): readonly MemoryHybridMatch[] {
+  type Entry = {
+    readonly memoryId: string;
+    readonly text: string;
+    readonly summary: string;
+    readonly sourceWorkspace: string;
+    readonly sourceSessionId: string;
+    readonly createdAt: string;
+    score: number;
+    readonly via: Set<MemoryRecallPath>;
+  };
+  const entries = new Map<string, Entry>();
+  const accumulate = (
+    match: {
+      readonly memoryId: string;
+      readonly text: string;
+      readonly summary: string;
+      readonly sourceWorkspace: string;
+      readonly sourceSessionId: string;
+      readonly createdAt: string;
+    },
+    rank: number,
+    path: MemoryRecallPath,
+  ): void => {
+    const contribution = 1 / (MEMORY_RRF_K + rank);
+    const existing = entries.get(match.memoryId);
+    if (existing !== undefined) {
+      existing.score += contribution;
+      existing.via.add(path);
+      return;
+    }
+    entries.set(match.memoryId, {
+      memoryId: match.memoryId,
+      text: match.text,
+      summary: match.summary,
+      sourceWorkspace: match.sourceWorkspace,
+      sourceSessionId: match.sourceSessionId,
+      createdAt: match.createdAt,
+      score: contribution,
+      via: new Set([path]),
+    });
+  };
+  input.vector.forEach((match, index) => accumulate(match, index + 1, "vector"));
+  input.fts.forEach((match, index) => accumulate(match, index + 1, "fts"));
+
+  return Object.freeze(
+    [...entries.values()]
+      .sort(
+        (left, right) =>
+          right.score - left.score ||
+          right.createdAt.localeCompare(left.createdAt) ||
+          left.memoryId.localeCompare(right.memoryId),
+      )
+      .map((entry) =>
+        Object.freeze({
+          memoryId: entry.memoryId,
+          text: entry.text,
+          summary: entry.summary,
+          score: entry.score,
+          via: Object.freeze(
+            (["vector", "fts"] as const).filter((path) => entry.via.has(path)),
+          ),
+          sourceWorkspace: entry.sourceWorkspace,
+          sourceSessionId: entry.sourceSessionId,
+          createdAt: entry.createdAt,
+        }),
+      ),
+  );
+}