typeclaw 0.36.7 → 0.37.0

package/src/bundled-plugins/memory/memory-retrieval.ts

@@ -2,7 +2,7 @@ import { z } from 'zod'
 
 import { lsTool, readTool, type Subagent, writeTool } from '@/plugin'
 
-import { memorySearchTool } from './search-tool'
+import { createMemorySearchTool } from './search-tool'
 
 export const memoryRetrievalPayloadSchema = z.object({
   parentSessionId: z.string().min(1),
@@ -29,7 +29,7 @@ export type CreateMemoryRetrievalSubagentOptions = {
   timeoutMs?: number
 }
 
-export const MEMORY_RETRIEVAL_SYSTEM_PROMPT = `You are the memory-retrieval subagent. Read the user's most recent prompt and decide what's relevant from BOTH topic shards in \`memory/topics/\` (consolidated long-term memory) AND undreamed daily-stream events under \`memory/streams/\` (recent fragments not yet folded into shards). Use \`memory_search\` to query both surfaces; use \`read\`/\`ls\` to pull full shard bodies when needed. Synthesize a focused ≤8 KB summary of the relevant memory. Save by \`write\`ing it to the exact path provided in your payload as \`cacheFilePath\`. Be ruthlessly concise. Do NOT write anywhere else. Do NOT delete files.
+export const MEMORY_RETRIEVAL_SYSTEM_PROMPT = `You are the memory-retrieval subagent. Read the user's most recent prompt and decide what's relevant across topic shards in \`memory/topics/\` (consolidated long-term memory), references in \`memory/references/\` (verbatim artifacts), AND undreamed daily-stream events under \`memory/streams/\` (recent fragments not yet folded into shards). Use \`memory_search\` to query all three surfaces; use \`read\`/\`ls\` to pull full shard bodies when needed. Synthesize a focused ≤8 KB summary of the relevant memory. Save by \`write\`ing it to the exact path provided in your payload as \`cacheFilePath\`. Be ruthlessly concise. Do NOT write anywhere else. Do NOT delete files.
 
 Search discipline: issue ALL your \`memory_search\` queries in a SINGLE response as parallel tool calls (up to 3 at once), then wait for every result before deciding what to do next. Different angles in parallel, NEVER one search per turn — sequential searches waste a full LLM round-trip per query (~3s each) on file I/O that takes milliseconds. Pick queries that match the user's literal phrasing — not framing vocabulary, not metadata (session ids, dates), not words from your own system prompt. If the parallel batch turns up nothing relevant, write the empty-context note and stop.`
 
@@ -62,7 +62,7 @@ export function createMemoryRetrievalSubagent(
     // operator hasn't configured it, so this is safe by construction.
     profile: 'fast',
     tools: [readTool, writeTool, lsTool],
-    customTools: [memorySearchTool],
+    customTools: [createMemorySearchTool()],
     payloadSchema: memoryRetrievalPayloadSchema,
     inFlightKey: (payload) => payload.parentSessionId,
     ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),

package/src/bundled-plugins/memory/parent-link.ts

@@ -0,0 +1,33 @@
+import { splitCitationsBySection } from './citations'
+import type { TopicShard } from './load-shards'
+
+export type ParentLinks = {
+  supersededFragmentIds: Set<string>
+  parentSlugsByFragmentId: Map<string, Set<string>>
+}
+
+export function buildParentLinks(shards: TopicShard[]): ParentLinks {
+  const parentSlugsByFragmentId = new Map<string, Set<string>>()
+  const supersededFragmentIds = new Set<string>()
+
+  for (const shard of shards) {
+    const { active, superseded } = splitCitationsBySection(shard.body)
+    // A fragment can be cited by multiple topics (it backs more than one belief),
+    // so collect every citing slug — first-wins would drop the fragment's other
+    // parents and a match would collapse to only one of them.
+    for (const fragmentId of active) addSlug(parentSlugsByFragmentId, fragmentId, shard.slug)
+    for (const fragmentId of superseded) supersededFragmentIds.add(fragmentId)
+  }
+
+  // Active in one shard outranks superseded in another: the fragment still backs
+  // a live belief, so it stays a valid retrieval hook.
+  for (const fragmentId of parentSlugsByFragmentId.keys()) supersededFragmentIds.delete(fragmentId)
+
+  return { parentSlugsByFragmentId, supersededFragmentIds }
+}
+
+function addSlug(map: Map<string, Set<string>>, fragmentId: string, slug: string): void {
+  const slugs = map.get(fragmentId)
+  if (slugs === undefined) map.set(fragmentId, new Set([slug]))
+  else slugs.add(slug)
+}

package/src/bundled-plugins/memory/paths.ts

@@ -6,6 +6,7 @@ export const MEMORY_DIR = 'memory'
 export const TOPICS_SUBDIR = 'topics'
 export const STREAMS_SUBDIR = 'streams'
 export const SKILLS_SUBDIR = 'skills'
+export const REFERENCES_SUBDIR = 'references'
 export const PRE_SHARD_BACKUP_FILENAME = 'MEMORY.md.pre-shard.bak'
 export const MIGRATING_TMPDIR = 'memory/.migrating'
 
@@ -15,6 +16,10 @@ export function topicsDir(agentDir: string): string {
   return join(agentDir, MEMORY_DIR, TOPICS_SUBDIR)
 }
 
+export function referencesDir(agentDir: string): string {
+  return join(agentDir, MEMORY_DIR, REFERENCES_SUBDIR)
+}
+
 export function streamsDir(agentDir: string): string {
   return join(agentDir, MEMORY_DIR, STREAMS_SUBDIR)
 }
@@ -26,6 +31,13 @@ export function topicShardPath(agentDir: string, slug: string): string {
   return join(agentDir, MEMORY_DIR, TOPICS_SUBDIR, `${slug}.md`)
 }
 
+export function referenceFilePath(agentDir: string, slug: string): string {
+  if (slug.includes('..') || slug.includes('/') || slug.includes('\\') || slug.startsWith('.')) {
+    throw new Error(`invalid reference slug: ${JSON.stringify(slug)}`)
+  }
+  return join(agentDir, MEMORY_DIR, REFERENCES_SUBDIR, `${slug}.md`)
+}
+
 export function streamFilePath(agentDir: string, date: string): string {
   if (!STREAM_DATE_RE.test(date)) {
     throw new Error(`invalid stream date: ${JSON.stringify(date)}`)

package/src/bundled-plugins/memory/references/frontmatter.ts

@@ -0,0 +1,197 @@
+export type ReferenceOrigin = 'episode' | 'curated' | 'external'
+
+export type ReferenceFrontmatter = {
+  title: string
+  origin: ReferenceOrigin
+  created: string
+  lastAccessed: string
+  accessCount: number
+  pinned: boolean
+  demoted: boolean
+  tags: string[]
+}
+
+const ORIGINS = new Set<ReferenceOrigin>(['episode', 'curated', 'external'])
+const ISO_WITH_TIMEZONE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/
+
+export function parseReference(text: string): { frontmatter: ReferenceFrontmatter; body: string } {
+  const normalized = text.replaceAll('\r\n', '\n')
+
+  if (!normalized.startsWith('---\n')) {
+    throw new Error('frontmatter delimiter missing')
+  }
+
+  const closeIndex = normalized.indexOf('\n---', 4)
+  if (closeIndex === -1) {
+    throw new Error('frontmatter delimiter missing')
+  }
+
+  const fmText = normalized.slice(4, closeIndex)
+  const body = normalized.slice(closeIndex + 5)
+
+  const frontmatter = parseFrontmatterBlock(fmText)
+  return { frontmatter, body }
+}
+
+export function renderReference(frontmatter: ReferenceFrontmatter, body: string): string {
+  const lines = ['---']
+  lines.push(`title: ${frontmatter.title}`)
+  lines.push(`origin: ${frontmatter.origin}`)
+  lines.push(`created: ${frontmatter.created}`)
+  lines.push(`lastAccessed: ${frontmatter.lastAccessed}`)
+  lines.push(`accessCount: ${frontmatter.accessCount}`)
+  lines.push(`pinned: ${frontmatter.pinned}`)
+  lines.push(`demoted: ${frontmatter.demoted}`)
+  if (frontmatter.tags.length === 0) {
+    lines.push('tags: []')
+  } else {
+    lines.push(`tags: [${frontmatter.tags.join(', ')}]`)
+  }
+  lines.push('---')
+  lines.push(body)
+  return lines.join('\n')
+}
+
+function parseFrontmatterBlock(text: string): ReferenceFrontmatter {
+  const lines = text.split('\n')
+  const values: Record<string, unknown> = {}
+
+  let i = 0
+  while (i < lines.length) {
+    const line = lines[i]!
+    if (line.trim() === '') {
+      i++
+      continue
+    }
+
+    const colonIndex = line.indexOf(':')
+    if (colonIndex === -1) {
+      i++
+      continue
+    }
+
+    const key = line.slice(0, colonIndex).trim()
+    const rest = line.slice(colonIndex + 1).trim()
+
+    if (key === 'tags') {
+      if (rest === '') {
+        const listItems: string[] = []
+        i++
+        while (i < lines.length) {
+          const listLine = lines[i]!
+          if (!listLine.startsWith('  - ')) break
+          listItems.push(listLine.slice(4).trim())
+          i++
+        }
+        values.tags = listItems
+        continue
+      }
+      if (rest.startsWith('[') && rest.endsWith(']')) {
+        values.tags = rest
+          .slice(1, -1)
+          .split(',')
+          .map((s) => s.trim())
+          .filter(Boolean)
+        i++
+        continue
+      }
+      throw new Error(`frontmatter field 'tags': expected array, got '${rest}'`)
+    }
+
+    if (key in FRONTMATTER_PARSERS) {
+      try {
+        values[key] = FRONTMATTER_PARSERS[key as keyof typeof FRONTMATTER_PARSERS](rest)
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err)
+        throw new Error(`frontmatter field '${key}': ${message}`)
+      }
+    } else {
+      throw new Error(`frontmatter field '${key}': unknown`)
+    }
+
+    i++
+  }
+
+  return buildReferenceFrontmatter(values)
+}
+
+const FRONTMATTER_PARSERS = {
+  title: (v: string) => v,
+  origin: parseOrigin,
+  created: parseIsoWithTimezone,
+  lastAccessed: parseIsoWithTimezone,
+  accessCount: parseNonNegativeInt,
+  pinned: parseBoolean,
+  demoted: parseBoolean,
+}
+
+function buildReferenceFrontmatter(values: Record<string, unknown>): ReferenceFrontmatter {
+  const title = values.title
+  if (typeof title !== 'string' || title.length === 0) {
+    throw new Error("frontmatter field 'title': required")
+  }
+
+  const origin = values.origin
+  if (!isReferenceOrigin(origin)) {
+    throw new Error(`frontmatter field 'origin': expected episode | curated | external, got '${values.origin}'`)
+  }
+
+  const created = values.created
+  if (typeof created !== 'string') {
+    throw new Error(`frontmatter field 'created': expected ISO 8601 datetime with timezone, got '${values.created}'`)
+  }
+
+  const pinned = values.pinned
+  if (typeof pinned !== 'boolean') {
+    throw new Error(`frontmatter field 'pinned': expected boolean, got '${values.pinned}'`)
+  }
+
+  const tags = values.tags
+  if (!Array.isArray(tags) || !tags.every((tag) => typeof tag === 'string')) {
+    throw new Error(`frontmatter field 'tags': expected array, got '${values.tags}'`)
+  }
+
+  return {
+    title,
+    origin,
+    created,
+    lastAccessed: typeof values.lastAccessed === 'string' ? values.lastAccessed : created,
+    accessCount: typeof values.accessCount === 'number' ? values.accessCount : 0,
+    pinned,
+    demoted: typeof values.demoted === 'boolean' ? values.demoted : false,
+    tags,
+  }
+}
+
+function parseOrigin(value: string): ReferenceOrigin {
+  if (!isReferenceOrigin(value)) {
+    throw new Error(`expected episode | curated | external, got '${value}'`)
+  }
+  return value
+}
+
+function parseIsoWithTimezone(value: string): string {
+  if (!ISO_WITH_TIMEZONE_REGEX.test(value)) {
+    throw new Error(`expected ISO 8601 datetime with timezone, got '${value}'`)
+  }
+  return value
+}
+
+function parseNonNegativeInt(value: string): number {
+  const trimmed = value.trim()
+  const num = Number(trimmed)
+  if (!Number.isInteger(num) || String(num) !== trimmed || num < 0) {
+    throw new Error(`expected non-negative integer, got '${value}'`)
+  }
+  return num
+}
+
+function parseBoolean(value: string): boolean {
+  if (value === 'true') return true
+  if (value === 'false') return false
+  throw new Error(`expected boolean, got '${value}'`)
+}
+
+function isReferenceOrigin(value: unknown): value is ReferenceOrigin {
+  return typeof value === 'string' && ORIGINS.has(value as ReferenceOrigin)
+}

package/src/bundled-plugins/memory/references/load-references.ts

@@ -0,0 +1,212 @@
+import { readdir, readFile, stat, writeFile } from 'node:fs/promises'
+
+import { referenceFilePath, referencesDir } from '../paths'
+import { parseReference, renderReference, type ReferenceFrontmatter } from './frontmatter'
+
+export type Reference = {
+  path: string
+  slug: string
+  frontmatter: ReferenceFrontmatter
+  body: string
+}
+
+type Logger = { warn(message: string): void }
+
+type ReferenceCacheEntry = {
+  mtimeMs: number
+  ctimeMs: number
+  size: number
+  reference: Reference | null
+}
+
+type AgentReferenceCache = {
+  entries: Map<string, ReferenceCacheEntry>
+  lastSlugs: string[] | null
+  lastReferences: Reference[] | null
+}
+
+const referenceCache = new Map<string, AgentReferenceCache>()
+
+export async function loadAllReferences(agentDir: string, options: { logger?: Logger } = {}): Promise<Reference[]> {
+  const slugs = await listReferenceSlugs(agentDir)
+  const cache = getOrCreateCache(agentDir)
+  const outcomes = await Promise.all(slugs.map((slug) => resolveReference(agentDir, slug, cache.entries, options)))
+
+  const references: Reference[] = []
+  const seen = new Set<string>()
+  let changed = !sameSlugs(slugs, cache.lastSlugs)
+
+  for (const outcome of outcomes) {
+    seen.add(outcome.slug)
+    if (outcome.kind === 'missing') {
+      changed = true
+      cache.entries.delete(outcome.slug)
+      continue
+    }
+    if (outcome.kind === 'read') {
+      changed = true
+      cache.entries.set(outcome.slug, outcome.entry)
+    }
+    if (outcome.reference !== null) references.push(outcome.reference)
+  }
+
+  for (const slug of cache.entries.keys()) {
+    if (!seen.has(slug)) {
+      changed = true
+      cache.entries.delete(slug)
+    }
+  }
+
+  if (!changed && cache.lastReferences !== null) return cache.lastReferences
+
+  cache.lastSlugs = slugs
+  cache.lastReferences = references
+  return references
+}
+
+export async function loadReference(
+  agentDir: string,
+  slug: string,
+  options: { logger?: Logger } = {},
+): Promise<Reference | null> {
+  return readAndParseReference(referenceFilePath(agentDir, slug), slug, options)
+}
+
+// Records an access against the given reference slugs: bumps accessCount and
+// stamps lastAccessed. Used by both the memory_search hit path and the vector
+// retrieval path so a reference surfaced by either counts toward the dreaming
+// saturation model — otherwise a reference repeatedly injected by vector
+// retrieval (but never explicitly searched) decays on time alone and gets
+// evicted despite being actively useful. Best-effort: a missing or malformed
+// slug is skipped, never thrown, so a caller can fire-and-forget off the turn
+// critical path.
+export async function bumpReferenceAccess(
+  agentDir: string,
+  slugs: Iterable<string>,
+  options: { logger?: Logger } = {},
+): Promise<void> {
+  const unique = [...new Set(slugs)]
+  await Promise.all(
+    unique.map(async (slug) => {
+      const reference = await readAndParseReference(referenceFilePath(agentDir, slug), slug, options)
+      if (reference === null) return
+      await writeFile(
+        reference.path,
+        renderReference(
+          {
+            ...reference.frontmatter,
+            lastAccessed: new Date().toISOString(),
+            accessCount: reference.frontmatter.accessCount + 1,
+          },
+          reference.body,
+        ),
+        'utf8',
+      )
+    }),
+  )
+}
+
+export function __resetReferenceCacheForTests(): void {
+  referenceCache.clear()
+}
+
+type ReferenceOutcome =
+  | { kind: 'missing'; slug: string }
+  | { kind: 'cached'; slug: string; reference: Reference | null }
+  | { kind: 'read'; slug: string; reference: Reference | null; entry: ReferenceCacheEntry }
+
+async function resolveReference(
+  agentDir: string,
+  slug: string,
+  cache: Map<string, ReferenceCacheEntry>,
+  options: { logger?: Logger },
+): Promise<ReferenceOutcome> {
+  const path = referenceFilePath(agentDir, slug)
+  const fileStat = await statReference(path)
+  if (fileStat === null) return { kind: 'missing', slug }
+
+  const cached = cache.get(slug)
+  if (
+    cached !== undefined &&
+    cached.mtimeMs === fileStat.mtimeMs &&
+    cached.ctimeMs === fileStat.ctimeMs &&
+    cached.size === fileStat.size
+  ) {
+    return { kind: 'cached', slug, reference: cached.reference }
+  }
+
+  const reference = await readAndParseReference(path, slug, options)
+  const entry: ReferenceCacheEntry = {
+    mtimeMs: fileStat.mtimeMs,
+    ctimeMs: fileStat.ctimeMs,
+    size: fileStat.size,
+    reference,
+  }
+  return { kind: 'read', slug, reference, entry }
+}
+
+export async function listReferenceSlugs(agentDir: string): Promise<string[]> {
+  let names: string[]
+  try {
+    names = await readdir(referencesDir(agentDir))
+  } catch (err) {
+    if (isEnoent(err)) return []
+    throw err
+  }
+
+  return names
+    .filter((name) => name.endsWith('.md'))
+    .map((name) => name.slice(0, -'.md'.length))
+    .sort()
+}
+
+async function readAndParseReference(
+  path: string,
+  slug: string,
+  options: { logger?: Logger },
+): Promise<Reference | null> {
+  let text: string
+  try {
+    text = await readFile(path, 'utf8')
+  } catch (err) {
+    if (isEnoent(err)) return null
+    throw err
+  }
+
+  try {
+    const { frontmatter, body } = parseReference(text)
+    return { path, slug, frontmatter, body }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    const logger = options.logger ?? console
+    logger.warn(`[memory] skipping malformed reference ${slug}: ${message}`)
+    return null
+  }
+}
+
+async function statReference(path: string): Promise<{ mtimeMs: number; ctimeMs: number; size: number } | null> {
+  try {
+    const s = await stat(path)
+    return { mtimeMs: s.mtimeMs, ctimeMs: s.ctimeMs, size: s.size }
+  } catch (err) {
+    if (isEnoent(err)) return null
+    throw err
+  }
+}
+
+function getOrCreateCache(agentDir: string): AgentReferenceCache {
+  let cache = referenceCache.get(agentDir)
+  if (cache === undefined) {
+    cache = { entries: new Map(), lastSlugs: null, lastReferences: null }
+    referenceCache.set(agentDir, cache)
+  }
+  return cache
+}
+
+function sameSlugs(slugs: string[], previous: string[] | null): boolean {
+  return previous !== null && slugs.length === previous.length && slugs.every((slug, index) => slug === previous[index])
+}
+
+function isEnoent(err: unknown): boolean {
+  return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOENT'
+}

package/src/bundled-plugins/memory/references/store-reference-tool.ts

@@ -0,0 +1,59 @@
+import { mkdir, writeFile } from 'node:fs/promises'
+
+import { z } from 'zod'
+
+import { defineTool } from '@/plugin'
+
+import { referenceFilePath, referencesDir } from '../paths'
+import { headingToSlug } from '../slug'
+import { renderReference } from './frontmatter'
+import { listReferenceSlugs } from './load-references'
+
+export type ReferenceStoredHook = (context: { slug: string; body: string }) => Promise<void>
+
+export const storeReferenceTool = createStoreReferenceTool()
+
+export function createStoreReferenceTool(onReferenceStored?: ReferenceStoredHook) {
+  return defineTool({
+    description:
+      'store_reference: Store a verbatim reference artifact under memory/references/ and return its slug. Use this for user-provided SQL, code blocks, runbooks, pasted specs, or other content explicitly meant to be remembered byte-for-byte. This tool does not write memory stream fragments.',
+    parameters: z.object({
+      title: z.string().min(1),
+      body: z.string(),
+      origin: z.enum(['episode', 'curated', 'external']),
+      tags: z.array(z.string()).optional(),
+    }),
+    async execute({ title, body, origin, tags }, ctx) {
+      const existingSlugs = new Set(await listReferenceSlugs(ctx.agentDir))
+      const slug = headingToSlug(title, existingSlugs)
+      const created = new Date().toISOString()
+      const path = referenceFilePath(ctx.agentDir, slug)
+
+      await mkdir(referencesDir(ctx.agentDir), { recursive: true })
+      await writeFile(
+        path,
+        renderReference(
+          {
+            title,
+            origin,
+            created,
+            lastAccessed: created,
+            accessCount: 0,
+            pinned: false,
+            demoted: false,
+            tags: tags ?? [],
+          },
+          body,
+        ),
+        'utf8',
+      )
+
+      if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
+
+      return {
+        content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
+        details: { path, slug },
+      }
+    },
+  })
+}