typeclaw 0.7.0 → 0.9.0

package/src/bundled-plugins/memory/frontmatter.ts

@@ -0,0 +1,165 @@
+export type ShardFrontmatter = {
+  heading: string
+  cites: number
+  days: number
+  lastReinforced: string
+  tags?: string[]
+}
+
+const DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/
+
+export function parseShard(text: string): { frontmatter: ShardFrontmatter; body: string } {
+  const normalized = text.replaceAll('\r\n', '\n')
+
+  if (!normalized.startsWith('---\n')) {
+    throw new Error('frontmatter delimiter missing')
+  }
+
+  const closeIndex = normalized.indexOf('\n---', 4)
+  if (closeIndex === -1) {
+    throw new Error('frontmatter delimiter missing')
+  }
+
+  const fmText = normalized.slice(4, closeIndex)
+  const body = normalized.slice(closeIndex + 5)
+
+  const frontmatter = parseFrontmatterBlock(fmText)
+  return { frontmatter, body }
+}
+
+function parseFrontmatterBlock(text: string): ShardFrontmatter {
+  const lines = text.split('\n')
+  const values: Record<string, unknown> = {}
+
+  let i = 0
+  while (i < lines.length) {
+    const line = lines[i]!
+    if (line.trim() === '') {
+      i++
+      continue
+    }
+
+    const colonIndex = line.indexOf(':')
+    if (colonIndex === -1) {
+      i++
+      continue
+    }
+
+    const key = line.slice(0, colonIndex).trim()
+    const rest = line.slice(colonIndex + 1).trim()
+
+    if (key === 'tags') {
+      if (rest === '') {
+        const listItems: string[] = []
+        i++
+        while (i < lines.length) {
+          const listLine = lines[i]!
+          if (!listLine.startsWith('  - ')) break
+          listItems.push(listLine.slice(4).trim())
+          i++
+        }
+        values.tags = listItems
+        continue
+      } else if (rest.startsWith('[') && rest.endsWith(']')) {
+        values.tags = rest
+          .slice(1, -1)
+          .split(',')
+          .map((s) => s.trim())
+          .filter(Boolean)
+        i++
+        continue
+      } else if (rest === '[]') {
+        values.tags = []
+        i++
+        continue
+      } else {
+        throw new Error(`frontmatter field 'tags': expected array, got '${rest}'`)
+      }
+    }
+
+    if (key in FRONTMATTER_PARSERS) {
+      try {
+        values[key] = FRONTMATTER_PARSERS[key as keyof typeof FRONTMATTER_PARSERS](rest)
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err)
+        throw new Error(`frontmatter field '${key}': ${message}`)
+      }
+    } else {
+      throw new Error(`frontmatter field '${key}': unknown`)
+    }
+
+    i++
+  }
+
+  const heading = values.heading
+  if (typeof heading !== 'string' || heading.length === 0) {
+    throw new Error("frontmatter field 'heading': required")
+  }
+
+  const cites = values.cites
+  if (typeof cites !== 'number') {
+    throw new Error(`frontmatter field 'cites': expected integer, got '${values.cites}'`)
+  }
+
+  const days = values.days
+  if (typeof days !== 'number') {
+    throw new Error(`frontmatter field 'days': expected integer, got '${values.days}'`)
+  }
+
+  const lastReinforced = values.lastReinforced
+  if (typeof lastReinforced !== 'string' || !DATE_REGEX.test(lastReinforced)) {
+    throw new Error(`frontmatter field 'lastReinforced': expected YYYY-MM-DD, got '${values.lastReinforced}'`)
+  }
+
+  const result: ShardFrontmatter = { heading, cites, days, lastReinforced }
+  if ('tags' in values) {
+    result.tags = values.tags as string[]
+  }
+
+  return result
+}
+
+const FRONTMATTER_PARSERS: {
+  [K in keyof Omit<ShardFrontmatter, 'tags'>]: (value: string) => unknown
+} = {
+  heading: (v) => v,
+  cites: parseNonNegativeInt,
+  days: parseNonNegativeInt,
+  lastReinforced: (v) => v,
+}
+
+function parseNonNegativeInt(value: string): number {
+  const trimmed = value.trim()
+  const num = Number(trimmed)
+  if (!Number.isInteger(num) || String(num) !== trimmed || num < 0) {
+    throw new Error(`expected non-negative integer, got '${value}'`)
+  }
+  return num
+}
+
+export function renderShard(frontmatter: ShardFrontmatter, body: string): string {
+  const lines = ['---']
+  lines.push(`heading: ${frontmatter.heading}`)
+  lines.push(`cites: ${frontmatter.cites}`)
+  lines.push(`days: ${frontmatter.days}`)
+  lines.push(`lastReinforced: ${frontmatter.lastReinforced}`)
+  if (frontmatter.tags !== undefined) {
+    if (frontmatter.tags.length === 0) {
+      lines.push('tags: []')
+    } else {
+      lines.push(`tags: [${frontmatter.tags.join(', ')}]`)
+    }
+  }
+  lines.push('---')
+  lines.push(body)
+  return lines.join('\n')
+}
+
+export function updateFrontmatter(text: string, patch: Partial<ShardFrontmatter>): string {
+  const { frontmatter, body } = parseShard(text)
+  const updated: ShardFrontmatter = { ...frontmatter, ...patch }
+  if ('tags' in patch && patch.tags === undefined) {
+    delete (updated as Record<string, unknown>).tags
+  }
+  return renderShard(updated, body)
+}

package/src/bundled-plugins/memory/index.ts

@@ -1,6 +1,6 @@
 import { existsSync } from 'node:fs'
-import { access, constants as fsConstants, mkdir, readdir, stat, writeFile } from 'node:fs/promises'
-import { dirname, join } from 'node:path'
+import { access, constants as fsConstants, mkdir, readdir, stat, unlink, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
 
 import { CronExpressionParser } from 'cron-parser'
 import { z } from 'zod'
@@ -9,8 +9,13 @@ import type { SessionOrigin } from '@/agent/session-origin'
 import { definePlugin } from '@/plugin'
 
 import { createDreamingSubagent, type DreamingPayload } from './dreaming'
+import { buildInjectionPlan, DEFAULT_INJECTION_BUDGET_BYTES, MIN_INJECTION_BUDGET_BYTES } from './injection-plan'
+import { loadAllShards } from './load-shards'
 import { createMemoryLoggerSubagent, type MemoryLoggerPayload } from './memory-logger'
-import { runMigration } from './migration'
+import { createMemoryRetrievalSubagent, type MemoryRetrievalPayload } from './memory-retrieval'
+import { runMigration, runShardingMigration } from './migration'
+import { preShardBackupPath, streamFilePath, streamsDir, topicsDir } from './paths'
+import { memorySearchTool } from './search-tool'
 
 const DEFAULT_IDLE_MS = 60_000
 const DEFAULT_BUFFER_BYTES = 500_000
@@ -75,6 +80,7 @@ const memoryConfigSchema = z
         message: `memory.bufferBytes must be 0 (disabled) or >= ${MIN_BUFFER_BYTES}`,
       })
       .default(DEFAULT_BUFFER_BYTES),
+    injectionBudgetBytes: z.number().int().min(MIN_INJECTION_BUDGET_BYTES).default(DEFAULT_INJECTION_BUDGET_BYTES),
     // Test seam: per-spawn ceiling for memory-logger. Operators have no
     // reason to tune this; it exists so the wedge-recovery test can fire
     // the timeout in milliseconds instead of the production 50s. Kept
@@ -82,7 +88,12 @@ const memoryConfigSchema = z
     spawnTimeoutMs: z.number().int().min(1).default(SPAWN_TIMEOUT_MS),
     dreaming: dreamingConfigSchema.optional(),
   })
-  .default({ idleMs: DEFAULT_IDLE_MS, bufferBytes: DEFAULT_BUFFER_BYTES, spawnTimeoutMs: SPAWN_TIMEOUT_MS })
+  .default({
+    idleMs: DEFAULT_IDLE_MS,
+    bufferBytes: DEFAULT_BUFFER_BYTES,
+    injectionBudgetBytes: DEFAULT_INJECTION_BUDGET_BYTES,
+    spawnTimeoutMs: SPAWN_TIMEOUT_MS,
+  })
 
 export default definePlugin({
   configSchema: memoryConfigSchema,
@@ -100,6 +111,18 @@ export default definePlugin({
       ctx.logger.info(`[memory] migrated ${migrationResult.migrated.length} daily stream(s) to JSONL`)
     }
 
+    const shardingResult = await runShardingMigration({
+      agentDir: ctx.agentDir,
+      logger: ctx.logger,
+    })
+    if (shardingResult.migrated) {
+      ctx.logger.info(
+        `[memory] sharded ${shardingResult.topicCount} topics + ${shardingResult.streamCount} streams (pre-shard backup at memory/MEMORY.md.pre-shard.bak)`,
+      )
+    } else if (shardingResult.error !== undefined) {
+      ctx.logger.warn(`[memory] sharding migration aborted: ${shardingResult.error}`)
+    }
+
     const idleTimers = new Map<string, ReturnType<typeof setTimeout>>()
     const lastIdleEvent = new Map<string, { parentTranscriptPath: string | undefined; origin?: SessionOrigin }>()
     const bytesAtLastRun = new Map<string, number>()
@@ -164,6 +187,30 @@ export default definePlugin({
       return currentSize - baseline >= bufferBytes
     }
 
+    const runMemoryRetrieval = async (event: {
+      sessionId: string
+      agentDir: string
+      userPrompt: string
+      origin?: SessionOrigin
+    }): Promise<void> => {
+      const shards = await loadAllShards(event.agentDir)
+      const plan = buildInjectionPlan(shards, { budgetBytes: ctx.config.injectionBudgetBytes })
+      if (plan.mode === 'direct') return
+
+      const cacheFilePath = join(event.agentDir, 'memory', '.retrieval-cache', `${event.sessionId}.md`)
+      const payload: MemoryRetrievalPayload = {
+        parentSessionId: event.sessionId,
+        agentDir: event.agentDir,
+        recentPrompt: event.userPrompt,
+        cacheFilePath,
+        ...(event.origin !== undefined ? { origin: event.origin } : {}),
+      }
+      await ctx.spawnSubagent('memory-retrieval', payload, {
+        parentSessionId: event.sessionId,
+        ...(event.origin !== undefined ? { spawnedByOrigin: event.origin } : {}),
+      })
+    }
+
     // Subagents are constructed at boot here (rather than imported as constants)
     // so their lifecycle logs route through the plugin logger and pick up the
     // `[plugin:memory]` prefix. Without this, they would write directly to
@@ -177,8 +224,12 @@ export default definePlugin({
     return {
       subagents: {
         'memory-logger': createMemoryLoggerSubagent({ logger: subagentLogger }),
+        'memory-retrieval': createMemoryRetrievalSubagent({ logger: subagentLogger }),
         dreaming: createDreamingSubagent({ logger: subagentLogger }),
       },
+      tools: {
+        memory_search: memorySearchTool,
+      },
       cronJobs: {
         dreaming: {
           schedule: dreamingSchedule,
@@ -193,7 +244,7 @@ export default definePlugin({
         // directly, appended LAST in the system prompt). It does not run from a
         // plugin hook because positioning matters for cache-prefix stability:
         // the daily-stream file grows after every channel turn (memory-logger
-        // appends a fragment + watermark) and MEMORY.md changes on every dream.
+        // appends a fragment + watermark) and memory/topics/ changes on every dream.
         // A volatile region in the middle of the system prompt invalidates the
         // entire cacheable suffix below it on every session resurrection
         // (channel sessions evicted by idle GC, container restarts). Pinning
@@ -230,27 +281,91 @@ export default definePlugin({
             await fireMemoryLogger(sessionId, 'buffer-trip')
           }
         },
+        // memory-retrieval used to run from `session.prompt`, which fires
+        // during system-prompt assembly (createResourceLoader) and carries
+        // the ASSEMBLING SYSTEM PROMPT as `event.prompt` — not the user's
+        // message. The plugin was feeding that string into the subagent as
+        // `recentPrompt`, so the LLM keyword-mined TypeClaw's framing prose
+        // (`TypeClaw`, `subagent`, `AGENTS.md`, `systemPromptLeak`, etc.)
+        // and burned 15+ memory_search calls per session on terms the user
+        // never said. `session.turn.start` is the correct trigger: it fires
+        // before each `session.prompt(text)` call with the actual text the
+        // session is about to receive.
+        //
+        // The hook body is fully detached. `runSessionTurnStart` has no
+        // per-handler timeout (unlike session.prompt/idle/end), and the
+        // caller awaits it before `session.prompt(text)` runs — so an
+        // inline `await loadAllShards(...)` would gate every channel turn
+        // on N shard reads. Detaching mirrors PR #337's "fire-and-forget
+        // the slow work" pattern, pushed one level earlier to cover both
+        // the shard read AND the LLM spawn.
+        //
+        // Per-turn instead of per-session-creation means N spawns over the
+        // session's lifetime, but the subagent's own `inFlightKey` on
+        // `parentSessionId` (memory-retrieval.ts) coalesces overlapping
+        // fires: if turn N's retrieval is still running when turn N+1 fires,
+        // the second spawn is dropped with a warning, and the cache from
+        // turn N is still consumed by turn N+1 via the documented
+        // lag-by-one-prompt contract (load-memory.ts:appendRetrievalCache).
+        //
+        // Known limitation: a detached spawn can theoretically settle after
+        // `session.end` has unlinked the cache file, leaving a single ~5 KB
+        // file in memory/.retrieval-cache/ that never gets read. The next
+        // session.end that matches this sessionId would clean it up, but
+        // sessionIds are UUIDv7 so reuse is effectively never. Fix would
+        // serialize session.end behind the in-flight spawn, which
+        // re-introduces the cold-start blocking shape PR #337 fixed. The
+        // leaked file is bounded (one per disconnected session) and lives
+        // in a dir already marked transient.
+        //
+        // ctx.spawnSubagent IS reject-able in production: it wraps
+        // dispatchSpawnSubagent (src/run/index.ts) which calls invokeSubagent
+        // directly with no try/catch. SubagentConsumer's catch only protects
+        // stream-initiated spawns (target.kind === 'new-session'), not the
+        // direct ctx.spawnSubagent path the hooks use. Same for
+        // loadAllShards' fs errors. The .catch() on the void-discarded
+        // promise below is load-bearing — without it, every shard-read or
+        // handler failure (LLM provider error, payload validation throw)
+        // would surface as an unhandled rejection because nothing awaits
+        // the promise.
+        'session.turn.start': (event) => {
+          if (event.origin?.kind === 'subagent') return
+          void runMemoryRetrieval(event).catch((err) => {
+            ctx.logger.error(`memory-retrieval spawn failed: ${err instanceof Error ? err.message : String(err)}`)
+          })
+        },
         'session.end': async (event) => {
           if (event.origin?.kind === 'subagent') return
           cancelTimer(event.sessionId)
           await fireMemoryLogger(event.sessionId, 'session-end')
+          const cacheFilePath = join(ctx.agentDir, 'memory', '.retrieval-cache', `${event.sessionId}.md`)
+          try {
+            await unlink(cacheFilePath)
+          } catch (err) {
+            if (!isEnoent(err)) ctx.logger.warn(`[memory] failed to clean retrieval cache: ${err}`)
+          }
           lastIdleEvent.delete(event.sessionId)
           bytesAtLastRun.delete(event.sessionId)
         },
       },
       doctorChecks: {
         'dir-writable': {
-          description: 'memory/ exists and is writable',
+          description: 'memory/topics/ exists and is writable',
           run: async (dctx) => {
-            const dir = join(dctx.agentDir, 'memory')
+            const dir = topicsDir(dctx.agentDir)
             try {
               await access(dir, fsConstants.W_OK)
               return { status: 'ok', message: `${dir} writable` }
             } catch {
-              return {
-                status: 'error',
-                message: `${dir} is missing or not writable`,
-                fix: { description: 'Create memory/ in the agent folder or fix its permissions on the host.' },
+              try {
+                await mkdir(dir, { recursive: true })
+                return { status: 'ok', message: `created ${dir}` }
+              } catch {
+                return {
+                  status: 'error',
+                  message: `${dir} is missing and could not be created`,
+                  fix: { description: 'Create memory/topics/ in the agent folder or fix its permissions on the host.' },
+                }
               }
             }
           },
@@ -259,8 +374,8 @@ export default definePlugin({
           description: "today's daily stream file exists",
           run: async (dctx) => {
             const today = new Date().toISOString().slice(0, 10)
-            const rel = `memory/${today}.jsonl`
-            const abs = join(dctx.agentDir, rel)
+            const rel = join('memory', 'streams', `${today}.jsonl`)
+            const abs = streamFilePath(dctx.agentDir, today)
             if (existsSync(abs)) return { status: 'ok', message: `${rel} present` }
             return {
               status: 'warning',
@@ -268,7 +383,7 @@ export default definePlugin({
               fix: {
                 description: `Create empty ${rel} so memory-logger has a target.`,
                 apply: async () => {
-                  await mkdir(dirname(abs), { recursive: true })
+                  await mkdir(streamsDir(dctx.agentDir), { recursive: true })
                   await writeFile(abs, '', 'utf8')
                   return { summary: `created ${rel}`, changedPaths: [rel] }
                 },
@@ -277,17 +392,85 @@ export default definePlugin({
           },
         },
         'legacy-md-cleanup': {
-          description: 'Check for legacy .md daily stream files that should have been migrated to .jsonl',
+          description: 'Check for legacy .md daily stream files and un-migrated root MEMORY.md',
           run: async (dctx) => {
             const memoryDir = join(dctx.agentDir, 'memory')
+            // kept: pre-migration agents may still have a root MEMORY.md.
+            const rootMemoryPath = join(dctx.agentDir, 'MEMORY.md')
+            const hasRootMemory = existsSync(rootMemoryPath)
+            const hasTopicsDir = existsSync(topicsDir(dctx.agentDir))
+
             let files: string[]
             try {
               files = await readdir(memoryDir)
             } catch {
-              return { status: 'ok', message: 'memory/ does not exist yet' }
+              if (!hasRootMemory) return { status: 'ok', message: 'memory/ does not exist yet' }
+              return {
+                status: 'warning',
+                message: 'root MEMORY.md present but not sharded',
+                fix: {
+                  description: 'Run sharding migration to convert root MEMORY.md to topic shards',
+                  apply: async (fixCtx) => {
+                    const result = await runShardingMigration({ agentDir: fixCtx.agentDir, logger: fixCtx.logger })
+                    return {
+                      summary: result.migrated
+                        ? `sharded ${result.topicCount} topic(s) and ${result.streamCount} stream(s)`
+                        : `sharding migration did not run${result.error ? `: ${result.error}` : ''}`,
+                      changedPaths: result.migrated
+                        ? [
+                            join('memory', 'topics'),
+                            join('memory', 'streams'),
+                            join('memory', 'MEMORY.md.pre-shard.bak'),
+                          ]
+                        : [],
+                    }
+                  },
+                },
+              }
             }
 
             const mdFiles = files.filter((f) => /^\d{4}-\d{2}-\d{2}\.md$/.test(f))
+
+            if (hasRootMemory) {
+              if (!hasTopicsDir) {
+                const mdMsg = mdFiles.length > 0 ? `; also ${mdFiles.length} legacy .md daily stream(s)` : ''
+                return {
+                  status: 'warning',
+                  message: `root MEMORY.md present but not sharded${mdMsg}`,
+                  fix: {
+                    description: 'Run sharding migration to convert root MEMORY.md to topic shards',
+                    apply: async (fixCtx) => {
+                      const result = await runShardingMigration({ agentDir: fixCtx.agentDir, logger: fixCtx.logger })
+                      return {
+                        summary: result.migrated
+                          ? `sharded ${result.topicCount} topic(s) and ${result.streamCount} stream(s)`
+                          : `sharding migration did not run${result.error ? `: ${result.error}` : ''}`,
+                        changedPaths: result.migrated
+                          ? [
+                              join('memory', 'topics'),
+                              join('memory', 'streams'),
+                              join('memory', 'MEMORY.md.pre-shard.bak'),
+                            ]
+                          : [],
+                      }
+                    },
+                  },
+                }
+              }
+              const mdMsg = mdFiles.length > 0 ? `; also ${mdFiles.length} legacy .md daily stream(s)` : ''
+              return {
+                status: 'warning',
+                message: `orphaned root MEMORY.md after sharding migration${mdMsg}`,
+                fix: {
+                  description: 'Delete the orphaned root MEMORY.md file',
+                  apply: async () => {
+                    await unlink(rootMemoryPath)
+                    return { summary: 'deleted orphaned root MEMORY.md', changedPaths: ['MEMORY.md'] }
+                  },
+                },
+              }
+            }
+
             if (mdFiles.length === 0) return { status: 'ok', message: 'no legacy .md daily streams found' }
 
             const caseA: string[] = []
@@ -335,6 +518,39 @@ export default definePlugin({
             return { status: 'ok', message: 'no legacy .md daily streams found' }
           },
         },
+        'pre-shard-backup-age': {
+          description: 'Warn when pre-shard backup is older than 30 days',
+          run: async (dctx) => {
+            const backupPath = preShardBackupPath(dctx.agentDir)
+            let s
+            try {
+              s = await stat(backupPath)
+            } catch {
+              return { status: 'ok', message: 'no pre-shard backup present' }
+            }
+            const ageDays = (Date.now() - s.mtimeMs) / 86_400_000
+            if (ageDays > 30) {
+              return {
+                status: 'warning',
+                message: `pre-shard backup is ${Math.round(ageDays)} days old; safe to delete if migration is verified`,
+                fix: {
+                  description: 'Delete the pre-shard backup file',
+                  apply: async () => {
+                    await unlink(backupPath)
+                    return {
+                      summary: 'deleted pre-shard backup',
+                      changedPaths: [join('memory', 'MEMORY.md.pre-shard.bak')],
+                    }
+                  },
+                },
+              }
+            }
+            return {
+              status: 'ok',
+              message: `pre-shard backup is ${Math.round(ageDays)} days old (under 30-day threshold)`,
+            }
+          },
+        },
       },
     }
   },
@@ -360,3 +576,7 @@ async function raceSpawn(work: Promise<void>, ms: number): Promise<void> {
     if (timer !== null) clearTimeout(timer)
   }
 }
+
+function isEnoent(err: unknown): boolean {
+  return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOENT'
+}

package/src/bundled-plugins/memory/injection-plan.ts

@@ -0,0 +1,15 @@
+import type { TopicShard } from './load-shards'
+
+export const DEFAULT_INJECTION_BUDGET_BYTES = 16 * 1024
+export const MIN_INJECTION_BUDGET_BYTES = 4 * 1024
+
+export type InjectionPlan =
+  | { mode: 'direct'; shards: TopicShard[] }
+  | { mode: 'index'; shards: TopicShard[]; budget: number; totalBytes: number }
+
+export function buildInjectionPlan(shards: TopicShard[], options: { budgetBytes?: number } = {}): InjectionPlan {
+  const budget = options.budgetBytes ?? DEFAULT_INJECTION_BUDGET_BYTES
+  const totalBytes = shards.reduce((sum, shard) => sum + Buffer.byteLength(shard.body, 'utf8'), 0)
+  if (totalBytes <= budget) return { mode: 'direct', shards }
+  return { mode: 'index', shards, budget, totalBytes }
+}