typeclaw 0.1.4 → 0.1.6

package/src/agent/subagents.ts

@@ -6,6 +6,7 @@ import type { Stream, Unsubscribe } from '@/stream'
 
 import { type AgentSession, createSession } from './index'
 import type { SessionOrigin } from './session-origin'
+import type { ToolResultBudget } from './tool-result-budget'
 
 type AgentSessionTools = NonNullable<Parameters<typeof createSession>[0]>['tools']
 
@@ -19,10 +20,15 @@ export type RunSession = (override?: { userPrompt?: string }) => Promise<void>
 
 export type Subagent<P = unknown> = {
   systemPrompt: string
+  // Model profile this subagent prefers. Resolved against `config.models` at
+  // session construction. Unknown profile names fall back to `default` with
+  // a warning. See `Subagent` in `@/plugin/types` for the full contract.
+  profile?: string
   tools?: AgentSessionTools
   customTools?: ToolDefinition[]
   payloadSchema?: z.ZodType<P>
   handler?: (ctx: SubagentContext<P>, runSession: RunSession) => Promise<void>
+  toolResultBudget?: ToolResultBudget
 }
 
 export type SubagentRegistry = Readonly<Record<string, Subagent<any>>>
@@ -62,6 +68,8 @@ export type CreateSessionForSubagentResult = {
 export type CreateSessionForSubagentOptions = {
   name?: string
   parentSessionId?: string
+  spawnedByRole?: string
+  spawnedByOrigin?: SessionOrigin
 }
 export type CreateSessionForSubagent = (
   subagent: Subagent<any>,
@@ -75,9 +83,13 @@ export const defaultCreateSessionForSubagent: CreateSessionForSubagent = (subage
       kind: 'subagent',
       subagent: options?.name ?? '<unknown>',
       parentSessionId: options?.parentSessionId ?? '<unknown>',
+      ...(options?.spawnedByRole !== undefined ? { spawnedByRole: options.spawnedByRole } : {}),
+      ...(options?.spawnedByOrigin !== undefined ? { spawnedByOrigin: options.spawnedByOrigin } : {}),
     },
     ...(subagent.tools ? { tools: subagent.tools } : {}),
     customTools: subagent.customTools ?? [],
+    ...(subagent.profile !== undefined ? { profile: subagent.profile } : {}),
+    ...(subagent.toolResultBudget !== undefined ? { toolResultBudget: subagent.toolResultBudget } : {}),
   })
 
 type NormalizedSubagentSession = {
@@ -120,6 +132,8 @@ export type InvokeSubagentOptions = {
   userPrompt: string
   payload?: unknown
   parentSessionId?: string
+  spawnedByRole?: string
+  spawnedByOrigin?: SessionOrigin
 }
 
 export async function invokeSubagent(name: string, options: InvokeSubagentOptions): Promise<void> {
@@ -131,6 +145,8 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
   const sessionOptions: CreateSessionForSubagentOptions = {
     name,
     ...(options.parentSessionId !== undefined ? { parentSessionId: options.parentSessionId } : {}),
+    ...(options.spawnedByRole !== undefined ? { spawnedByRole: options.spawnedByRole } : {}),
+    ...(options.spawnedByOrigin !== undefined ? { spawnedByOrigin: options.spawnedByOrigin } : {}),
   }
 
   const runSession: RunSession = async (override) => {
@@ -157,11 +173,12 @@ export async function invokeSubagent(name: string, options: InvokeSubagentOption
           sessionId,
           parentTranscriptPath: getTranscriptPath?.(),
           idleMs: 0,
+          ...(origin !== undefined ? { origin } : {}),
         })
       }
     } finally {
       if (hooks && sessionId !== undefined) {
-        await hooks.runSessionEnd({ sessionId })
+        await hooks.runSessionEnd({ sessionId, ...(origin !== undefined ? { origin } : {}) })
       }
       session.dispose()
       await dispose()
@@ -215,6 +232,40 @@ const consoleLogger: SubagentConsumerLogger = {
   error: (m) => console.error(m),
 }
 
+function parseSpawnedByOriginJson(
+  raw: string | undefined,
+  logger: SubagentConsumerLogger,
+  subagentName: string,
+): SessionOrigin | undefined {
+  if (raw === undefined) return undefined
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(raw)
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err)
+    logger.warn(`[subagent] ${subagentName}: ignoring malformed spawnedByOriginJson on stream target: ${message}`)
+    return undefined
+  }
+  // Shape-validate the decoded value so a malformed sender (or a future
+  // bug in cron consumer's encode side) cannot poison the subagent's
+  // origin with arbitrary shapes. The check is narrow: object with a
+  // `kind` field whose value is one of the SessionOrigin discriminator
+  // strings. Permission resolution treats unknown shapes as guest, so
+  // failing closed here matches the rest of the system.
+  if (!isSessionOriginShape(parsed)) {
+    logger.warn(`[subagent] ${subagentName}: ignoring spawnedByOriginJson with unrecognized SessionOrigin shape`)
+    return undefined
+  }
+  return parsed
+}
+
+const SESSION_ORIGIN_KINDS = new Set(['tui', 'cron', 'channel', 'subagent'])
+function isSessionOriginShape(value: unknown): value is SessionOrigin {
+  if (value === null || typeof value !== 'object') return false
+  const kind = (value as { kind?: unknown }).kind
+  return typeof kind === 'string' && SESSION_ORIGIN_KINDS.has(kind)
+}
+
 export function createSubagentConsumer({
   stream,
   getRegistry,
@@ -234,6 +285,8 @@ export function createSubagentConsumer({
           kind: 'new-session'
           subagent: string
           parentSessionId?: string
+          spawnedByRole?: string
+          spawnedByOriginJson?: string
         }
         const name = target.subagent
         const registry = getRegistry()
@@ -248,6 +301,7 @@ export function createSubagentConsumer({
         }
         inFlight.add(key)
         try {
+          const spawnedByOrigin = parseSpawnedByOriginJson(target.spawnedByOriginJson, logger, name)
           await invokeSubagent(name, {
             registry,
             ...(createSessionForSubagent !== undefined ? { createSessionForSubagent } : {}),
@@ -255,6 +309,8 @@ export function createSubagentConsumer({
             userPrompt: '',
             payload: msg.payload,
             ...(target.parentSessionId !== undefined ? { parentSessionId: target.parentSessionId } : {}),
+            ...(target.spawnedByRole !== undefined ? { spawnedByRole: target.spawnedByRole } : {}),
+            ...(spawnedByOrigin !== undefined ? { spawnedByOrigin } : {}),
           })
         } catch (err) {
           const message = err instanceof Error ? err.message : String(err)

package/src/agent/system-prompt.ts

@@ -20,7 +20,7 @@ These files are not decoration. They shape how you behave. If a task reveals som
 
 - **\`workspace/\`** — the directory where you are free to create files: drafts, notes, downloads, scratch work, generated artifacts, temporary outputs. **Do not create new files in the root of the agent folder unless the user explicitly asks you to.** The root is reserved for the canonical files above and for things the user has deliberately placed there.
 - **\`sessions/\`** — transcripts of past conversations (\`<sessionid>.jsonl\`). Read-only for you in spirit; the runtime manages these.
-- **\`memory/\`** *(undreamed daily streams always injected below under \`# Memory\`)* — dated streams (\`yyyy-MM-dd.md\`) of fragments captured by the memory-logger between sessions. Newest day is closest to the current task. Once dreaming consolidates a day's stream into MEMORY.md, the runtime stops injecting it.
+- **\`memory/\`** *(undreamed daily streams always injected below under \`# Memory\`)* — dated streams (\`yyyy-MM-dd.jsonl\`) of fragments captured by the memory-logger between sessions. Newest day is closest to the current task. Once dreaming consolidates a day's stream into MEMORY.md, the runtime stops injecting it.
 - **\`memory/skills/\`** — *muscle memory*. Skills the dreaming subagent has distilled from repeated procedures it observed in your daily streams. Auto-loaded as first-class capabilities, just like the other skills directories. **You do not write here directly** — dreaming owns it. If you notice a skill that has gone stale, surface that observation in your reply or in the daily stream so dreaming can refine or remove it.
 - **\`.agents/skills/\`** — skills the user installed for you. Treat these as first-class capabilities.
 

package/src/agent/tool-result-budget.ts

@@ -0,0 +1,121 @@
+import type { AgentTool } from '@mariozechner/pi-agent-core'
+import type { ToolDefinition } from '@mariozechner/pi-coding-agent'
+import type { TSchema } from '@sinclair/typebox'
+
+// Subagents that read large files (memory-logger and dreaming each read parent
+// session transcripts that can run hundreds of KB) are vulnerable to a class
+// of bug where a single tool malfunction — a broken `find_entry`, a missing
+// watermark, a transcript that no longer contains the watermark id — causes
+// the agent to fall back to scanning the file in 50KB chunks. Every chunk
+// stays in the subagent's conversation history and gets re-sent to the model
+// on every turn until the subagent stops, so a 1MB transcript can balloon a
+// memory-logger run from ~10K input tokens to several hundred thousand.
+//
+// The budget here is a fail-safe ceiling on the total bytes of tool-result
+// text a subagent run is allowed to accumulate from a chosen set of tools.
+// Once exhausted, subsequent calls to those tools short-circuit with a
+// constant-size message that tells the agent to advance the watermark to the
+// latest entry and exit. The budget is per-run (one BudgetState per session)
+// and tracked only for the named tools; tools like `append` (which write,
+// not read) are unaffected.
+
+export type ToolResultBudget = {
+  maxTotalBytes: number
+  toolNames: readonly string[]
+  exhaustedMessage?: (used: number, max: number) => string
+}
+
+export type BudgetState = {
+  used: number
+  exhausted: boolean
+}
+
+export function createBudgetState(): BudgetState {
+  return { used: 0, exhausted: false }
+}
+
+function defaultExhaustedMessage(used: number, max: number): string {
+  const usedKb = Math.round(used / 1024)
+  const maxKb = Math.round(max / 1024)
+  return [
+    `[tool-result budget exhausted: used ${usedKb}KB of ${maxKb}KB this run]`,
+    '',
+    'Stop reading. This session has consumed its byte budget across calls to',
+    'this tool. Do not call this tool again. Stop and exit; future runs will',
+    'continue from wherever your normal end-of-run bookkeeping left off.',
+  ].join('\n')
+}
+
+function bytesOfContent(content: { type: string; text?: string }[] | undefined): number {
+  if (!content) return 0
+  let total = 0
+  for (const part of content) {
+    if (part.type === 'text' && typeof part.text === 'string') {
+      total += Buffer.byteLength(part.text, 'utf8')
+    }
+  }
+  return total
+}
+
+function buildExhaustedResult(budget: ToolResultBudget, state: BudgetState) {
+  const text = (budget.exhaustedMessage ?? defaultExhaustedMessage)(state.used, budget.maxTotalBytes)
+  return {
+    content: [{ type: 'text' as const, text }],
+    details: { budgetExhausted: true, used: state.used, max: budget.maxTotalBytes },
+  }
+}
+
+// Wraps an AgentTool's execute so that returned text content is counted against
+// `state` and the tool short-circuits once `budget.maxTotalBytes` is exceeded.
+// Tools whose name is not in `budget.toolNames` are returned unchanged so the
+// caller can pass an entire `tools` array through and only the tracked tools
+// are affected. The original tool object is preserved by spreading; only
+// `execute` is replaced.
+export function wrapAgentToolWithBudget<TParams extends TSchema, TDetails = unknown>(
+  tool: AgentTool<TParams, TDetails>,
+  budget: ToolResultBudget,
+  state: BudgetState,
+): AgentTool<TParams, TDetails> {
+  if (!budget.toolNames.includes(tool.name)) return tool
+  const originalExecute = tool.execute.bind(tool)
+  return {
+    ...tool,
+    async execute(toolCallId, args, signal, onUpdate) {
+      if (state.exhausted) {
+        return buildExhaustedResult(budget, state) as Awaited<ReturnType<typeof originalExecute>>
+      }
+      const result = await originalExecute(toolCallId, args, signal, onUpdate)
+      state.used += bytesOfContent(result.content as { type: string; text?: string }[] | undefined)
+      if (state.used >= budget.maxTotalBytes) {
+        state.exhausted = true
+      }
+      return result
+    },
+  }
+}
+
+// Same wrapper for ToolDefinition (the customTools surface). Identical
+// semantics; ToolDefinition's execute has an extra `onUpdate` callback and a
+// `ctx` argument that we forward verbatim.
+export function wrapToolDefinitionWithBudget<TParams extends TSchema, TDetails = unknown, TState = unknown>(
+  tool: ToolDefinition<TParams, TDetails, TState>,
+  budget: ToolResultBudget,
+  state: BudgetState,
+): ToolDefinition<TParams, TDetails, TState> {
+  if (!budget.toolNames.includes(tool.name)) return tool
+  const originalExecute = tool.execute.bind(tool)
+  return {
+    ...tool,
+    async execute(toolCallId, args, signal, onUpdate, ctx) {
+      if (state.exhausted) {
+        return buildExhaustedResult(budget, state) as Awaited<ReturnType<typeof originalExecute>>
+      }
+      const result = await originalExecute(toolCallId, args, signal, onUpdate, ctx)
+      state.used += bytesOfContent(result.content as { type: string; text?: string }[] | undefined)
+      if (state.used >= budget.maxTotalBytes) {
+        state.exhausted = true
+      }
+      return result
+    },
+  }
+}

package/src/bundled-plugins/backup/index.ts

@@ -1,6 +1,6 @@
 import { z } from 'zod'
 
-import { definePlugin, type Subagent } from '@/plugin'
+import { definePlugin, type PluginContext, type SpawnSubagentOptions, type Subagent } from '@/plugin'
 
 import { COMMIT_TIMEOUT_MS, makeDefaultGitSpawn, NETWORK_TIMEOUT_MS, runBackup, type BackupResult } from './runner'
 import {
@@ -78,10 +78,19 @@ export default definePlugin({
       if (activeTurns.size > 0) return
       inFlight = true
       try {
-        await ctx.spawnSubagent(SUBAGENT_BACKUP_RUNNER, {
-          agentDir: ctx.agentDir,
-          pushToOrigin,
-        } satisfies RunnerPayload)
+        await ctx.spawnSubagent(
+          SUBAGENT_BACKUP_RUNNER,
+          {
+            agentDir: ctx.agentDir,
+            pushToOrigin,
+          } satisfies RunnerPayload,
+          // The backup runner is a system-level operation that commits +
+          // pushes on the operator's behalf. It runs after every idle
+          // window regardless of which session caused activity, so it has
+          // no single user session to inherit from. Mark it as TUI-equivalent
+          // so it resolves to `owner` and can use git push, etc.
+          { spawnedByOrigin: { kind: 'tui', sessionId: 'backup-runner' } },
+        )
       } catch (err) {
         ctx.logger.error(`backup runner spawn failed: ${err instanceof Error ? err.message : String(err)}`)
       } finally {
@@ -152,12 +161,18 @@ async function runBackupOnce(
   ctx: {
     agentDir: string
     logger: { info: (m: string) => void; warn: (m: string) => void }
-    spawnSubagent: (name: string, payload?: unknown) => Promise<void>
+    spawnSubagent: PluginContext['spawnSubagent']
   },
 ): Promise<BackupResult> {
   const messagePath = messageFilePath(payload.agentDir)
   await ensureMessageDir(messagePath)
   await cleanupMessageFile(messagePath)
+  // Inherit the backup-runner's owner privileges for the message-picking
+  // and diagnose subagents it spawns. Same rationale as the runner itself
+  // — these are system-level operations on the operator's behalf.
+  const inheritOwner: SpawnSubagentOptions = {
+    spawnedByOrigin: { kind: 'tui', sessionId: 'backup-runner' },
+  }
 
   const result = await runBackup(
     { cwd: payload.agentDir, pushToOrigin: payload.pushToOrigin },
@@ -172,7 +187,7 @@ async function runBackupOnce(
           outputPath: messagePath,
         }
         try {
-          await ctx.spawnSubagent(SUBAGENT_COMMIT_MESSAGE, messagePayload)
+          await ctx.spawnSubagent(SUBAGENT_COMMIT_MESSAGE, messagePayload, inheritOwner)
         } catch (err) {
           ctx.logger.warn(
             `${SUBAGENT_COMMIT_MESSAGE} subagent failed, using fallback: ${err instanceof Error ? err.message : String(err)}`,
@@ -191,7 +206,7 @@ async function runBackupOnce(
           stdout: input.stdout,
         }
         try {
-          await ctx.spawnSubagent(SUBAGENT_DIAGNOSE, diagPayload)
+          await ctx.spawnSubagent(SUBAGENT_DIAGNOSE, diagPayload, inheritOwner)
         } catch (err) {
           ctx.logger.warn(`${SUBAGENT_DIAGNOSE} subagent failed: ${err instanceof Error ? err.message : String(err)}`)
         }

package/src/bundled-plugins/backup/runner.ts

@@ -84,6 +84,28 @@ export async function runBackup(options: BackupRunnerOptions, deps: BackupRunner
     diffstat: diffstat.stdout.slice(0, 4096),
   })
 
+  // `pickCommitMessage` may spawn a subagent (the backup plugin's
+  // `backup-message`) whose session JSONL lands under `sessions/` after we
+  // already staged. Without this second pass that file would sit dirty in
+  // the worktree until the NEXT backup cycle, which would then commit it
+  // and create another orphan via the same path — a steady-state of
+  // one-cycle-behind churn. Re-status, filter to `sessions/` additions
+  // only (don't accidentally stage user work that arrived during the
+  // window), and force-add anything new.
+  const reStatus = await deps.gitSpawn(['status', '--porcelain=v1', '--untracked-files=all'], {
+    cwd,
+    timeoutMs: COMMIT_TIMEOUT_MS,
+  })
+  if (reStatus.exitCode === 0) {
+    const lateForce = filterForceAdd(parsePorcelain(reStatus.stdout)).filter((p) => existsSync(join(cwd, p)))
+    if (lateForce.length > 0) {
+      const lateAdd = await deps.gitSpawn(['add', '-f', '--', ...lateForce], { cwd, timeoutMs: COMMIT_TIMEOUT_MS })
+      if (lateAdd.exitCode !== 0) {
+        return { ok: false, kind: 'commit-failed', reason: `git add -f (post-message) failed: ${shortErr(lateAdd)}` }
+      }
+    }
+  }
+
   const safeMessage = sanitizeCommitMessage(message)
   const commit = await deps.gitSpawn(['commit', '-m', safeMessage], { cwd, timeoutMs: COMMIT_TIMEOUT_MS })
   if (commit.exitCode !== 0)

package/src/bundled-plugins/memory/README.md

@@ -1,6 +1,6 @@
 # typeclaw-plugin-memory
 
-The bundled memory plugin. Owns `MEMORY.md` (long-term memory) and `memory/yyyy-MM-dd.md` (daily streams) plus the two subagents that write them: `memory-logger` and `dreaming`.
+The bundled memory plugin. Owns `MEMORY.md` (long-term memory) and `memory/yyyy-MM-dd.jsonl` (daily streams) plus the two subagents that write them: `memory-logger` and `dreaming`.
 
 This plugin is **auto-loaded** by every TypeClaw agent. There is no `plugins[]` entry to add and no opt-out. To configure it, add a `memory` block to `typeclaw.json`.
 
@@ -27,19 +27,22 @@ All fields are **restart-required** — the plugin reads them once at boot.
 
 ## What it contributes
 
-| Kind     | Name                       | Notes                                                                                                                                                                                                                                                                             |
-| -------- | -------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Subagent | `memory-logger`            | Reads a parent transcript past a watermark and appends fragments to `memory/<today>.md`. Coalesced per `agentDir`; the plugin chains spawn calls onto a per-agent Promise so two concurrent channel sessions never race on the same daily stream file.                            |
-| Subagent | `dreaming`                 | Reads `MEMORY.md` plus undreamed daily-stream tails, rewrites `MEMORY.md`, optionally writes muscle-memory skills under `memory/skills/<name>/SKILL.md`, advances the per-day watermark, and `git commit -m Dream` the result. Coalesced per `agentDir`.                          |
-| Cron job | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.                                                                                                                                                                                               |
-| Hook     | `session.prompt`           | Appends the rendered memory section (`# Memory`, `MEMORY.md`, undreamed stream tails) to `event.prompt`.                                                                                                                                                                          |
-| Hook     | `session.idle`             | Per-session debouncer with size-based ceiling. Resets a `setTimeout(idleMs)` on every event; on fire, calls `ctx.spawnSubagent('memory-logger', ...)`. Also `fs.stat`s the transcript on every event and spawns immediately when growth since the last run reaches `bufferBytes`. |
-| Hook     | `session.end`              | Cancels the debounce timer and immediately spawns `memory-logger` (so the final transcript is captured even when the user disconnects right away).                                                                                                                                |
+| Kind     | Name                       | Notes                                                                                                                                                                                                                                                                                                                                                |
+| -------- | -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Subagent | `memory-logger`            | Reads a parent transcript past a watermark and appends fragments to `memory/<today>.jsonl`. Coalesced per `agentDir`; the plugin chains spawn calls onto a per-agent Promise so two concurrent channel sessions never race on the same daily stream file.                                                                                            |
+| Subagent | `dreaming`                 | Reads `MEMORY.md` plus undreamed daily-stream tails, rewrites `MEMORY.md`, optionally writes muscle-memory skills under `memory/skills/<name>/SKILL.md`, advances the per-day watermark, and commits the result with a summary message (`dream: <summary> <emoji>`, e.g. `dream: 3 fragments + new skill 'pr-review' 🔮`). Coalesced per `agentDir`. |
+| Cron job | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.                                                                                                                                                                                                                                                                  |
+| Hook     | `session.idle`             | Per-session debouncer with size-based ceiling. Resets a `setTimeout(idleMs)` on every event; on fire, calls `ctx.spawnSubagent('memory-logger', ...)`. Also `fs.stat`s the transcript on every event and spawns immediately when growth since the last run reaches `bufferBytes`.                                                                    |
+| Hook     | `session.end`              | Cancels the debounce timer and immediately spawns `memory-logger` (so the final transcript is captured even when the user disconnects right away).                                                                                                                                                                                                   |
+
+## Memory injection
+
+The rendered `# Memory` section (MEMORY.md + undreamed daily-stream tails) is injected into every session's system prompt by core (`src/agent/index.ts` `createResourceLoader` → `loadMemory`), **not** by a plugin hook. It is appended as the last block of the system prompt, after `gitNudge`, so the most-volatile content (daily streams that grow after every memory-logger fire) sits at the bottom of the cache-suffix region. This way a memory change only invalidates the memory section itself rather than everything downstream of it.
 
 ## Files on disk
 
 - **`MEMORY.md`** — long-term memory. Created by the dreaming subagent on first run if absent. Force-committed by the runtime; `skip-worktree` flag is set so the human's `git status` stays clean.
-- **`memory/yyyy-MM-dd.md`** — daily fragment streams. Appended to by `memory-logger`. Created on demand. Gitignored at the agent's level but force-committed alongside `MEMORY.md` after each dreaming run.
+- **`memory/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`, lossy-preserving one-shot migration from older `.md` streams. Appended to by `memory-logger`. Created on demand. Gitignored at the agent's level but force-committed alongside `MEMORY.md` after each dreaming run.
 - **`memory/skills/<name>/SKILL.md`** — _muscle memory_. Skills the dreaming subagent distills from repeated procedures it sees in daily streams. Auto-discovered as first-class skills by `createResourceLoader`, and force-committed under the same `memory/` snapshot path as the daily streams. Written or refined with the standard `write` / `edit` tools; the bundled guard plugin enforces the exact `memory/skills/<name>/SKILL.md` path shape, single-segment kebab/snake-case names, matching frontmatter, and symlink/path-traversal safety. There is no runtime skill-delete tool; outright deletion of muscle-memory skills remains a user decision.
 - **`memory/.dreaming-state.json`** — per-day watermarks (line counts already consolidated into `MEMORY.md`). Plain JSON; on malformed input the plugin fails open with empty state.
 

package/src/bundled-plugins/memory/append-tool.ts

@@ -1,84 +1,110 @@
-import { appendFile, mkdir, open, readFile, stat } from 'node:fs/promises'
-import { dirname } from 'node:path'
+import { randomUUID } from 'node:crypto'
+import { mkdir } from 'node:fs/promises'
+import { dirname, join } from 'node:path'
 
 import { z } from 'zod'
 
 import { defineTool } from '@/plugin'
+import { formatLocalDate } from '@/shared'
 
-import { fragmentContentHash, parseFragments } from './fragment-parser'
+import { fragmentContentHash } from './fragment-parser'
 import { detectSecrets } from './secret-detector'
-
-const NEWLINE_BYTE = 0x0a
+import type { FragmentEvent, WatermarkEvent } from './stream-events'
+import { appendEvents, readEvents } from './stream-io'
 
 export const appendTool = defineTool({
   description:
-    'Append content to a file. Creates the file (and any missing parent directories) if needed. Never truncates or overwrites existing content. If the file is non-empty and does not already end in a newline, a single newline is inserted before the appended content so consecutive appends do not run together. Refuses to write content that contains recognizable credential patterns (API keys, tokens, private keys); record the variable name and how it was discovered, never the value. Refuses to append a fragment whose topic+body already exists in the file (case-by-case; topics legitimately repeat across days, but byte-equivalent fragments within the same daily stream are duplicates by design).',
+    "Append a memory fragment to today's JSONL daily stream and advance the watermark. The runtime serializes your call into a JSON line and chooses the filename — do not emit raw JSON and do not pass a path. `topic`/`body` are the fragment's substance; `source` is the parent session id; `entry` is the transcript-entry-id this fragment anchors to; `latestEntryId` is the latest transcript-entry-id you evaluated in this run (advances the watermark, may equal `entry` or be later). Refuses content with recognized credential patterns and refuses byte-equivalent topic+body within the same daily stream.",
   parameters: z.object({
-    path: z.string().describe('Path to the file to append to (relative or absolute).'),
-    content: z.string().describe('Content to append, exactly as given.'),
+    topic: z.string().min(1),
+    body: z.string().min(1),
+    source: z.string().min(1),
+    entry: z.string().min(1),
+    latestEntryId: z.string().min(1),
   }),
-  async execute({ path, content }) {
-    const secrets = detectSecrets(content)
-    if (secrets.length > 0) {
-      const ruleNames = [...new Set(secrets.map((s) => s.rule))].join(', ')
+  async execute({ topic, body, source, entry, latestEntryId }, ctx) {
+    const streamPath = dailyStreamPath(ctx.agentDir)
+    assertNoSecrets(`${topic}\n${body}`)
+
+    const hash = fragmentContentHash({ topic, body })
+    const events = await readEvents(streamPath)
+    const duplicate = events
+      .filter((event) => event.type === 'fragment')
+      .find((event) => fragmentContentHash(event) === hash)
+    if (duplicate !== undefined) {
       throw new Error(
-        `Refusing to append: content contains a recognized credential pattern (${ruleNames}). ` +
-          `Memory fragments must never quote secret values verbatim. Record the env var name and how it ` +
-          `was discovered, not the value itself.`,
+        `Refusing to append: fragment "${duplicate.topic}" already exists in ${streamPath} with byte-equivalent content. ` +
+          `The dreaming subagent will see the existing fragment; do not write it again. If the new occurrence ` +
+          `is genuinely informative, write a fragment that says so explicitly rather than restating the original.`,
       )
     }
-    const incomingFragments = parseFragments(content)
-    if (incomingFragments.length > 0) {
-      const existingHashes = await readExistingFragmentHashes(path)
-      const duplicates = incomingFragments.filter((f) => existingHashes.has(fragmentContentHash(f)))
-      if (duplicates.length > 0) {
-        const topics = duplicates.map((d) => `"${d.topic}"`).join(', ')
-        throw new Error(
-          `Refusing to append: ${duplicates.length} fragment${duplicates.length === 1 ? '' : 's'} (${topics}) ` +
-            `already exist in ${path} with byte-equivalent content. The dreaming subagent will see the existing ` +
-            `fragment; do not write it again. If the new occurrence is genuinely informative (e.g. a recurrence ` +
-            `that establishes a pattern), write a fragment that says so explicitly rather than restating the ` +
-            `original.`,
-        )
-      }
+
+    const fragment: FragmentEvent = {
+      type: 'fragment',
+      id: randomUUID(),
+      ts: new Date().toISOString(),
+      source,
+      entry,
+      topic,
+      body,
+    }
+    const watermark: WatermarkEvent = {
+      type: 'watermark',
+      id: randomUUID(),
+      ts: new Date().toISOString(),
+      source,
+      entry: latestEntryId,
+    }
+
+    await mkdir(dirname(streamPath), { recursive: true })
+    await appendEvents(streamPath, [fragment, watermark])
+
+    return {
+      content: [{ type: 'text' as const, text: `Appended memory fragment and watermark to ${streamPath}` }],
+      details: { path: streamPath, fragmentId: fragment.id, watermarkId: watermark.id },
+    }
+  },
+})
+
+export const advanceWatermarkTool = defineTool({
+  description:
+    'Advance the daily-stream watermark without writing a fragment. Use this when you evaluated transcript entries this run but decided none warranted a fragment — still call this once so the next run does not re-read the same prefix. The runtime writes the watermark line and chooses the filename.',
+  parameters: z.object({
+    source: z.string().min(1),
+    latestEntryId: z.string().min(1),
+  }),
+  async execute({ source, latestEntryId }, ctx) {
+    const streamPath = dailyStreamPath(ctx.agentDir)
+    const watermark: WatermarkEvent = {
+      type: 'watermark',
+      id: randomUUID(),
+      ts: new Date().toISOString(),
+      source,
+      entry: latestEntryId,
     }
-    await mkdir(dirname(path), { recursive: true })
-    const prefix = (await needsLeadingNewline(path)) ? '\n' : ''
-    await appendFile(path, prefix + content, 'utf-8')
-    const bytesAppended = prefix.length + content.length
+
+    await mkdir(dirname(streamPath), { recursive: true })
+    await appendEvents(streamPath, [watermark])
+
     return {
-      content: [{ type: 'text' as const, text: `Appended ${bytesAppended} bytes to ${path}` }],
-      details: { path, bytesAppended, leadingNewlineInserted: prefix.length > 0 },
+      content: [{ type: 'text' as const, text: `Advanced memory watermark in ${streamPath}` }],
+      details: { path: streamPath, watermarkId: watermark.id },
     }
   },
 })
 
-async function readExistingFragmentHashes(path: string): Promise<Set<string>> {
-  let content: string
-  try {
-    content = await readFile(path, 'utf8')
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return new Set()
-    throw err
-  }
-  return new Set(parseFragments(content).map((f) => fragmentContentHash(f)))
+function dailyStreamPath(agentDir: string): string {
+  return join(agentDir, 'memory', `${formatLocalDate()}.jsonl`)
 }
 
-async function needsLeadingNewline(path: string): Promise<boolean> {
-  let info: Awaited<ReturnType<typeof stat>>
-  try {
-    info = await stat(path)
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false
-    throw err
-  }
-  if (info.size === 0) return false
-  const fh = await open(path, 'r')
-  try {
-    const buf = Buffer.alloc(1)
-    await fh.read(buf, 0, 1, info.size - 1)
-    return buf[0] !== NEWLINE_BYTE
-  } finally {
-    await fh.close()
-  }
+function assertNoSecrets(content: string): void {
+  const secrets = detectSecrets(content)
+  if (secrets.length === 0) return
+
+  const ruleNames = [...new Set(secrets.map((s) => s.rule))].join(', ')
+  throw new Error(
+    `Refusing to append: content contains a recognized credential pattern (${ruleNames}). ` +
+      `Memory fragments must never quote secret values verbatim. Record the env var name and how it ` +
+      `was discovered, not the value itself.`,
+  )
 }