typeclaw 0.1.4 → 0.1.6

package/src/bundled-plugins/memory/memory-logger.ts

@@ -6,8 +6,9 @@ import type { SessionOrigin } from '@/agent/session-origin'
 import { type Subagent, readTool } from '@/plugin'
 import { formatLocalDate } from '@/shared'
 
-import { appendTool } from './append-tool'
-import { readWatermark } from './watermark'
+import { appendTool, advanceWatermarkTool } from './append-tool'
+import { findEntryTool } from './find-entry-tool'
+import { readLatestWatermark } from './watermark'
 
 export const memoryLoggerPayloadSchema = z.object({
   parentSessionId: z.string().min(1),
@@ -16,6 +17,39 @@ export const memoryLoggerPayloadSchema = z.object({
   origin: z.custom<SessionOrigin>().optional(),
 })
 
+// Recovery message for the read-budget short-circuit. The watermark contract
+// in MEMORY_LOGGER_SYSTEM_PROMPT requires advancing to the latest evaluated
+// entry on every run, but once read is short-circuited the subagent cannot keep
+// scanning to pick a "latest evaluated entry id". `find_entry` and `append` are not
+// budgeted, so the recovery is: call find_entry on the transcript to learn
+// `totalLines` without re-reading content, then advance the watermark to any
+// entry id the subagent already saw earlier in the run. When zero
+// transcript content has been read (budget consumed entirely on MEMORY.md or
+// the stream file), no advancement is possible and the run should exit
+// silently — that is the explicit second branch below. Both branches are
+// safer than the prior generic "advance to the latest id you have seen"
+// hint, which was self-contradictory in the zero-content case.
+export function memoryLoggerExhaustedMessage(used: number, max: number): string {
+  const usedKb = Math.round(used / 1024)
+  const maxKb = Math.round(max / 1024)
+  return [
+    `[read budget exhausted: used ${usedKb}KB of ${maxKb}KB this run]`,
+    '',
+    'Stop reading. The session has consumed its byte budget across read calls.',
+    'Do not call `read` again — every subsequent call will return this same notice.',
+    '',
+    'Recovery (in order):',
+    '1. If you already saw at least one transcript entry id in earlier read output,',
+    '   either call `append` with `latestEntryId=<that id>` for a real fragment, or',
+    '   call the watermark-advance tool with `{ source, latestEntryId: <that id> }`, then exit.',
+    '2. If you saw NO transcript entries (the budget was consumed on MEMORY.md and',
+    '   the daily stream file before you reached the transcript), exit immediately',
+    '   WITHOUT writing a watermark. The next run will retry from the same point.',
+    '',
+    'Do not invent or reuse a watermark id. Do not call `read` again.',
+  ].join('\n')
+}
+
 export type MemoryLoggerPayload = z.infer<typeof memoryLoggerPayloadSchema>
 
 export function isMemoryLoggerPayload(value: unknown): value is MemoryLoggerPayload {
@@ -28,7 +62,21 @@ Your job is to read a session transcript and capture, as fragments, everything m
 
 A separate \`dreaming\` subagent runs later. It consolidates your fragments into long-term memory, dedupes, drops near-duplicates, resolves contradictions, and decides what generalizes. **You are the additive layer; dreaming is the filter.** This division of labor is the whole point: capture broadly here, and let dreaming throw away what doesn't last.
 
-You have exactly two tools: \`read\` and \`append\`. You cannot run shell commands, overwrite files, or edit existing content.
+You have exactly four tools: \`read\`, \`find_entry\`, \`append\`, and the watermark-advance tool. You cannot run shell commands, overwrite files, or edit existing content.
+
+# Reading the transcript past the watermark
+
+Session transcripts are JSONL files where each line is an entry with an \`id\` field. They are often large (hundreds of KB). The \`read\` tool truncates output to 50 KB or 2000 lines, whichever comes first, and tells you the line range it returned plus the offset to continue. If you start \`read\` at \`offset=1\` on a 500 KB transcript, the first call returns roughly the first 10% of the file, the next call (\`offset=<next>\`) returns the following slice, and so on. Scrolling through a long prefix that you've already consolidated past is wasted tokens.
+
+**Always use \`find_entry\` before \`read\` when a watermark is set.** It scans the JSONL file for the line whose own \`id\` field equals a given entry id and returns the line number, the total line count, and the offset to pass to \`read\` so you resume immediately after the watermark. It matches \`"id":"<entryId>"\` exactly, so \`parentId\` references to the same id do not confuse it. It returns a "not found" string (no throw) when the watermark id is not in the file — that can happen if a parent session was compacted; treat it as "start from offset=1" or, if the transcript is huge and obviously unrelated, write the watermark forward and skip the run.
+
+Typical flow with a watermark:
+
+1. \`find_entry(path=<transcript>, entryId=<watermark>)\` → returns \`line=N, totalLines=T, offset=N+1\`.
+2. \`read(path=<transcript>, offset=N+1)\` → returns the chunk starting AT the first unread entry. Repeat with the next offset until the read tool's continuation notice stops appearing.
+3. As you read, track the most recent \`id\` you see. That is your new watermark value — pass it as \`latestEntryId\` on the final \`append\` call, or to the watermark-advance tool when there are zero fragments.
+
+Never write the same watermark id you were given as input. If the transcript has no new entries past the watermark, evaluate the entries you can see, then advance the watermark to the latest \`id\` in the transcript (which is on line \`totalLines\` from \`find_entry\`'s reply). The whole point of the watermark is to move forward each run.
 
 # Capture philosophy: when in doubt, capture
 
@@ -81,7 +129,7 @@ The \`append\` tool will refuse content that contains a recognizable credential
 
 # Read existing memory first
 
-Before reading the transcript, read \`MEMORY.md\` and the current \`memory/yyyy-MM-dd.md\` stream file. You need that context for three reasons:
+Before reading the transcript, read \`MEMORY.md\` and the current \`memory/yyyy-MM-dd.jsonl\` stream file. You need that context for three reasons:
 
 - **Notice contradictions.** If the transcript supersedes existing memory, write a fragment that names the prior memory and supersedes it.
 - **Notice violations.** If existing memory contains a commitment the agent just broke, that's a high-value fragment.
@@ -93,17 +141,10 @@ The \`append\` tool refuses byte-equivalent fragments within the same daily stre
 
 # Fragment format
 
-Each fragment is an HTML comment marker followed by a topic heading and a body:
+Call \`append\` with \`{topic, body, source, entry, latestEntryId}\`. The runtime serializes your call into a JSON line in the daily stream — you never write raw JSON. \`source\` is the parent session id from the user message. \`entry\` is the specific transcript-entry-id this fragment anchors to. \`latestEntryId\` is the latest transcript-entry-id you evaluated in this run; it advances the watermark and may equal \`entry\` or be later.
 
-\`\`\`
-<!-- fragment source=<sessionId> entry=<entryId> -->
-## <topic>
-<body — see below>
-\`\`\`
-
-- \`source\` is the parent session id from the user message.
 - \`entry\` is the stable id of the **specific** transcript entry that anchors this fragment's evidence. Each fragment carries its own entry id — do not stamp every fragment with the same "latest evaluated" id. The provenance is per-fragment.
-- \`<topic>\` is a short noun phrase naming what the fragment is about.
+- \`topic\` is a short noun phrase naming what the fragment is about.
 
 The body is the substance of the fragment. The form is flexible, but every body must satisfy two requirements:
 
@@ -131,21 +172,17 @@ A fragment doesn't need to articulate how a future agent will use it. If the imp
 
 **One topic per fragment.** If you have two unrelated things to say, write two fragments. Don't pile multiple stable facts into a single body.
 
-Separate fragments with a blank line.
-
 # Watermark contract
 
-The watermark is a separate concern from per-fragment provenance. After all fragments (or zero of them), append exactly one trailing watermark marker that records the latest transcript entry id you considered. This marker is what prevents you from re-reading the same transcript prefix on the next run.
+Every \`append\` call advances the watermark via the \`latestEntryId\` field. You no longer emit a separate watermark marker. Ensure the FINAL \`append\` call's \`latestEntryId\` is the latest transcript-entry-id you read this run. The watermark is what prevents you from re-reading the same transcript prefix on the next run.
 
-\`\`\`
-<!-- watermark source=<sessionId> entry=<latestEntryId> -->
-\`\`\`
+- \`latestEntryId\` is the latest transcript entry you evaluated, **regardless of which entries actually anchored fragments**. You may have evaluated 50 entries and written 2 fragments anchored to entries 5 and 23; the final \`latestEntryId\` is still the latest of the 50.
+- When you write multiple fragments, every \`append\` call may carry the same latest value if you already know it, but the final call must carry the farthest evaluated id.
+- Never reuse the watermark trick of stamping a fragment's \`entry\` with the latest evaluated entry — fragments carry per-evidence provenance, and \`latestEntryId\` carries progress.
 
-- The watermark's \`entry=\` is the latest transcript entry you evaluated, **regardless of which entries actually anchored fragments**. You may have evaluated 50 entries and written 2 fragments anchored to entries 5 and 23; the watermark is still the latest of the 50.
-- The watermark must always be the **last** marker in your appended output, after any fragments.
-- Write exactly one watermark per run, never more.
+# Zero-fragments path
 
-Never exit without a new watermark marker. Never reuse the watermark trick of stamping a fragment's \`entry=\` with the latest evaluated entry — fragments carry per-evidence provenance, and the watermark is its own marker.
+When you evaluated the transcript but found nothing worth a fragment, call the watermark-advance tool with \`{source, latestEntryId}\` so the next run does not re-read the same prefix. Do not call \`append\` with fake content just to move the watermark.
 
 # Stopping
 
@@ -171,9 +208,9 @@ function buildInitialPrompt(payload: MemoryLoggerPayload, streamFile: string, wa
     '',
     "Per-fragment provenance: each fragment's `entry=` is the specific transcript entry that anchors that fragment's evidence — not the latest entry you evaluated. Two fragments anchored to two different entries get two different `entry=` values. Do not stamp every fragment with the same id.",
     '',
-    'Watermark: regardless of how many fragments you wrote (zero or more), append exactly one trailing watermark marker `<!-- watermark source=' +
+    'Watermark: every `append` call must include the `latestEntryId` argument. Ensure the final `append` call uses the latest transcript entry you evaluated, regardless of whether it anchored a fragment. If you evaluated transcript entries but found zero fragments, call the watermark-advance tool with `{ source: "' +
       payload.parentSessionId +
-      ' entry=<latestEntryId> -->` as the last line of your appended output. `<latestEntryId>` is the latest transcript entry you evaluated, regardless of whether it anchored a fragment. Never exit without writing this marker.',
+      '", latestEntryId: "<latestEntryId>" }` instead of writing a fake fragment.',
   )
   return lines.join('\n')
 }
@@ -229,16 +266,22 @@ export function createMemoryLoggerSubagent(
   return {
     systemPrompt: MEMORY_LOGGER_SYSTEM_PROMPT,
     tools: [readTool],
-    customTools: [appendTool],
+    customTools: [findEntryTool, appendTool, advanceWatermarkTool],
     payloadSchema: memoryLoggerPayloadSchema,
     inFlightKey: (payload) => payload.agentDir,
+    toolResultBudget: {
+      maxTotalBytes: 256 * 1024,
+      toolNames: ['read'],
+      exhaustedMessage: memoryLoggerExhaustedMessage,
+    },
     handler: async (ctx, runSession) => {
       const today = formatLocalDate()
-      const streamFile = join(ctx.payload.agentDir, 'memory', `${today}.md`)
-      const watermark = readWatermark(streamFile, ctx.payload.parentSessionId)
+      const memoryDir = join(ctx.payload.agentDir, 'memory')
+      const streamFile = join(memoryDir, `${today}.jsonl`)
+      const watermark = await readLatestWatermark(memoryDir, ctx.payload.parentSessionId)
       const start = Date.now()
       logger.info(
-        `[memory-logger] ${ctx.payload.parentSessionId} start stream=${today}.md watermark=${watermark ?? 'none'}`,
+        `[memory-logger] ${ctx.payload.parentSessionId} start stream=${today}.jsonl watermark=${watermark ?? 'none'}`,
       )
       try {
         await runSession({ userPrompt: buildInitialPrompt(ctx.payload, streamFile, watermark) })

package/src/bundled-plugins/memory/migration.ts

@@ -0,0 +1,276 @@
+import { randomUUID } from 'node:crypto'
+import { existsSync } from 'node:fs'
+import { readdir, readFile, unlink } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import { loadDreamingState, saveDreamingState, setDreamedLines } from './dreaming-state'
+import { type StreamEvent, streamEventSchema } from './stream-events'
+import { writeEventsAtomic as defaultWriteEventsAtomic } from './stream-io'
+
+export type MigrationResult = {
+  migrated: string[]
+  skipped: string[]
+  legacyProseCount: number
+  fragmentCount: number
+  watermarkCount: number
+}
+
+export type MigrationLogger = {
+  info: (message: string) => void
+  warn: (message: string) => void
+  error: (message: string) => void
+}
+
+export type MigrationGit = {
+  spawn?: (args: string[], options: { cwd: string }) => Promise<{ exitCode: number; stdout: string; stderr: string }>
+}
+
+export type RunMigrationOptions = {
+  agentDir: string
+  logger: MigrationLogger
+  git?: MigrationGit
+  writeEventsAtomic?: (path: string, events: readonly StreamEvent[]) => Promise<void>
+}
+
+const DAILY_MD_NAME = /^(\d{4}-\d{2}-\d{2})\.md$/
+const DAILY_JSONL_NAME = /^(\d{4}-\d{2}-\d{2})\.jsonl$/
+const LEGACY_FRAGMENT_RE =
+  /<!-- fragment source=(\S+) entry=(\S+) -->\n## (.+)\n([\s\S]*?)(?=<!-- fragment |<!-- watermark |$)/g
+const LEGACY_WATERMARK_RE = /<!-- watermark source=(\S+) entry=(\S+) -->/g
+
+export async function runMigration(options: RunMigrationOptions): Promise<MigrationResult> {
+  const memoryDir = join(options.agentDir, 'memory')
+  const result: MigrationResult = {
+    migrated: [],
+    skipped: [],
+    legacyProseCount: 0,
+    fragmentCount: 0,
+    watermarkCount: 0,
+  }
+
+  let entries: string[]
+  try {
+    entries = await readdir(memoryDir)
+  } catch {
+    return result
+  }
+
+  const dates = collectDailyDates(entries)
+  for (const date of dates) {
+    const mdPath = join(memoryDir, `${date}.md`)
+    const jsonlPath = join(memoryDir, `${date}.jsonl`)
+    const hasMd = existsSync(mdPath)
+    const hasJsonl = existsSync(jsonlPath)
+
+    if (hasJsonl && !hasMd) {
+      result.skipped.push(date)
+      continue
+    }
+
+    if (hasJsonl && hasMd) {
+      options.logger.warn(`[memory:migration] ${date}: skipped because both .md and .jsonl exist`)
+      result.skipped.push(date)
+      continue
+    }
+
+    if (!hasMd) continue
+
+    const content = await readFile(mdPath, 'utf8')
+    const events = parseLegacyMarkdown(content)
+    const invalid = findInvalidEvent(events)
+    if (invalid !== null) {
+      options.logger.error(
+        `[memory:migration] ${date}.md: event ${invalid.index + 1} failed validation: ${invalid.reason}`,
+      )
+      result.skipped.push(date)
+      continue
+    }
+
+    const counts = countEvents(events)
+    try {
+      await (options.writeEventsAtomic ?? defaultWriteEventsAtomic)(jsonlPath, events)
+    } catch (err) {
+      options.logger.error(`[memory:migration] ${date}.md: failed to write JSONL: ${describeError(err)}`)
+      result.skipped.push(date)
+      continue
+    }
+    await unlink(mdPath)
+
+    result.fragmentCount += counts.fragmentCount
+    result.watermarkCount += counts.watermarkCount
+    result.legacyProseCount += counts.legacyProseCount
+    result.migrated.push(date)
+    options.logger.info(
+      `[memory:migration] ${date}: ${counts.fragmentCount} fragments, ${counts.watermarkCount} watermarks, ${counts.legacyProseCount} legacy_prose regions`,
+    )
+  }
+
+  if (result.migrated.length > 0) {
+    await resetDreamingWatermarks(options.agentDir, result.migrated)
+    await commitMigration(options.agentDir, result.migrated, options.logger, options.git)
+  }
+
+  return result
+}
+
+function collectDailyDates(entries: readonly string[]): string[] {
+  const dates = new Set<string>()
+  for (const entry of entries) {
+    const md = DAILY_MD_NAME.exec(entry)
+    if (md?.[1] !== undefined) dates.add(md[1])
+    const jsonl = DAILY_JSONL_NAME.exec(entry)
+    if (jsonl?.[1] !== undefined) dates.add(jsonl[1])
+  }
+  return Array.from(dates).sort()
+}
+
+function parseLegacyMarkdown(content: string): StreamEvent[] {
+  const events: StreamEvent[] = []
+  let cursor = 0
+
+  while (cursor < content.length) {
+    const fragment = nextMatch(LEGACY_FRAGMENT_RE, content, cursor)
+    const watermark = nextMatch(LEGACY_WATERMARK_RE, content, cursor)
+    const next = earliest(fragment, watermark)
+    if (next === null) break
+
+    addLegacyProse(events, content.slice(cursor, next.match.index))
+    if (next.kind === 'fragment') {
+      events.push({
+        type: 'fragment',
+        id: randomUUID(),
+        ts: new Date().toISOString(),
+        source: next.match[1]!,
+        entry: next.match[2]!,
+        topic: next.match[3]!,
+        body: next.match[4]!,
+      })
+    } else {
+      events.push({
+        type: 'watermark',
+        id: randomUUID(),
+        ts: new Date().toISOString(),
+        source: next.match[1]!,
+        entry: next.match[2]!,
+      })
+    }
+    cursor = next.match.index + next.match[0].length
+  }
+
+  addLegacyProse(events, content.slice(cursor))
+  return events
+}
+
+function addLegacyProse(events: StreamEvent[], text: string): void {
+  if (text.trim() === '') return
+  events.push({ type: 'legacy_prose', ts: new Date().toISOString(), text, origin: 'migration' })
+}
+
+function nextMatch(regex: RegExp, content: string, cursor: number): RegExpExecArray | null {
+  regex.lastIndex = cursor
+  return regex.exec(content)
+}
+
+function earliest(
+  fragment: RegExpExecArray | null,
+  watermark: RegExpExecArray | null,
+): { kind: 'fragment' | 'watermark'; match: RegExpExecArray } | null {
+  if (fragment === null && watermark === null) return null
+  if (fragment === null) return { kind: 'watermark', match: watermark! }
+  if (watermark === null) return { kind: 'fragment', match: fragment }
+  return fragment.index <= watermark.index
+    ? { kind: 'fragment', match: fragment }
+    : { kind: 'watermark', match: watermark }
+}
+
+function findInvalidEvent(events: readonly StreamEvent[]): { index: number; reason: string } | null {
+  for (let i = 0; i < events.length; i++) {
+    const parsed = streamEventSchema.safeParse(events[i])
+    if (!parsed.success) {
+      return { index: i, reason: parsed.error.issues.map((issue) => issue.message).join('; ') }
+    }
+  }
+  return null
+}
+
+function countEvents(
+  events: readonly StreamEvent[],
+): Pick<MigrationResult, 'fragmentCount' | 'watermarkCount' | 'legacyProseCount'> {
+  let fragmentCount = 0
+  let watermarkCount = 0
+  let legacyProseCount = 0
+  for (const event of events) {
+    if (event.type === 'fragment') fragmentCount++
+    if (event.type === 'watermark') watermarkCount++
+    if (event.type === 'legacy_prose') legacyProseCount++
+  }
+  return { fragmentCount, watermarkCount, legacyProseCount }
+}
+
+async function resetDreamingWatermarks(agentDir: string, dates: readonly string[]): Promise<void> {
+  let state = await loadDreamingState(agentDir)
+  const ts = new Date().toISOString()
+  for (const date of dates) {
+    state = setDreamedLines(state, date, 0, ts)
+  }
+  await saveDreamingState(agentDir, state)
+}
+
+async function commitMigration(
+  agentDir: string,
+  dates: readonly string[],
+  logger: MigrationLogger,
+  git: MigrationGit | undefined,
+): Promise<void> {
+  const spawn = git?.spawn ?? spawnGit
+  const inside = await spawn(['rev-parse', '--is-inside-work-tree'], { cwd: agentDir })
+  if (inside.exitCode !== 0) {
+    logger.info('[memory:migration] not in a git repo; skipping git commit')
+    return
+  }
+
+  const jsonlPaths = dates.map((date) => `memory/${date}.jsonl`)
+  const addJsonl = await spawn(['add', '--', ...jsonlPaths], { cwd: agentDir })
+  if (addJsonl.exitCode !== 0) {
+    logger.warn(`[memory:migration] git add failed: ${addJsonl.stderr || addJsonl.stdout}`.trim())
+    return
+  }
+
+  for (const date of dates) {
+    const mdPath = `memory/${date}.md`
+    const tracked = await spawn(['ls-files', '--error-unmatch', '--', mdPath], { cwd: agentDir })
+    if (tracked.exitCode !== 0) continue
+    const addDeletedMd = await spawn(['add', '-u', '--', mdPath], { cwd: agentDir })
+    if (addDeletedMd.exitCode !== 0) {
+      logger.warn(`[memory:migration] git add failed: ${addDeletedMd.stderr || addDeletedMd.stdout}`.trim())
+      return
+    }
+  }
+
+  const commit = await spawn(
+    ['commit', '-m', `memory: migrate ${dates.length} daily stream(s) to JSONL`, '--no-edit'],
+    {
+      cwd: agentDir,
+    },
+  )
+  if (commit.exitCode !== 0) {
+    logger.warn(`[memory:migration] git commit failed: ${commit.stderr || commit.stdout}`.trim())
+  }
+}
+
+async function spawnGit(
+  args: string[],
+  options: { cwd: string },
+): Promise<{ exitCode: number; stdout: string; stderr: string }> {
+  const proc = Bun.spawn({ cmd: ['git', ...args], cwd: options.cwd, stdout: 'pipe', stderr: 'pipe' })
+  const [stdout, stderr, exitCode] = await Promise.all([
+    new Response(proc.stdout).text(),
+    new Response(proc.stderr).text(),
+    proc.exited,
+  ])
+  return { exitCode, stdout, stderr }
+}
+
+function describeError(err: unknown): string {
+  return err instanceof Error ? err.message : String(err)
+}

package/src/bundled-plugins/memory/stream-events.ts

@@ -0,0 +1,55 @@
+import { z } from 'zod'
+
+export const fragmentEventSchema = z
+  .object({
+    type: z.literal('fragment'),
+    id: z.string().min(1),
+    ts: z.string().datetime(),
+    source: z.string(),
+    entry: z.string(),
+    topic: z.string(),
+    body: z.string(),
+  })
+  .passthrough()
+
+export const watermarkEventSchema = z
+  .object({
+    type: z.literal('watermark'),
+    id: z.string().min(1),
+    ts: z.string().datetime(),
+    source: z.string(),
+    entry: z.string(),
+  })
+  .passthrough()
+
+export const legacyProseEventSchema = z
+  .object({
+    type: z.literal('legacy_prose'),
+    ts: z.string().datetime(),
+    text: z.string(),
+    origin: z.literal('migration'),
+  })
+  .passthrough()
+
+export const streamEventSchema = z.discriminatedUnion('type', [
+  fragmentEventSchema,
+  watermarkEventSchema,
+  legacyProseEventSchema,
+])
+
+export type FragmentEvent = z.infer<typeof fragmentEventSchema>
+export type WatermarkEvent = z.infer<typeof watermarkEventSchema>
+export type LegacyProseEvent = z.infer<typeof legacyProseEventSchema>
+export type StreamEvent = FragmentEvent | WatermarkEvent | LegacyProseEvent
+
+export function parseEventLine(line: string): StreamEvent | null {
+  let raw: unknown
+  try {
+    raw = JSON.parse(line)
+  } catch {
+    return null
+  }
+  const result = streamEventSchema.safeParse(raw)
+  if (!result.success) return null
+  return result.data
+}

package/src/bundled-plugins/memory/stream-io.ts

@@ -0,0 +1,63 @@
+import { readFile, appendFile, writeFile, rename } from 'node:fs/promises'
+
+import { parseEventLine, type StreamEvent } from './stream-events'
+
+export async function readEvents(path: string): Promise<StreamEvent[]> {
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf-8')
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === 'ENOENT') return []
+    throw e
+  }
+
+  const lines = raw.split('\n')
+  const events: StreamEvent[] = []
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i]!
+    if (line === '') continue
+    const event = parseEventLine(line)
+    if (event === null) {
+      console.warn(`[stream-io] ${path}: skipping malformed line ${i + 1}`)
+      continue
+    }
+    events.push(event)
+  }
+
+  return events
+}
+
+export async function appendEvents(path: string, events: readonly StreamEvent[]): Promise<void> {
+  if (events.length === 0) return
+  const joined = events.map((e) => `${JSON.stringify(e)}\n`).join('')
+  await appendFile(path, joined, 'utf-8')
+}
+
+export async function writeEventsAtomic(path: string, events: readonly StreamEvent[]): Promise<void> {
+  const joined = events.map((e) => `${JSON.stringify(e)}\n`).join('')
+  const tmp = `${path}.tmp`
+  await writeFile(tmp, joined, 'utf-8')
+  await rename(tmp, path)
+}
+
+export async function countEvents(path: string): Promise<number> {
+  let raw: string
+  try {
+    raw = await readFile(path, 'utf-8')
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === 'ENOENT') return 0
+    throw e
+  }
+
+  const lines = raw.split('\n')
+  let count = 0
+
+  for (const line of lines) {
+    if (line === '') continue
+    const event = parseEventLine(line)
+    if (event !== null) count++
+  }
+
+  return count
+}

package/src/bundled-plugins/memory/watermark.ts

@@ -1,15 +1,55 @@
-import { existsSync, readFileSync } from 'node:fs'
+import { readdir } from 'node:fs/promises'
+import { join } from 'node:path'
 
-const WATERMARK_MARKER = /<!--\s*(?:fragment|watermark)\s+source=(\S+)\s+entry=(\S+)(?:\s+\S+=\S+)*\s*-->/g
+import { readEvents } from './stream-io'
 
-export function readWatermark(streamFilePath: string, parentSessionId: string): string | null {
-  if (!existsSync(streamFilePath)) return null
-  const content = readFileSync(streamFilePath, 'utf8')
+// Daily stream files are named `YYYY-MM-DD.jsonl` (see `formatLocalDate` in
+// `src/shared`). The cross-day lookup ignores any other file the user or a
+// plugin may have dropped into `memory/`.
+const DAILY_STREAM_NAME = /^\d{4}-\d{2}-\d{2}\.jsonl$/
+
+export async function readWatermarkFromFile(streamFilePath: string, parentSessionId: string): Promise<string | null> {
+  const events = await readEvents(streamFilePath)
 
   let lastEntryId: string | null = null
-  for (const match of content.matchAll(WATERMARK_MARKER)) {
-    const [, source, entry] = match
-    if (source === parentSessionId) lastEntryId = entry ?? null
+  for (const event of events) {
+    if ((event.type === 'fragment' || event.type === 'watermark') && event.source === parentSessionId) {
+      lastEntryId = event.entry
+    }
   }
   return lastEntryId
 }
+
+// Returns the latest watermark entry id for `parentSessionId` across all
+// `YYYY-MM-DD.jsonl` daily-stream files under `memoryDir`, walking newest-first
+// (by filename, which is equivalent to chronological order). Short-circuits
+// on the first file that contains a matching marker — for the common case
+// where memory-logger ran yesterday, this reads exactly one file.
+//
+// Why cross-day: channel sessions (Slack, Discord, KakaoTalk) routinely
+// survive the midnight rollover because the same human keeps the same
+// session alive across days. If `readWatermark` only looked at today's
+// stream file, every midnight would force a full transcript reread for
+// every long-lived session — burning ~135k input tokens per memory-logger
+// run on a 762KB transcript (observed on a real Discord agent: PR #207).
+//
+// The append target stays today's file; only the lookup crosses the day
+// boundary. This means yesterday's stream is treated as read-only history,
+// which it already is by construction (dreaming snapshots full days, never
+// touches in-progress days).
+export async function readLatestWatermark(memoryDir: string, parentSessionId: string): Promise<string | null> {
+  let entries: string[]
+  try {
+    entries = await readdir(memoryDir)
+  } catch {
+    return null
+  }
+  const dailyStreams = entries
+    .filter((name) => DAILY_STREAM_NAME.test(name))
+    .sort((a, b) => (a < b ? 1 : a > b ? -1 : 0))
+  for (const name of dailyStreams) {
+    const watermark = await readWatermarkFromFile(join(memoryDir, name), parentSessionId)
+    if (watermark !== null) return watermark
+  }
+  return null
+}