typeclaw 0.8.0 → 0.9.0

package/src/bundled-plugins/memory/migration.ts

@@ -1,10 +1,23 @@
 import { existsSync } from 'node:fs'
-import { readdir, readFile, unlink } from 'node:fs/promises'
+import { cp, mkdir, readdir, readFile, rename, rm, rmdir, unlink, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
 
+import { checkCitationSupersetAcrossShards, summarizeMissingCitations } from './citation-superset'
+import { normalizeCitation, parseCitations } from './citations'
 import { clearDreamedIds, loadDreamingState, saveDreamingState } from './dreaming-state'
+import { renderShard, type ShardFrontmatter } from './frontmatter'
+import {
+  migratingTmpDir,
+  PRE_SHARD_BACKUP_FILENAME,
+  preShardBackupPath,
+  streamFilePath,
+  streamsDir,
+  topicsDir,
+} from './paths'
+import { headingToSlug } from './slug'
 import { newEventId, type StreamEvent, streamEventSchema, timestampFromId } from './stream-events'
 import { writeEventsAtomic as defaultWriteEventsAtomic } from './stream-io'
+import { parseTopicsWithBodies } from './topics'
 
 export type MigrationResult = {
   migrated: string[]
@@ -31,12 +44,104 @@ export type RunMigrationOptions = {
   writeEventsAtomic?: (path: string, events: readonly StreamEvent[]) => Promise<void>
 }
 
+export type ShardingMigrationResult = {
+  migrated: boolean
+  topicCount: number
+  streamCount: number
+  legacy: MigrationResult
+  error?: string
+}
+
+export type RunShardingMigrationOptions = RunMigrationOptions & {
+  hooks?: {
+    onAfterStageTopics?: () => Promise<void> | void
+    onAfterStageStreams?: () => Promise<void> | void
+    onAfterStageBackup?: () => Promise<void> | void
+  }
+}
+
 const DAILY_MD_NAME = /^(\d{4}-\d{2}-\d{2})\.md$/
 const DAILY_JSONL_NAME = /^(\d{4}-\d{2}-\d{2})\.jsonl$/
 const LEGACY_FRAGMENT_RE =
   /<!-- fragment source=(\S+) entry=(\S+) -->\n## (.+)\n([\s\S]*?)(?=<!-- fragment |<!-- watermark |$)/g
 const LEGACY_WATERMARK_RE = /<!-- watermark source=(\S+) entry=(\S+) -->/g
 
+export async function runShardingMigration(options: RunShardingMigrationOptions): Promise<ShardingMigrationResult> {
+  await recoverShardingMigration(options.agentDir, options.logger)
+  const legacy = await runMigration(options)
+  const empty = (extra?: Partial<ShardingMigrationResult>): ShardingMigrationResult => ({
+    migrated: false,
+    topicCount: 0,
+    streamCount: 0,
+    legacy,
+    ...extra,
+  })
+
+  await recoverShardingOrphans(options.agentDir, options.logger)
+
+  if (existsSync(topicsDir(options.agentDir)) || !existsSync(rootMemoryPath(options.agentDir))) {
+    return empty()
+  }
+
+  const memoryDir = join(options.agentDir, 'memory')
+  const tmpDir = migratingTmpDir(options.agentDir)
+  await rm(tmpDir, { recursive: true, force: true })
+  await mkdir(join(tmpDir, 'topics'), { recursive: true })
+  await mkdir(join(tmpDir, 'streams'), { recursive: true })
+
+  const rootContent = await readFile(rootMemoryPath(options.agentDir), 'utf8')
+  const topics = parseTopicsWithBodies(rootContent)
+  if (topics.length === 0) {
+    await rm(tmpDir, { recursive: true, force: true })
+    options.logger.warn('[memory:migration] MEMORY.md has no topics; skipping sharding migration')
+    return empty()
+  }
+
+  const existingSlugs = new Set<string>()
+  const orderedSlugs: string[] = []
+  for (const topic of topics) {
+    const slug = headingToSlug(topic.heading, existingSlugs)
+    existingSlugs.add(slug)
+    orderedSlugs.push(slug)
+    const body = normalizeCitation(topic.body)
+    const frontmatter = frontmatterForTopic(topic.heading, body)
+    await writeFile(join(tmpDir, 'topics', `${slug}.md`), renderShard(frontmatter, body), 'utf8')
+  }
+  await options.hooks?.onAfterStageTopics?.()
+
+  const streamDates = await collectFlatJsonlDates(memoryDir)
+  for (const date of streamDates) {
+    await cp(join(memoryDir, `${date}.jsonl`), join(tmpDir, 'streams', `${date}.jsonl`))
+  }
+  await options.hooks?.onAfterStageStreams?.()
+
+  await cp(rootMemoryPath(options.agentDir), join(tmpDir, 'MEMORY.md.pre-shard.bak'))
+  await options.hooks?.onAfterStageBackup?.()
+
+  const newShardTexts = await readShardTexts(join(tmpDir, 'topics'))
+  const verdict = checkCitationSupersetAcrossShards(new Map([['MEMORY.md', rootContent]]), newShardTexts)
+  if (!verdict.ok) {
+    const error = `citation superset violation: ${summarizeMissingCitations(verdict.missing)}`
+    options.logger.error(`[memory:migration] ${error}`)
+    return empty({ error })
+  }
+
+  const finalized = await finalizeShardingMigration(options.agentDir, streamDates, options.logger)
+  if (!finalized.ok) return empty({ error: finalized.error })
+
+  await commitShardingMigration(
+    options.agentDir,
+    { slugs: orderedSlugs, streamDates, hadRootMemory: true },
+    options.logger,
+    options.git,
+  )
+
+  options.logger.info(
+    `[memory:migration] sharded MEMORY.md into ${topics.length} topic shard(s) and ${streamDates.length} stream file(s)`,
+  )
+  return { migrated: true, topicCount: topics.length, streamCount: streamDates.length, legacy }
+}
+
 export async function runMigration(options: RunMigrationOptions): Promise<MigrationResult> {
   const memoryDir = join(options.agentDir, 'memory')
   const result: MigrationResult = {
@@ -123,6 +228,129 @@ function collectDailyDates(entries: readonly string[]): string[] {
   return Array.from(dates).sort()
 }
 
+async function recoverShardingMigration(agentDir: string, logger: MigrationLogger): Promise<void> {
+  const tmpDir = migratingTmpDir(agentDir)
+  if (!existsSync(tmpDir)) return
+
+  const hasTopics = existsSync(topicsDir(agentDir))
+  await rm(tmpDir, { recursive: true, force: true })
+  logger.info(
+    hasTopics
+      ? '[memory:migration] removed leftover sharding tmpdir after completed migration'
+      : '[memory:migration] removed stale sharding tmpdir; retrying migration from originals',
+  )
+}
+
+async function recoverShardingOrphans(agentDir: string, logger: MigrationLogger): Promise<void> {
+  if (!existsSync(topicsDir(agentDir))) return
+
+  let cleaned = false
+  const memoryPath = rootMemoryPath(agentDir)
+  if (existsSync(memoryPath)) {
+    await unlink(memoryPath)
+    cleaned = true
+  }
+
+  const memoryDir = join(agentDir, 'memory')
+  const dates = await collectFlatJsonlDates(memoryDir)
+  for (const date of dates) {
+    if (!existsSync(streamFilePath(agentDir, date))) continue
+    await unlink(join(memoryDir, `${date}.jsonl`))
+    cleaned = true
+  }
+
+  if (cleaned) logger.info('[memory:migration] cleaned orphaned pre-shard memory files')
+}
+
+async function collectFlatJsonlDates(memoryDir: string): Promise<string[]> {
+  let entries: string[]
+  try {
+    entries = await readdir(memoryDir)
+  } catch {
+    return []
+  }
+  return entries
+    .map((entry) => DAILY_JSONL_NAME.exec(entry)?.[1])
+    .filter((date): date is string => date !== undefined)
+    .sort()
+}
+
+function frontmatterForTopic(heading: string, body: string): ShardFrontmatter {
+  const citations = parseCitations(body)
+  const dates = [...citations.keys()].sort()
+  let cites = 0
+  for (const ids of citations.values()) cites += ids.size
+
+  return {
+    heading,
+    cites,
+    days: dates.length,
+    lastReinforced: dates.at(-1) ?? todayDate(),
+  }
+}
+
+async function readShardTexts(dir: string): Promise<Map<string, string>> {
+  const entries = (await readdir(dir)).filter((entry) => entry.endsWith('.md')).sort()
+  const out = new Map<string, string>()
+  for (const entry of entries) {
+    out.set(entry, await readFile(join(dir, entry), 'utf8'))
+  }
+  return out
+}
+
+async function finalizeShardingMigration(
+  agentDir: string,
+  streamDates: readonly string[],
+  logger: MigrationLogger,
+): Promise<{ ok: true } | { ok: false; error: string }> {
+  const tmpDir = migratingTmpDir(agentDir)
+  const renames: Array<[string, string]> = [
+    [join(tmpDir, 'topics'), topicsDir(agentDir)],
+    [join(tmpDir, 'streams'), streamsDir(agentDir)],
+    [join(tmpDir, 'MEMORY.md.pre-shard.bak'), preShardBackupPath(agentDir)],
+  ]
+
+  for (const [from, to] of renames) {
+    try {
+      await rename(from, to)
+    } catch (err) {
+      const error = `failed to finalize sharding migration: ${describeError(err)}`
+      logger.error(`[memory:migration] ${error}`)
+      return { ok: false, error }
+    }
+  }
+
+  for (const date of streamDates) {
+    try {
+      await unlink(join(agentDir, 'memory', `${date}.jsonl`))
+    } catch (err) {
+      logger.warn(`[memory:migration] failed to remove flat stream ${date}.jsonl: ${describeError(err)}`)
+    }
+  }
+
+  try {
+    await unlink(rootMemoryPath(agentDir))
+  } catch (err) {
+    logger.warn(`[memory:migration] failed to remove root MEMORY.md: ${describeError(err)}`)
+  }
+
+  try {
+    await rmdir(tmpDir)
+  } catch (err) {
+    logger.warn(`[memory:migration] failed to remove sharding tmpdir: ${describeError(err)}`)
+  }
+
+  return { ok: true }
+}
+
+function rootMemoryPath(agentDir: string): string {
+  return join(agentDir, 'MEMORY.md')
+}
+
+function todayDate(): string {
+  return new Date().toISOString().slice(0, 10)
+}
+
 function parseLegacyMarkdown(content: string): StreamEvent[] {
   const events: StreamEvent[] = []
   let cursor = 0
@@ -259,6 +487,59 @@ async function commitMigration(
   }
 }
 
+async function commitShardingMigration(
+  agentDir: string,
+  details: { slugs: readonly string[]; streamDates: readonly string[]; hadRootMemory: boolean },
+  logger: MigrationLogger,
+  git: MigrationGit | undefined,
+): Promise<void> {
+  const spawn = git?.spawn ?? spawnGit
+  const inside = await spawn(['rev-parse', '--is-inside-work-tree'], { cwd: agentDir })
+  if (inside.exitCode !== 0) {
+    logger.info('[memory:migration] not in a git repo; skipping git commit')
+    return
+  }
+
+  const newPaths = [
+    ...details.slugs.map((slug) => `memory/topics/${slug}.md`),
+    ...details.streamDates.map((date) => `memory/streams/${date}.jsonl`),
+    `memory/${PRE_SHARD_BACKUP_FILENAME}`,
+  ]
+  const addNew = await spawn(['add', '--', ...newPaths], { cwd: agentDir })
+  if (addNew.exitCode !== 0) {
+    logger.warn(`[memory:migration] git add failed: ${addNew.stderr || addNew.stdout}`.trim())
+    return
+  }
+
+  const candidateDeletions = [...details.streamDates.map((date) => `memory/${date}.jsonl`)]
+  if (details.hadRootMemory) candidateDeletions.push('MEMORY.md')
+  const trackedDeletions: string[] = []
+  for (const path of candidateDeletions) {
+    const tracked = await spawn(['ls-files', '--error-unmatch', '--', path], { cwd: agentDir })
+    if (tracked.exitCode === 0) trackedDeletions.push(path)
+  }
+  if (trackedDeletions.length > 0) {
+    const addDeletions = await spawn(['add', '-u', '--', ...trackedDeletions], { cwd: agentDir })
+    if (addDeletions.exitCode !== 0) {
+      logger.warn(`[memory:migration] git add failed: ${addDeletions.stderr || addDeletions.stdout}`.trim())
+      return
+    }
+  }
+
+  const commitSharding = await spawn(
+    [
+      'commit',
+      '-m',
+      `memory: shard MEMORY.md into ${details.slugs.length} topic(s) and ${details.streamDates.length} daily stream(s)`,
+      '--no-edit',
+    ],
+    { cwd: agentDir },
+  )
+  if (commitSharding.exitCode !== 0) {
+    logger.warn(`[memory:migration] git commit failed: ${commitSharding.stderr || commitSharding.stdout}`.trim())
+  }
+}
+
 async function spawnGit(
   args: string[],
   options: { cwd: string },

package/src/bundled-plugins/memory/paths.ts

@@ -0,0 +1,42 @@
+import { join } from 'node:path'
+
+export { DREAMING_STATE_FILE } from './dreaming-state'
+
+export const MEMORY_DIR = 'memory'
+export const TOPICS_SUBDIR = 'topics'
+export const STREAMS_SUBDIR = 'streams'
+export const SKILLS_SUBDIR = 'skills'
+export const PRE_SHARD_BACKUP_FILENAME = 'MEMORY.md.pre-shard.bak'
+export const MIGRATING_TMPDIR = 'memory/.migrating'
+
+const STREAM_DATE_RE = /^\d{4}-\d{2}-\d{2}$/
+
+export function topicsDir(agentDir: string): string {
+  return join(agentDir, MEMORY_DIR, TOPICS_SUBDIR)
+}
+
+export function streamsDir(agentDir: string): string {
+  return join(agentDir, MEMORY_DIR, STREAMS_SUBDIR)
+}
+
+export function topicShardPath(agentDir: string, slug: string): string {
+  if (slug.includes('..') || slug.includes('/') || slug.includes('\\') || slug.startsWith('.')) {
+    throw new Error(`invalid topic slug: ${JSON.stringify(slug)}`)
+  }
+  return join(agentDir, MEMORY_DIR, TOPICS_SUBDIR, `${slug}.md`)
+}
+
+export function streamFilePath(agentDir: string, date: string): string {
+  if (!STREAM_DATE_RE.test(date)) {
+    throw new Error(`invalid stream date: ${JSON.stringify(date)}`)
+  }
+  return join(agentDir, MEMORY_DIR, STREAMS_SUBDIR, `${date}.jsonl`)
+}
+
+export function preShardBackupPath(agentDir: string): string {
+  return join(agentDir, MEMORY_DIR, PRE_SHARD_BACKUP_FILENAME)
+}
+
+export function migratingTmpDir(agentDir: string): string {
+  return join(agentDir, MEMORY_DIR, '.migrating')
+}

package/src/bundled-plugins/memory/search-tool.ts

@@ -0,0 +1,232 @@
+import { z } from 'zod'
+
+import { defineTool } from '@/plugin'
+
+import { loadAllShards, type TopicShard } from './load-shards'
+import type { FragmentEvent, LegacyProseEvent, StreamEvent } from './stream-events'
+import { readAllUndreamedStreamDays, type UndreamedStreamDay } from './stream-io'
+
+const DEFAULT_MAX_RESULTS = 10
+const EXCERPT_CONTEXT_LINES = 3
+
+type TopicMatch = {
+  source: 'topic'
+  shardPath: string
+  slug: string
+  heading: string
+  excerpt: string
+  fullBody?: string
+}
+
+type StreamMatch = {
+  source: 'stream'
+  streamPath: string
+  date: string
+  eventId?: string
+  topic: string
+  excerpt: string
+  fullBody?: string
+}
+
+type MemorySearchMatch = TopicMatch | StreamMatch
+
+type MemorySearchResult = { matches: MemorySearchMatch[]; truncatedAt?: number } | { error: string }
+
+type Matcher = (haystack: string) => boolean
+
+export const memorySearchTool = defineTool({
+  description:
+    'Search the agent\'s long-term memory. Covers both topic shards under memory/topics/ (consolidated facts) and undreamed daily-stream events under memory/streams/ (recent fragments not yet folded into shards). Case-insensitive substring by default; asRegex=true treats query as a JavaScript regex. Returns matches discriminated by `source: "topic" | "stream"`, each with line-context excerpts; full=true includes complete bodies. Topic matches come first (alphabetical by slug), then stream matches (newest day first).',
+  parameters: z.object({
+    query: z.string(),
+    asRegex: z.boolean().default(false),
+    full: z.boolean().default(false),
+    maxResults: z.number().int().min(0).default(DEFAULT_MAX_RESULTS),
+  }),
+  async execute({ query, asRegex, full, maxResults }, ctx) {
+    const matcherOrError = buildMatcher(query, asRegex)
+    if (typeof matcherOrError === 'string') {
+      return resultToToolResult({ error: matcherOrError })
+    }
+
+    const [shards, streamDays] = await Promise.all([
+      loadAllShards(ctx.agentDir, { logger: ctx.logger }),
+      readAllUndreamedStreamDays(ctx.agentDir),
+    ])
+    if (shards.length === 0 && streamDays.length === 0) {
+      return resultToToolResult({ matches: [], truncatedAt: 0 })
+    }
+
+    const result = searchAll(shards, streamDays, matcherOrError, { full, maxResults })
+    return resultToToolResult(result)
+  },
+})
+
+function buildMatcher(query: string, asRegex: boolean): Matcher | string {
+  if (asRegex) {
+    try {
+      const regex = new RegExp(query, 'i')
+      return (haystack) => regex.test(haystack)
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err)
+      return `invalid regex: ${message}`
+    }
+  }
+
+  const needle = query.toLowerCase()
+  return (haystack) => haystack.toLowerCase().includes(needle)
+}
+
+// Result-ordering contract: topic shards first (alphabetical by slug, the
+// order `loadAllShards` returns), then stream days (newest first). Truncation
+// cuts from the tail of this concatenation — stream matches are sacrificed
+// before topic matches when `maxResults` is exhausted. The agent reading
+// results in this order sees long-term consolidated truth before recent
+// ephemeral fragments, which mirrors the injection-side rendering order.
+function searchAll(
+  shards: TopicShard[],
+  streamDays: UndreamedStreamDay[],
+  matcher: Matcher,
+  options: { full: boolean; maxResults: number },
+): MemorySearchResult {
+  const matches: MemorySearchMatch[] = []
+  let truncatedAt: number | undefined
+
+  const push = (match: MemorySearchMatch): boolean => {
+    if (matches.length >= options.maxResults) {
+      truncatedAt = options.maxResults
+      return false
+    }
+    matches.push(match)
+    return true
+  }
+
+  for (const shard of shards) {
+    const match = matchShard(shard, matcher, options.full)
+    if (match === null) continue
+    if (!push(match)) return { matches, truncatedAt: truncatedAt! }
+  }
+
+  for (let i = streamDays.length - 1; i >= 0; i--) {
+    const day = streamDays[i]!
+    for (const event of day.events) {
+      const match = matchStreamEvent(day, event, matcher, options.full)
+      if (match === null) continue
+      if (!push(match)) return { matches, truncatedAt: truncatedAt! }
+    }
+  }
+
+  return truncatedAt === undefined ? { matches } : { matches, truncatedAt }
+}
+
+function matchShard(shard: TopicShard, matcher: Matcher, full: boolean): TopicMatch | null {
+  const bodyLines = splitBodyLines(shard.body)
+  const firstBodyLineIndex = bodyLines.findIndex((line) => matcher(line))
+
+  const matched =
+    matcher(shard.slug) ||
+    matcher(shard.frontmatter.heading) ||
+    (shard.frontmatter.tags?.some((tag) => matcher(tag)) ?? false) ||
+    firstBodyLineIndex !== -1
+  if (!matched) return null
+
+  const match: TopicMatch = {
+    source: 'topic',
+    shardPath: shard.path,
+    slug: shard.slug,
+    heading: shard.frontmatter.heading,
+    excerpt:
+      firstBodyLineIndex === -1 ? fallbackExcerpt(shard, matcher) : excerptForLine(bodyLines, firstBodyLineIndex),
+  }
+  if (full) match.fullBody = shard.body
+  return match
+}
+
+// Stream-event matcher. `fragment` events expose `topic` + `body` for search;
+// `legacy_prose` exposes `text` (no id, no topic). `watermark` events carry
+// no human content and are skipped — they only mark dreaming progress.
+//
+// Fragment matches set `eventId` to the canonical citation format
+// `streams/yyyy-MM-dd#<id>` so the agent can paste search hits straight into
+// shard citations. Legacy prose has no fragment id and therefore omits
+// `eventId` — `parseCitations` would reject any value we synthesised, so we
+// leave the field absent to make the asymmetry visible to the agent.
+function matchStreamEvent(
+  day: UndreamedStreamDay,
+  event: StreamEvent,
+  matcher: Matcher,
+  full: boolean,
+): StreamMatch | null {
+  if (event.type === 'watermark') return null
+  if (event.type === 'fragment') return matchFragmentEvent(day, event, matcher, full)
+  return matchLegacyProseEvent(day, event, matcher, full)
+}
+
+function matchFragmentEvent(
+  day: UndreamedStreamDay,
+  event: FragmentEvent,
+  matcher: Matcher,
+  full: boolean,
+): StreamMatch | null {
+  const bodyLines = splitBodyLines(event.body)
+  const firstBodyLineIndex = bodyLines.findIndex((line) => matcher(line))
+  const matched = matcher(event.topic) || firstBodyLineIndex !== -1
+  if (!matched) return null
+
+  const match: StreamMatch = {
+    source: 'stream',
+    streamPath: day.path,
+    date: day.date,
+    eventId: `streams/${day.date}#${event.id}`,
+    topic: event.topic,
+    excerpt: firstBodyLineIndex === -1 ? event.topic : excerptForLine(bodyLines, firstBodyLineIndex),
+  }
+  if (full) match.fullBody = event.body
+  return match
+}
+
+function matchLegacyProseEvent(
+  day: UndreamedStreamDay,
+  event: LegacyProseEvent,
+  matcher: Matcher,
+  full: boolean,
+): StreamMatch | null {
+  const lines = splitBodyLines(event.text)
+  const firstLineIndex = lines.findIndex((line) => matcher(line))
+  if (firstLineIndex === -1) return null
+
+  const match: StreamMatch = {
+    source: 'stream',
+    streamPath: day.path,
+    date: day.date,
+    topic: '[legacy prose from pre-shard migration]',
+    excerpt: excerptForLine(lines, firstLineIndex),
+  }
+  if (full) match.fullBody = event.text
+  return match
+}
+
+function splitBodyLines(body: string): string[] {
+  const lines = body.split('\n')
+  return lines.length > 0 && lines[lines.length - 1] === '' ? lines.slice(0, -1) : lines
+}
+
+function fallbackExcerpt(shard: TopicShard, matcher: Matcher): string {
+  if (matcher(shard.frontmatter.heading)) return shard.frontmatter.heading
+  if (matcher(shard.slug)) return shard.slug
+  const matchedTag = shard.frontmatter.tags?.find((tag) => matcher(tag))
+  return matchedTag ?? shard.frontmatter.heading
+}
+
+function excerptForLine(lines: string[], matchIndex: number): string {
+  const start = Math.max(0, matchIndex - EXCERPT_CONTEXT_LINES)
+  const end = Math.min(lines.length, matchIndex + EXCERPT_CONTEXT_LINES + 1)
+  return lines.slice(start, end).join('\n')
+}
+
+function resultToToolResult(result: MemorySearchResult) {
+  return {
+    content: [{ type: 'text' as const, text: JSON.stringify(result) }],
+    details: result,
+  }
+}

package/src/bundled-plugins/memory/secret-detector.ts

@@ -1,8 +1,8 @@
 // Defense-in-depth backstop against credential leakage into memory streams.
 // The memory-logger system prompt forbids quoting secret values, but the LLM
 // occasionally violates that rule by quoting `env | grep` output verbatim as
-// "evidence". Once a secret reaches a daily stream file, dreaming promotes it
-// into MEMORY.md and the runtime force-commits both to git — at which point
+// "evidence". Once a secret reaches a daily stream file, dreaming can promote it
+// into memory/topics/ and the runtime force-commits both to git — at which point
 // rotation is the only recourse. We deliberately avoid generic high-entropy
 // heuristics: false positives here would silently lose legitimate fragments.
 

package/src/bundled-plugins/memory/shard-snapshot.ts

@@ -0,0 +1,51 @@
+import { mkdir, readdir, readFile, unlink, writeFile } from 'node:fs/promises'
+import { dirname, extname, resolve } from 'node:path'
+
+export async function captureShardSnapshot(topicsDir: string): Promise<Map<string, Buffer>> {
+  let entries: string[]
+  try {
+    entries = await readdir(topicsDir)
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === 'ENOENT') {
+      return new Map()
+    }
+    throw e
+  }
+
+  const snapshot = new Map<string, Buffer>()
+  for (const entry of entries) {
+    if (extname(entry) !== '.md') continue
+    const absPath = resolve(topicsDir, entry)
+    const bytes = await readFile(absPath)
+    snapshot.set(absPath, bytes)
+  }
+
+  const sorted = new Map([...snapshot.entries()].sort((a, b) => a[0].localeCompare(b[0])))
+  return sorted
+}
+
+export async function restoreShardSnapshot(snapshot: Map<string, Buffer>, topicsDir: string): Promise<void> {
+  for (const [absPath, bytes] of snapshot) {
+    await mkdir(dirname(absPath), { recursive: true })
+    await writeFile(absPath, bytes)
+  }
+
+  let currentEntries: string[]
+  try {
+    currentEntries = await readdir(topicsDir)
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === 'ENOENT') {
+      return
+    }
+    throw e
+  }
+
+  const snapshotKeys = new Set(snapshot.keys())
+  for (const entry of currentEntries) {
+    if (extname(entry) !== '.md') continue
+    const absPath = resolve(topicsDir, entry)
+    if (!snapshotKeys.has(absPath)) {
+      await unlink(absPath)
+    }
+  }
+}