tulingcode 0.1.0

package/src/session/checkpoint-context.ts

@@ -0,0 +1,36 @@
+import type { SessionID } from "./schema"
+
+export type CheckpointContext = {
+  priorTitles: Set<string>
+  expectedRevisions: { id: string; expectedText: string }[]
+}
+
+const store = new Map<string, CheckpointContext>()
+
+function key(sessionID: SessionID, actorID: string): string {
+  return `${sessionID}:${actorID}`
+}
+
+export function set(sessionID: SessionID, actorID: string, ctx: CheckpointContext): void {
+  store.set(key(sessionID, actorID), ctx)
+}
+
+export function get(sessionID: SessionID, actorID: string): CheckpointContext | undefined {
+  const ctx = store.get(key(sessionID, actorID))
+  if (!ctx) return undefined
+  return structuredClone(ctx)
+}
+
+export function remove(sessionID: SessionID, actorID: string): void {
+  store.delete(key(sessionID, actorID))
+}
+
+/** Test-only escape hatch. Resets store between tests. */
+export function _reset(): void {
+  store.clear()
+}
+
+/** Test-only escape hatch. Returns total entry count. */
+export function _size(): number {
+  return store.size
+}

package/src/session/checkpoint-paths.ts

@@ -0,0 +1,86 @@
+import path from "path"
+import fs from "fs/promises"
+import { Global } from "@/global"
+import type { ProjectID } from "@/project/schema"
+import { SessionID } from "./schema"
+
+// ---------------------------------------------------------------------------
+// File helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Session memory root. Houses checkpoint artifacts, task narratives, and
+ * other per-session memory files under `<data>/memory/sessions/<sid>/`.
+ */
+export function metaDir(sessionID: SessionID): string {
+  return path.join(Global.Path.data, "memory", "sessions", sessionID)
+}
+
+/**
+ * v5 single-file checkpoint at `<sid>/checkpoint.md` (no subdir).
+ */
+export function checkpointPath(sessionID: SessionID): string {
+  return path.join(metaDir(sessionID), "checkpoint.md")
+}
+
+/**
+ * v5 per-project memory file at `<data>/memory/projects/<pid>/MEMORY.md`.
+ */
+export function memoryPath(projectID: ProjectID): string {
+  return path.join(Global.Path.data, "memory", "projects", projectID, "MEMORY.md")
+}
+
+/**
+ * Single global memory file at `<data>/memory/global/MEMORY.md`. User-level
+ * cross-project preferences. Read-only from the agent side; no auto-create.
+ */
+export function globalMemoryPath(): string {
+  return path.join(Global.Path.data, "memory", "global", "MEMORY.md")
+}
+
+/**
+ * One-shot rename of a legacy `projects/<pid>/memory.md` to the canonical
+ * `MEMORY.md`. Idempotent: no-op when the uppercase file already exists or
+ * when neither exists. The rename is atomic, so concurrent readers see either
+ * the old or new name, never a missing file. Call before reading/writing
+ * project memory so the uppercase path is authoritative.
+ */
+export async function migrateProjectMemory(projectID: ProjectID): Promise<void> {
+  const upper = memoryPath(projectID)
+  const lower = path.join(path.dirname(upper), "memory.md")
+  if (await Bun.file(upper).exists()) return
+  if (await Bun.file(lower).exists())
+    // Two migrators (e.g. concurrent sessions/writers on the same project) can
+    // both pass the exists() checks; the loser's rename then sees lower already
+    // gone. ENOENT means the peer won — treat as success. Re-throw real FS
+    // errors (permissions, disk).
+    await fs.rename(lower, upper).catch((e) => {
+      if ((e as NodeJS.ErrnoException).code !== "ENOENT") throw e
+    })
+}
+
+/**
+ * v8 session-scoped notes file at `<sid>/notes.md`. Main-agent-only
+ * scratchpad; writer reconciles entries at checkpoint events.
+ */
+export function notesPath(sessionID: SessionID): string {
+  return path.join(metaDir(sessionID), "notes.md")
+}
+
+/**
+ * Per-session tasks directory at `<sid>/tasks/`. Houses per-task progress
+ * journals authored either by subagents (Spec ②) or by the splitover
+ * plugin (when main checkpoint.md grows past caps).
+ */
+export function tasksDir(sessionID: SessionID): string {
+  return path.join(metaDir(sessionID), "tasks")
+}
+
+/**
+ * Per-task progress journal at `<sid>/tasks/<TID>/progress.md`. Authored
+ * by subagents (Spec ② actor.postStop) and read by the checkpoint writer's
+ * reconcile preprocessor (Spec ② Chain 2).
+ */
+export function progressPath(sessionID: SessionID, taskID: string): string {
+  return path.join(tasksDir(sessionID), taskID, "progress.md")
+}

package/src/session/checkpoint-progress-reconcile.ts

@@ -0,0 +1,111 @@
+import fs from "fs/promises"
+import path from "path"
+import { checkpointPath, tasksDir } from "./checkpoint-paths"
+import type { SessionID } from "./schema"
+import { Log } from "../util"
+
+const log = Log.create({ service: "session.progress-reconcile" })
+
+export interface ProgressDiffItem {
+  taskId: string
+  writtenAt: number
+  status: "NEW" | "CHANGED"
+  prior?: number
+}
+
+// Parse written-at field from markdown frontmatter
+// Returns null when frontmatter or field is absent
+export function parseWrittenAt(body: string): number | null {
+  const fm = body.match(/^---\n([\s\S]*?)\n---\n/)
+  if (!fm) return null
+  const line = fm[1].split("\n").find((l) => l.startsWith("written-at:"))
+  if (!line) return null
+  const value = Number(line.slice("written-at:".length).trim())
+  if (!Number.isFinite(value)) return null
+  return value
+}
+
+// Parse main checkpoint.md for last-reconciled-written-at markers
+// Returns Map of task ID to reconciled timestamp
+export function parseReconciledMap(mainCheckpoint: string): Map<string, number> {
+  const map = new Map<string, number>()
+  // Negative lookbehind on backtick avoids matching a marker quoted inside a
+  // code span (blast radius is tiny — at most one missed reconcile round — but
+  // cheap to harden).
+  const re = /(?<!`)\(progress:\s*tasks\/([^/]+)\/progress\.md,\s*last-reconciled-written-at:\s*(\d+)\)/g
+  for (const m of mainCheckpoint.matchAll(re)) {
+    const value = Number(m[2])
+    if (Number.isFinite(value)) map.set(m[1], value)
+  }
+  return map
+}
+
+// Scan tasks/*/progress.md and compare written-at against prior reconciliation
+// Returns NEW and CHANGED items; UNCHANGED and unparseable files are omitted
+export async function buildProgressDiffItems(sessionID: SessionID): Promise<ProgressDiffItem[]> {
+  const main = await Bun.file(checkpointPath(sessionID)).text().catch(() => "")
+  const reconciled = parseReconciledMap(main)
+
+  const root = tasksDir(sessionID)
+  let entries: string[]
+  try {
+    entries = await fs.readdir(root)
+  } catch {
+    return []
+  }
+
+  const items: ProgressDiffItem[] = []
+  for (const entry of entries) {
+    const fp = path.join(root, entry, "progress.md")
+    let body: string
+    try {
+      body = await Bun.file(fp).text()
+    } catch {
+      continue
+    }
+    const writtenAt = parseWrittenAt(body)
+    if (writtenAt === null) continue // no written-at marker (e.g. subagent write incomplete); skip
+
+    const prior = reconciled.get(entry)
+    if (prior === undefined) {
+      items.push({ taskId: entry, writtenAt, status: "NEW" })
+    } else if (writtenAt > prior) {
+      items.push({ taskId: entry, writtenAt, status: "CHANGED", prior })
+    }
+    // else UNCHANGED: skip
+  }
+
+  return items
+}
+
+// Render markdown injection block for writer prompt
+// Returns empty string when there are no NEW/CHANGED items
+export function renderProgressDiffBlock(items: ProgressDiffItem[]): string {
+  if (items.length === 0) return ""
+  const lines = ["SUBAGENT PROGRESS to integrate (since last reconcile):"]
+  for (const it of items) {
+    if (it.status === "NEW") {
+      lines.push(`  - ${it.taskId} (NEW, written-at=${it.writtenAt})`)
+    } else {
+      lines.push(`  - ${it.taskId} (CHANGED, written-at=${it.writtenAt}, prior=${it.prior})`)
+    }
+  }
+  lines.push("")
+  lines.push(
+    "For each: Read tasks/<TID>/progress.md, integrate §4 (verbatim commands) verbatim into main §5 Current work; integrate §5 (outcome+discoveries) into main §5 or §7 as appropriate. Then update the corresponding §4 line in main checkpoint to:",
+  )
+  lines.push(
+    "  (progress: tasks/<TID>/progress.md, last-reconciled-written-at: <written-at from above>)",
+  )
+  return lines.join("\n")
+}
+
+// High-level convenience: scan + render in one call
+// Returns empty string when nothing to reconcile
+export async function buildProgressDiff(sessionID: SessionID): Promise<string> {
+  const items = await buildProgressDiffItems(sessionID).catch((err) => {
+    log.warn("buildProgressDiff failed; reconcile skipped this turn", { err, sessionID })
+    return [] as ProgressDiffItem[]
+  })
+  return renderProgressDiffBlock(items)
+}

package/src/session/checkpoint-retry.ts

@@ -0,0 +1,192 @@
+import fs from "fs/promises"
+import path from "path"
+import { SessionID } from "./schema"
+import { ProjectID } from "@/project/schema"
+import {
+  metaDir,
+  checkpointPath,
+  memoryPath,
+} from "./checkpoint-paths"
+import {
+  validateSnapshot,
+  validateLearning,
+  validateMemory,
+  validateProgress,
+  validateBudget,
+  validateBudgetSections,
+  extractTitlesFromLearning,
+  type Violation,
+} from "./checkpoint-validator"
+import { CHECKPOINT_SECTION_BUDGETS, MEMORY_SECTION_BUDGETS } from "./checkpoint-templates"
+
+/**
+ * Returns the set of every Discovered-section title currently present in
+ * the single `checkpoint.md` file. In v5, checkpoints are not numbered —
+ * the writer overwrites `checkpoint.md` each pass (carrying forward prior
+ * entries), so all titles in the file represent "prior" knowledge the
+ * validator can use for de-duplication.
+ */
+export async function loadPriorDiscoveredTitles(
+  sessionID: SessionID,
+): Promise<Set<string>> {
+  const text = await Bun.file(checkpointPath(sessionID)).text().catch(() => "")
+  if (!text) return new Set()
+  return new Set(extractTitlesFromLearning(text))
+}
+
+/**
+ * Read the single-file checkpoint artifacts and run their validators.
+ * Caller passes pre-computed `priorTitles` and `expectedRevisions` so they
+ * are not re-read on each retry attempt.
+ *
+ * Used by `tryStartCheckpointWriter`'s validate-retry loop.
+ */
+export async function runValidatorsForCkpt(
+  sessionID: SessionID,
+  injected: {
+    priorTitles: Set<string>
+    expectedRevisions: { id: string; expectedText: string }[]
+    projectID: ProjectID
+    budgets?: { checkpoint: number; memory: number; progress_per_task: number }
+  },
+): Promise<Violation[]> {
+  const checkpointContent = await Bun.file(checkpointPath(sessionID)).text().catch(() => "")
+  const memoryContent = await Bun.file(memoryPath(injected.projectID)).text().catch(() => "")
+
+  const violations: Violation[] = []
+
+  if (checkpointContent) {
+    violations.push(...validateSnapshot(checkpointContent, "checkpoint.md"))
+    violations.push(...validateLearning(checkpointContent, "checkpoint.md", injected.priorTitles))
+  } else {
+    violations.push({
+      file: "checkpoint.md",
+      rule: "topic-missing",
+      severity: "error",
+      detail: `checkpoint file did not exist after writer finished`,
+    })
+  }
+
+  if (memoryContent && injected.expectedRevisions.length > 0) {
+    violations.push(...validateMemory(memoryContent, injected.expectedRevisions))
+  }
+
+  // Budget validation (v5)
+  const budgets = injected.budgets ?? { checkpoint: 11_000, memory: 10_000, progress_per_task: 6_000 }
+
+  if (checkpointContent) {
+    violations.push(...validateBudget(checkpointContent, budgets.checkpoint, "checkpoint.md"))
+  }
+
+  if (memoryContent) {
+    violations.push(...validateBudget(memoryContent, budgets.memory, "MEMORY.md"))
+  }
+
+  // Per-section budget validation (v5)
+  if (checkpointContent) {
+    violations.push(...validateBudgetSections(checkpointContent, CHECKPOINT_SECTION_BUDGETS, "checkpoint.md"))
+  }
+  if (memoryContent) {
+    violations.push(...validateBudgetSections(memoryContent, MEMORY_SECTION_BUDGETS, "MEMORY.md"))
+  }
+
+  return violations
+}
+
+/**
+ * Validate every present `tasks/<id>/progress.md` once. Hoisted out of the
+ * retry loop because progress.md doesn't change between retry attempts
+ * (the writer is busy fixing checkpoint/memory at that point) and
+ * `next-filler` warnings on stale unrelated tasks are noise the writer
+ * can't fix on retry anyway.
+ *
+ * TODO: filter by mtime > interval-start so we don't surface stale warnings
+ * for tasks not touched in this interval at all.
+ */
+export async function runTaskProgressValidators(sessionID: SessionID): Promise<Violation[]> {
+  const violations: Violation[] = []
+  const taskMemRoot = path.join(metaDir(sessionID), "tasks")
+  const taskDirs = await fs.readdir(taskMemRoot).catch(() => [] as string[])
+  for (const tid of taskDirs) {
+    const progPath = path.join(taskMemRoot, tid, "progress.md")
+    const prog = await fs.readFile(progPath, "utf-8").catch(() => "")
+    if (prog) {
+      violations.push(...validateProgress(prog, `tasks/${tid}/progress.md`))
+    }
+  }
+  return violations
+}
+
+/**
+ * Rename `checkpoint.md` to `checkpoint.invalid.md` to mark it as
+ * failed-validation. File persists on disk for post-mortem inspection.
+ */
+export async function quarantineCheckpoint(
+  sessionID: SessionID,
+): Promise<void> {
+  const dir = metaDir(sessionID)
+  const from = path.join(dir, `checkpoint.md`)
+  const to = path.join(dir, `checkpoint.invalid.md`)
+  await fs.rename(from, to).catch(() => {}) // tolerate writer never produced one
+}
+
+/**
+ * Build the system-reminder body that the writer subagent receives on a
+ * validation-failure retry. Errors are grouped by file; each error's
+ * `detail` becomes a bullet under the file's heading. Followed by the
+ * absolute paths the writer must Edit/Read.
+ *
+ * Used by the validate-retry loop in `tryStartCheckpointWriter` (Task 13).
+ */
+export function buildReflectionMessage(
+  errors: Violation[],
+  paths: { checkpoint: string; memory: string },
+): string {
+  const grouped = new Map<string, string[]>()
+  for (const e of errors) {
+    const list = grouped.get(e.file) ?? []
+    list.push(`- ${e.detail}`)
+    grouped.set(e.file, list)
+  }
+  const sections = [...grouped.entries()].map(
+    ([file, lines]) => `${file}:\n${lines.join("\n")}`,
+  )
+  return [
+    "<system-reminder>",
+    "The previous attempt at this checkpoint had validation errors. Read your output at the absolute paths below, fix ONLY the issues listed, and write again. Other content may stay the same.",
+    "",
+    sections.join("\n\n"),
+    "",
+    `CHECKPOINT_PATH = ${paths.checkpoint}`,
+    `MEMORY_PATH     = ${paths.memory}`,
+    "</system-reminder>",
+  ].join("\n")
+}
+
+/**
+ * Build a reflection prompt for extract-required budget violations. Instructs
+ * the writer to split over-budget files into spillover files rather than
+ * fixing structural errors.
+ */
+export function buildExtractionReflection(violations: Violation[]): string {
+  const overBudget = violations.filter((v) => v.severity === "extract-required")
+  const files = overBudget.map((v) => `${v.file} (${v.detail})`).join(", ")
+  return `EXTRACTION REQUIRED: The following files exceed their token budget: ${files}.
+
+Extract the LESS-IMPORTANT topic cluster from the over-budget file into a new spillover file:
+  - Checkpoint spillover: checkpoint-<topic>.md (sibling of checkpoint.md)
+  - Memory spillover: MEMORY-<topic>.md (sibling of MEMORY.md)
+
+Selection criteria for "less important" (extract THESE first):
+  - Already-stable decisions unlikely to be revisited
+  - Dead ends before Discovered entries
+  - Historical / completed steps before recent / in-progress
+  - Topics not directly relevant to the current focus task
+
+After extraction, edit the main file to:
+  - REMOVE the extracted lines
+  - INSERT an index line near the bottom:
+    "- See <spillover-filename>.md (N entries) — short summary"
+
+Re-validation will run after this single extraction.`
+}

package/src/session/checkpoint-templates.ts

@@ -0,0 +1,114 @@
+export const CHECKPOINT_TEMPLATE = `# Session checkpoint
+_Generated by checkpoint writer; structure preserved across updates. Edit only content under italic instructions._
+
+## §1 Active intent
+_User's most recent explicit request, verbatim block-quoted from the conversation. The verbatim quote is the ground truth for what to do — do not paraphrase._
+
+(none yet)
+
+## §2 Next concrete action
+_The single next concrete step, derived from §1 and current state. Include verbatim quote when the user gave one._
+
+(none yet)
+
+## §3 Directives (this session)
+_This session's specific working style. Project-level rules are in projects/<pid>/MEMORY.md — do not duplicate them here. Only put items that are session-specific._
+
+(none)
+
+## §4 Task tree
+_Hierarchical view of tasks with current focus marked. Pulled from task tool's DB; one bullet per top-level task with status + summary. Append \` (progress: tasks/<id>/progress.md)\` so the next agent can Read the per-task journal directly. Indent sub-tasks two spaces under their parent._
+
+(none yet)
+
+## §5 Current work
+_Description of what was being done immediately before this checkpoint. Mention specific file paths and code locations._
+
+(none yet)
+
+## §6 Files and code sections
+_Files actively being read or modified. List with one-line purpose._
+
+(none yet)
+
+## §7 Discovered knowledge (cross-task)
+_Facts learned during this session that may apply to future tasks. Items here are candidates for promotion to projects/<pid>/MEMORY.md if they prove durable._
+
+(none yet)
+
+## §8 Errors and fixes
+_Errors encountered this session and how they were resolved. Newest first._
+
+(none)
+
+## §9 Live resources
+_Runtime state: branch, uncommitted files, running processes, temp artifacts. Most volatile — don't dwell on details that change every minute._
+
+(none yet)
+
+## §10 Design decisions and discussion outcomes
+_Decisions reached through discussion that produced no immediate code/file artifact. Captures user intent or trade-off rationale future agents need to understand "why we did it this way". Promote to MEMORY.md ## Architecture decisions when proven cross-session-durable._
+
+(none yet)
+
+## §11 Open notes
+_Writer-curated catch-all for items that don't fit §1-§10. Quotes from conversation, unresolved questions, micro-observations, miscellaneous. Cleaner than letting orphan content pollute §3 or §7. May be empty for many checkpoints._
+
+(none yet)
+`
+
+export const MEMORY_TEMPLATE = `# Project memory
+_Durable project-level knowledge. Persists across all sessions in this project. Edit only content under italic instructions._
+
+## Project context
+_What is this project? What's its goal? High-level identity._
+
+(none yet)
+
+## Rules
+_Hard constraints from user that every session must respect._
+
+(none yet)
+
+## Architecture decisions
+_Major design choices with rationale. The "why" matters more than the "what" for future sessions._
+
+(none yet)
+
+## Discovered durable knowledge
+_Cross-task facts that survive across sessions. Promoted from session checkpoints' §7 when proven durable._
+
+(none yet)
+`
+
+/**
+ * Section budgets for checkpoint.md (token estimates per section).
+ * Total ~11K, well under the 40K tail max.
+ */
+export const CHECKPOINT_SECTION_BUDGETS: Record<string, number> = {
+  "§1 Active intent": 500,
+  "§2 Next concrete action": 1000,
+  "§3 Directives (this session)": 800,
+  "§4 Task tree": 1000,
+  "§5 Current work": 2000,
+  "§6 Files and code sections": 1500,
+  "§7 Discovered knowledge (cross-task)": 2000,
+  "§8 Errors and fixes": 1500,
+  "§9 Live resources": 1000,
+  "§10 Design decisions and discussion outcomes": 3000,
+  "§11 Open notes": 800,
+}
+
+/**
+ * Section budgets for MEMORY.md (project-level). Total ~10K.
+ */
+export const MEMORY_SECTION_BUDGETS: Record<string, number> = {
+  "Project context": 1000,
+  "Rules": 2000,
+  "Architecture decisions": 3000,
+  "Discovered durable knowledge": 4000,
+}
+
+export const NOTES_TEMPLATE = `# Session notes
+_Free-form scratchpad for the main agent. Append entries as you go; the checkpoint writer reconciles them at checkpoint events. Format each entry as \`## [turn N · YYYY-MM-DDTHH:MM:SSZ]\` (minute precision UTC, seconds optional) followed by free-form body. Before appending: scan existing entries — if you've already noted substantially similar content, add a short \`(see entry above)\` reference instead of duplicating._
+`