picocode-core 0.9.199 → 0.9.201

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "picocode-core",
-  "version": "0.9.199",
+  "version": "0.9.201",
   "description": "The agent runtime behind pico: sessions, tools, subagents, MCP, memory, and model access",
   "type": "module",
   "license": "MIT",

package/src/daemon-patch.js

@@ -0,0 +1,35 @@
+import { isDeepStrictEqual } from 'node:util'
+
+export function diffSnapshot(previous, next, path = [], patches = []) {
+  if (Object.is(previous, next)) return patches
+  if (typeof previous === 'string' && typeof next === 'string' && next.startsWith(previous)) {
+    patches.push({ op: 'appendText', path, value: next.slice(previous.length) })
+  } else if (Array.isArray(previous) && Array.isArray(next)) {
+    const common = Math.min(previous.length, next.length)
+    for (let i = 0; i < common; i++) diffSnapshot(previous[i], next[i], [...path, i], patches)
+    if (next.length < previous.length) patches.push({ op: 'truncate', path, length: next.length })
+    if (next.length > previous.length) patches.push({ op: 'append', path, value: next.slice(previous.length) })
+  } else if (previous && next && Object.getPrototypeOf(previous) === Object.prototype && Object.getPrototypeOf(next) === Object.prototype) {
+    for (const key of Object.keys(previous)) if (!(key in next)) patches.push({ op: 'delete', path: [...path, key] })
+    for (const key of Object.keys(next)) diffSnapshot(previous[key], next[key], [...path, key], patches)
+  } else if (!isDeepStrictEqual(previous, next)) patches.push({ op: 'set', path, value: next })
+  return patches
+}
+
+export function applyPatches(snapshot, patches) {
+  function update(value, path, patch) {
+    if (!path.length) {
+      if (patch.op === 'appendText') return value + patch.value
+      if (patch.op === 'append') return [...value, ...patch.value]
+      if (patch.op === 'truncate') return value.slice(0, patch.length)
+      return patch.value
+    }
+    const copy = Array.isArray(value) ? [...value] : { ...value }
+    const [key, ...rest] = path
+    if (!rest.length && patch.op === 'delete') delete copy[key]
+    else copy[key] = update(value[key], rest, patch)
+    return copy
+  }
+  for (const patch of patches) snapshot = update(snapshot, patch.path, patch)
+  return snapshot
+}

package/src/daemon-projection.js

@@ -0,0 +1,34 @@
+import { plain } from './daemon-wire.js'
+
+export const defaultTranscriptLimit = 100
+const stateExcluded = new Set(['events', 'persisted', 'derived', 'attachments', 'rewindUndo'])
+const derivedExcluded = new Set(['transcript', 'providerHistory', 'historyEventIds', 'loadedContext', 'toolItems', 'peerMessages', 'consumedPeerIds', 'pendingPeerMessages', 'deferredConsumedPeers', 'openToolCalls', 'trimmedToolIds', 'toolTrimVersions'])
+const agentExcluded = new Set(['events', 'timeline', 'options', 'controller', 'done', 'resolve'])
+const select = (value, excluded) => Object.fromEntries(Object.entries(value || {}).filter(([key]) => !excluded.has(key)))
+
+export function createProjection() {
+  const clones = new WeakMap()
+  const derivedCache = new WeakMap()
+  const windows = new WeakMap()
+  function immutable(value) {
+    if (!value || typeof value !== 'object') return plain(value)
+    if (!clones.has(value)) clones.set(value, plain(value))
+    return clones.get(value)
+  }
+  return {
+    state(source, limit = defaultTranscriptLimit) {
+      const derived = source.derived || {}
+      if (!derivedCache.has(derived)) derivedCache.set(derived, plain(select(derived, derivedExcluded)))
+      const transcript = derived.transcript || []
+      let byLimit = windows.get(derived)
+      if (!byLimit) windows.set(derived, byLimit = new Map())
+      const offset = Math.max(0, transcript.length - limit)
+      if (!byLimit.has(limit)) byLimit.set(limit, transcript.slice(offset).map(immutable))
+      const attachments = new Map([...source.attachments || []].map(([key, value]) => [key, plain(Object.fromEntries(['kind', 'path', 'mediaType', 'hash', 'subject', 'root', 'url', 'selector', 'tag', 'component', 'fromLine', 'toLine', 'fromColumn', 'toColumn'].filter(k => value[k] !== undefined).map(k => [k, value[k]])))]))
+      return { ...plain(select(source, stateExcluded)), attachments, rewindUndo: !!source.rewindUndo, derived: { ...derivedCache.get(derived), transcript: byLimit.get(limit), transcriptOffset: offset, transcriptTotal: transcript.length } }
+    },
+    agents(values) {
+      return values.map(value => ({ ...plain(select(value, agentExcluded)), ...(value.timeline ? { timeline: value.timeline.map(immutable) } : {}), events: (value.events || []).filter(event => ['content', 'tool_executing', 'tool_complete', 'tool_error'].includes(event.type)).map(immutable) }))
+    },
+  }
+}

package/src/daemon-runtime.js

@@ -1,5 +1,7 @@
 import { randomUUID } from 'node:crypto'
 import { plain } from './daemon-wire.js'
+import { diffSnapshot } from './daemon-patch.js'
+import { createProjection, defaultTranscriptLimit } from './daemon-projection.js'
 import { forkSession, loadSession } from './session.js'
 
 const events = ['change', 'derived', 'flash', 'input', 'question', 'turn', 'session', 'resumed', 'mcp', 'shells', 'git', 'project', 'image']
@@ -7,7 +9,11 @@ const queries = ['activity', 'shellRows', 'costSummary', 'speedApplies', 'effort
 const busy = ctl => ctl.isWorking?.() || ctl.state.busy || ctl.state.compacting || ctl.agents?.list().some(a => ['running', 'queued'].includes(a.status))
 const named = ctl => !!ctl.state.derived?.title?.trim()
 
-export function snapshot(ctl) {
+const projections = new WeakMap()
+
+export function snapshot(ctl, limit = defaultTranscriptLimit) {
+  if (!projections.has(ctl)) projections.set(ctl, createProjection())
+  const projection = projections.get(ctl)
   const boot = ctl.boot
   const services = {}
   for (const [service, methods] of Object.entries({ shells: ['list', 'running'], wakeups: ['list', 'pending'], git: ['status'], mcp: ['list'], commands: ['list'], skills: ['list'] })) {
@@ -21,7 +27,7 @@ export function snapshot(ctl) {
   }
   const cached = {}
   for (const method of queries) if (typeof ctl[method] === 'function') cached[method] = plain(ctl[method]())
-  return { state: plain(ctl.state), boot: plain(Object.fromEntries(Object.entries(boot).filter(([key]) => !['refs', 'shells', 'wakeups', 'git', 'mcp', 'memory', 'commands', 'skills', 'tracker'].includes(key)))), services, cached, agents: plain(ctl.agents?.list() || []), methods: Object.keys(ctl).filter(k => typeof ctl[k] === 'function' && !['on', 'shutdown'].includes(k)) }
+  return { state: projection.state(ctl.state, limit), boot: plain(Object.fromEntries(Object.entries(boot).filter(([key]) => !['refs', 'shells', 'wakeups', 'git', 'mcp', 'memory', 'commands', 'skills', 'tracker'].includes(key)))), services, cached, agents: projection.agents(ctl.agents?.list() || []), methods: Object.keys(ctl).filter(k => typeof ctl[k] === 'function' && !['on', 'shutdown'].includes(k)) }
 }
 
 export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs = 16 }) {
@@ -37,19 +43,38 @@ export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs
   async function createRecord(client, options) {
     const created = await createSession(options)
     const ctl = created.controller || created
-    const record = { id: randomUUID(), ctl, dispose: created.dispose, views: new Set([client]), events: [], timer: null, unsub: [] }
+    const record = { id: randomUUID(), ctl, dispose: created.dispose, views: new Set([client]), events: [], timer: null, unsub: [], baselines: new Map(), limits: new Map() }
     sessions.set(record.id, record)
     for (const type of events) record.unsub.push(ctl.on(type, payload => publish(record, type, payload)))
     return record
   }
+  function fullSnapshot(record, client) {
+    const value = snapshot(record.ctl, record.limits.get(client) ?? defaultTranscriptLimit)
+    const sequence = (record.baselines.get(client)?.sequence || 0) + 1
+    record.baselines.set(client, { snapshot: value, sequence })
+    return { snapshot: value, sequence }
+  }
+  function flush(record) {
+    clearTimeout(record.timer)
+    record.timer = null
+    const snapshots = new Map()
+    const notifications = record.events.splice(0)
+    for (const client of record.views) {
+      const baseline = record.baselines.get(client)
+      if (!baseline) continue
+      const limit = record.limits.get(client) ?? defaultTranscriptLimit
+      if (!snapshots.has(limit)) snapshots.set(limit, snapshot(record.ctl, limit))
+      const next = snapshots.get(limit)
+      const sequence = baseline.sequence + 1
+      const patches = diffSnapshot(baseline.snapshot, next)
+      record.baselines.set(client, { snapshot: next, sequence })
+      client.send({ type: 'update', id: record.id, patches, baseSequence: baseline.sequence, sequence, events: notifications })
+    }
+    for (const client of clients) client.send({ type: 'workspace' })
+  }
   function publish(record, type, payload) {
-    if (type !== 'change') record.events.push({ type, payload: plain(type === 'project' ? null : payload) })
-    if (!record.timer) record.timer = setTimeout(() => {
-      record.timer = null
-      const update = { type: 'update', id: record.id, snapshot: snapshot(record.ctl), events: record.events.splice(0) }
-      for (const client of record.views) client.send(update)
-      for (const client of clients) client.send({ type: 'workspace' })
-    }, batchMs)
+    if (!['change', 'derived'].includes(type)) record.events.push({ type, payload: plain(type === 'project' ? null : payload) })
+    if (!record.timer) record.timer = setTimeout(() => flush(record), batchMs)
   }
   async function dispose(record) {
     if (!sessions.delete(record.id)) return
@@ -63,6 +88,8 @@ export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs
     const record = sessions.get(id)
     if (!record) return
     record.views.delete(client)
+    record.baselines.delete(client)
+    record.limits.delete(client)
     if (!record.views.size && !named(record.ctl)) await dispose(record)
   }
   return {
@@ -88,11 +115,19 @@ export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs
       })
       if (op === 'create') {
         const record = await createRecord(client, args[0] || {})
-        return { id: record.id, snapshot: snapshot(record.ctl) }
+        return { id: record.id, ...fullSnapshot(record, client) }
       }
       const record = sessions.get(id)
       if (!record) throw new Error('session is no longer running')
-      if (op === 'attach') { record.views.add(client); return { id, snapshot: snapshot(record.ctl) } }
+      if (op === 'attach') { record.views.add(client); return { id, ...fullSnapshot(record, client) } }
+      if (op === 'snapshot') { if (!record.views.has(client)) throw new Error('attach first'); return fullSnapshot(record, client) }
+      if (op === 'history') {
+        if (!record.views.has(client)) throw new Error('attach first')
+        const [limit] = args
+        if (!Number.isSafeInteger(limit) || limit < 1) throw new Error('history limit must be a positive safe integer')
+        record.limits.set(client, Math.max(record.limits.get(client) ?? defaultTranscriptLimit, limit))
+        return fullSnapshot(record, client)
+      }
       if (op === 'switch') {
         const [method, value] = args
         if (!record.views.has(client)) throw new Error('attach to the session first')
@@ -102,7 +137,7 @@ export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs
           if (existing) {
             existing.views.add(client)
             if (existing !== record) await detach(client, id)
-            return { id: existing.id, snapshot: snapshot(existing.ctl) }
+            return { id: existing.id, ...fullSnapshot(existing, client) }
           }
           const saved = await loadSession(meta.file)
           meta = { ...meta, header: saved.header }
@@ -125,7 +160,7 @@ export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs
             if (renamed === false || next.ctl.state.derived?.title !== name) throw new Error(`could not create session named "${name}"`)
           }
           await detach(client, id)
-          return { id: next.id, snapshot: snapshot(next.ctl) }
+          return { id: next.id, ...fullSnapshot(next, client) }
         } catch (error) { await dispose(next); throw error }
       }
       if (op === 'detach') { await detach(client, id); return }
@@ -150,11 +185,12 @@ export function createDaemonRuntime({ createSession, onEmpty = () => {}, batchMs
         if (target === 'controller' && method === 'compact') {
           Promise.resolve(object[method](...values)).catch(error => publish(record, 'flash', error.message))
           publish(record, 'change')
-          return { result: undefined, snapshot: snapshot(record.ctl) }
+          return { result: undefined, incremental: true }
         }
         const result = await object[method](...values)
         publish(record, 'change')
-        return { result, snapshot: snapshot(record.ctl) }
+        flush(record)
+        return { result, incremental: true }
       }
       if (op === 'configure') {
         const allowed = ['researchModel', 'shellModel', 'deliberationModel', 'participantAModel', 'participantBModel', 'researchAgentLimit', 'clouds', 'compactToolHistory', 'gitFooter', 'wideSidebar']

package/src/daemon-wire.js

@@ -32,12 +32,45 @@ export function wire(socket, receive, fail = () => {}) {
     } catch (error) { fail(error); socket.destroy() }
   })
   socket.on('error', fail)
-  return message => {
+  const updates = new Map()
+  let workspacePending = false
+  function write(message) {
+    if (socket.destroyed) return false
     const body = serialize(plain(message))
-    if (body.length > limit || socket.writableLength > limit) { socket.destroy(); return false }
+    if (body.length > limit) {
+      const error = new Error(`daemon frame exceeds limit (${body.length} bytes)`)
+      fail(error)
+      socket.destroy(error)
+      return false
+    }
     const header = Buffer.alloc(4)
     header.writeUInt32BE(body.length)
     socket.write(Buffer.concat([header, body]))
     return true
   }
+  socket.on('drain', () => {
+    for (const [id, update] of updates) {
+      if (socket.writableNeedDrain || socket.destroyed) break
+      updates.delete(id)
+      write(update)
+    }
+    if (workspacePending && !socket.writableNeedDrain && !socket.destroyed) {
+      workspacePending = false
+      write({ type: 'workspace' })
+    }
+  })
+  socket.on('close', () => updates.clear())
+  return message => {
+    if (socket.destroyed) return false
+    if (socket.writableNeedDrain && message.type === 'update') {
+      const previous = updates.get(message.id)
+      updates.set(message.id, { ...message, ...(message.patches && previous?.patches ? { baseSequence: previous.baseSequence, patches: [...previous.patches, ...message.patches] } : {}), events: [...(previous?.events || []), ...(message.events || [])] })
+      return true
+    }
+    if (socket.writableNeedDrain && message.type === 'workspace') {
+      workspacePending = true
+      return true
+    }
+    return write(message)
+  }
 }

package/src/remote-controller.js

@@ -1,4 +1,4 @@
-import { deriveState } from './derive.js'
+import { applyPatches } from './daemon-patch.js'
 
 export async function createRemoteController(connection, { id, options = {}, localBoot = {} } = {}) {
   const initial = await connection.request(id ? 'attach' : 'create', id, id ? [] : [options])
@@ -6,16 +6,21 @@ export async function createRemoteController(connection, { id, options = {}, loc
   const state = {}
   const boot = { refs: {}, ...localBoot }
   const listeners = new Map()
+  const sequences = new Map([[id, initial.sequence]])
+  let recovering = false
   let current
   let previewId = null
   let previewQueue = Promise.resolve()
   const displayedId = () => previewId || id
   const emit = (type, value) => { for (const fn of listeners.get(type) || []) fn(value) }
-  const call = async (target, method, ...args) => {
-    const reply = await connection.request('call', id, [target, method, ...args])
-    apply(reply.snapshot)
-    emit('change', state)
-    return reply.result
+  const call = (target, method, ...args) => {
+    const result = connection.request('call', id, [target, method, ...args]).then(reply => {
+      if (reply.snapshot) apply(reply.snapshot)
+      emit('change', state)
+      return reply.result
+    })
+    result.catch(error => emit('flash', error.message))
+    return result
   }
   function apply(snapshot) {
     current = snapshot
@@ -56,12 +61,12 @@ export async function createRemoteController(connection, { id, options = {}, loc
       if (nextId === id) { await clearPreview(); return }
       const previous = id
       const previousPreview = previewId
-      const next = await connection.request('attach', nextId)
+      const next = previousPreview === nextId ? null : await connection.request('attach', nextId)
       previewId = null
       if (previousPreview && previousPreview !== nextId) await connection.request('detach', previousPreview)
-      id = next.id
+      id = nextId
       controller.id = id
-      apply(next.snapshot)
+      if (next) { sequences.set(next.id, next.sequence); apply(next.snapshot) }
       await connection.request('detach', previous)
       emit('project', boot)
       emit('session', state.session)
@@ -74,6 +79,7 @@ export async function createRemoteController(connection, { id, options = {}, loc
     const next = await connection.request('switch', id, [method, value])
     id = next.id
     controller.id = id
+    sequences.set(next.id, next.sequence)
     apply(next.snapshot)
     if (method === 'newSession' && typeof value === 'string' && value.trim() && state.derived?.title !== value.trim()) {
       const renamed = await call('controller', 'rename', value.trim())
@@ -83,7 +89,15 @@ export async function createRemoteController(connection, { id, options = {}, loc
     emit('session', state.session)
     emit('change', state)
   }
-  controller.previewSteer = changes => changes?.length ? deriveState([...state.events, { id: '__steer_preview__', at: Date.now(), type: 'steer', data: { changes } }]) : state.derived
+  controller.previewSteer = changes => call('controller', 'previewSteer', changes)
+  controller.loadHistory = async limit => {
+    const target = displayedId()
+    const reply = await connection.request('history', target, [Number.isFinite(limit) ? limit : Number.MAX_SAFE_INTEGER])
+    if (displayedId() !== target) return
+    sequences.set(target, reply.sequence)
+    apply(reply.snapshot)
+    emit('change', state)
+  }
   function show(snapshot) {
     apply(snapshot)
     emit('project', boot)
@@ -96,6 +110,7 @@ export async function createRemoteController(connection, { id, options = {}, loc
     const previous = previewId
     const original = await connection.request('attach', id)
     previewId = null
+    sequences.set(id, original.sequence)
     show(original.snapshot)
     await connection.request('detach', previous)
   }
@@ -106,6 +121,7 @@ export async function createRemoteController(connection, { id, options = {}, loc
       const next = await connection.request('attach', nextId)
       const previous = previewId
       previewId = nextId
+      sequences.set(nextId, next.sequence)
       show(next.snapshot)
       if (previous) await connection.request('detach', previous)
     })
@@ -125,7 +141,26 @@ export async function createRemoteController(connection, { id, options = {}, loc
   const unsubscribe = connection.on(message => {
     if (message.type === 'disconnect') { state.busy = false; state.streaming = null; emit('flash', 'daemon disconnected; work was not restarted'); emit('change', state); return }
     if (message.type !== 'update' || message.id !== displayedId()) return
-    apply(message.snapshot)
+    if (message.patches) {
+      if (recovering) return
+      if (sequences.get(message.id) !== message.baseSequence) {
+        recovering = true
+        const recoveringId = displayedId()
+        connection.request('snapshot', recoveringId).then(reply => {
+          if (displayedId() !== recoveringId) return
+          sequences.set(recoveringId, reply.sequence)
+          apply(reply.snapshot)
+          emit('change', state)
+        }).catch(error => emit('flash', error.message)).finally(() => { recovering = false })
+        return
+      }
+      apply(applyPatches(current, message.patches))
+      sequences.set(message.id, message.sequence)
+      emit('derived', state.derived)
+    } else {
+      apply(message.snapshot)
+      emit('derived', state.derived)
+    }
     for (const event of message.events) {
       if (previewId && ['input', 'question', 'resumed', 'flash'].includes(event.type)) continue
       emit(event.type, event.type === 'project' ? boot : event.payload)