workoffdesk 0.0.0-stage → 1.0.0

package/dist/server/journal/db.js

@@ -0,0 +1,243 @@
+import { randomUUID } from 'node:crypto';
+import { chmodSync, existsSync, renameSync } from 'node:fs';
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS journal (epoch TEXT NOT NULL);
+CREATE TABLE IF NOT EXISTS session (id TEXT PRIMARY KEY, v INTEGER NOT NULL, conv TEXT, conv_read INTEGER NOT NULL DEFAULT 0);
+CREATE TABLE IF NOT EXISTS item (
+  session TEXT NOT NULL, id TEXT NOT NULL, conv TEXT NOT NULL, pos INTEGER NOT NULL, v INTEGER NOT NULL, body TEXT NOT NULL,
+  PRIMARY KEY (session, id)
+);
+CREATE INDEX IF NOT EXISTS item_by_pos ON item (session, conv, pos);
+CREATE INDEX IF NOT EXISTS item_by_v ON item (session, v);
+CREATE TABLE IF NOT EXISTS message (
+  session TEXT NOT NULL, id TEXT NOT NULL, v INTEGER NOT NULL, text TEXT NOT NULL, sent_at INTEGER NOT NULL,
+  state TEXT NOT NULL, ord REAL, item TEXT,
+  PRIMARY KEY (session, id)
+);
+CREATE INDEX IF NOT EXISTS message_by_v ON message (session, v);
+CREATE TABLE IF NOT EXISTS state (session TEXT NOT NULL, name TEXT NOT NULL, v INTEGER NOT NULL, body TEXT NOT NULL, PRIMARY KEY (session, name));
+CREATE INDEX IF NOT EXISTS state_by_v ON state (session, v);
+`;
+// SQLITE_CORRUPT and SQLITE_NOTADB: a file that cannot be read as a database at all.
+const UNREADABLE = new Set([11, 26]);
+// The item's id is the row's key: tool, result, task and bg items have an id of their own.
+const itemRow = (r) => ({ t: 'item', key: r.id, v: r.v, body: { ...JSON.parse(r.body), pos: r.pos, conv: r.conv } });
+const toMessage = (r) => ({ id: r.id, text: r.text, sentAt: r.sent_at, state: r.state, ord: r.ord, item: r.item });
+const messageRow = (r) => ({ t: 'message', key: r.id, v: r.v, body: toMessage(r) });
+const stateRow = (r) => ({ t: 'state', key: r.name, v: r.v, body: JSON.parse(r.body) });
+export class JournalDb {
+    epoch;
+    db;
+    q;
+    depth = 0;
+    constructor(file, log = () => { }) {
+        let opened;
+        try {
+            opened = open(file);
+        }
+        catch (e) {
+            if (!UNREADABLE.has(e.errcode ?? -1))
+                throw e;
+            const aside = `${file}.broken-${Date.now()}`;
+            renameSync(file, aside);
+            for (const side of ['-wal', '-shm'])
+                if (existsSync(file + side))
+                    renameSync(file + side, aside + side);
+            log('journal unreadable, set aside and started over', { aside, error: e.message });
+            opened = open(file);
+        }
+        this.db = opened.db;
+        this.epoch = opened.epoch;
+        const p = (sql) => this.db.prepare(sql);
+        this.q = {
+            session: p('SELECT v, conv, conv_read AS convRead FROM session WHERE id = ?'),
+            heads: p('SELECT id, v FROM session'),
+            setConv: p('INSERT INTO session (id, v, conv, conv_read) VALUES (?, 0, ?, ?) ON CONFLICT (id) DO UPDATE SET conv = excluded.conv, conv_read = excluded.conv_read'),
+            bump: p('INSERT INTO session (id, v) VALUES (?, 1) ON CONFLICT (id) DO UPDATE SET v = v + 1 RETURNING v'),
+            hasItem: p('SELECT 1 FROM item WHERE session = ? AND id = ?'),
+            putItem: p('INSERT INTO item (session, id, conv, pos, v, body) VALUES (?, ?, ?, ?, ?, ?)'),
+            lastPos: p('SELECT MAX(pos) AS pos FROM item WHERE session = ? AND conv = ?'),
+            itemsSince: p('SELECT id, body FROM item WHERE session = ? AND conv = ? AND pos > ? ORDER BY pos'),
+            putMessage: p(`INSERT INTO message (session, id, v, text, sent_at, state, ord, item) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT (session, id) DO UPDATE SET v = excluded.v, text = excluded.text, sent_at = excluded.sent_at,
+           state = excluded.state, ord = excluded.ord, item = excluded.item`),
+            messages: p('SELECT id, v, text, sent_at, state, ord, item FROM message WHERE session = ? ORDER BY v'),
+            message: p('SELECT id, v, text, sent_at, state, ord, item FROM message WHERE session = ? AND id = ?'),
+            inState: p('SELECT session, id, v, text, sent_at, state, ord, item FROM message WHERE state = ?'),
+            stateBody: p('SELECT body FROM state WHERE session = ? AND name = ?'),
+            withState: p('SELECT session FROM state WHERE name = ?'),
+            putState: p('INSERT INTO state (session, name, v, body) VALUES (?, ?, ?, ?) ON CONFLICT (session, name) DO UPDATE SET v = excluded.v, body = excluded.body'),
+            itemsAfter: p('SELECT id, conv, pos, v, body FROM item WHERE session = ? AND v > ? ORDER BY v LIMIT ?'),
+            messagesAfter: p('SELECT id, v, text, sent_at, state, ord, item FROM message WHERE session = ? AND v > ? ORDER BY v LIMIT ?'),
+            statesAfter: p('SELECT name, v, body FROM state WHERE session = ? AND v > ? ORDER BY v LIMIT ?'),
+            allMessages: p('SELECT id, v, text, sent_at, state, ord, item FROM message WHERE session = ? ORDER BY v'),
+            allStates: p('SELECT name, v, body FROM state WHERE session = ? ORDER BY v'),
+            itemsBefore: p('SELECT id, conv, pos, v, body FROM item WHERE session = ? AND conv = ? AND pos < ? ORDER BY pos DESC LIMIT ?'),
+            newestFirst: p('SELECT body FROM item WHERE session = ? AND conv = ? ORDER BY pos DESC'),
+            dropItems: p('DELETE FROM item WHERE session = ?'),
+            dropMessages: p('DELETE FROM message WHERE session = ?'),
+            dropStates: p('DELETE FROM state WHERE session = ?'),
+            dropSession: p('DELETE FROM session WHERE id = ?'),
+        };
+    }
+    // One transaction, which a call made inside another joins. f must not wait on anything.
+    tx(f) {
+        if (this.depth) {
+            this.depth++;
+            try {
+                return f();
+            }
+            finally {
+                this.depth--;
+            }
+        }
+        this.db.exec('BEGIN IMMEDIATE');
+        this.depth = 1;
+        try {
+            const out = f();
+            this.db.exec('COMMIT');
+            return out;
+        }
+        catch (e) {
+            this.db.exec('ROLLBACK');
+            throw e;
+        }
+        finally {
+            this.depth = 0;
+        }
+    }
+    session(id) {
+        const r = this.q.session.get(id);
+        return r ? { v: r.v, conv: r.conv, convRead: r.convRead } : null;
+    }
+    setConv(id, conv, convRead) {
+        this.q.setConv.run(id, conv, convRead);
+    }
+    bump(id) {
+        return this.q.bump.get(id).v;
+    }
+    // False when the item is there already: an item never changes once written.
+    putItem(id, conv, itemId, pos, body) {
+        return this.tx(() => {
+            if (this.q.hasItem.get(id, itemId))
+                return false;
+            this.q.putItem.run(id, itemId, conv, pos, this.bump(id), JSON.stringify(body));
+            return true;
+        });
+    }
+    lastPos(id, conv) {
+        return this.q.lastPos.get(id, conv).pos ?? -1;
+    }
+    itemsSince(id, conv, pos) {
+        return this.q.itemsSince.all(id, conv, pos).map((r) => ({ id: r.id, body: JSON.parse(r.body) }));
+    }
+    // Claude's words since the last message to it.
+    turnTexts(id, conv) {
+        const texts = [];
+        for (const r of this.q.newestFirst.iterate(id, conv)) {
+            const item = JSON.parse(r.body);
+            if (item.k === 'user')
+                break;
+            if (item.k === 'text')
+                texts.unshift(item);
+        }
+        return texts;
+    }
+    putMessage(id, m) {
+        this.tx(() => this.q.putMessage.run(id, m.id, this.bump(id), m.text, m.sentAt, m.state, m.ord, m.item));
+    }
+    // Oldest change first.
+    messages(id) {
+        return this.q.messages.all(id).map(toMessage);
+    }
+    message(id, qid) {
+        const r = this.q.message.get(id, qid);
+        return r ? messageRow(r) : null;
+    }
+    // Every session's messages in this state.
+    messagesIn(state) {
+        return this.q.inState.all(state).map((r) => ({ session: r.session, m: toMessage(r) }));
+    }
+    // False when the body is what is there already: the screen is read four times a second, and a read that
+    // finds it the same is no change.
+    putState(id, name, body) {
+        const json = JSON.stringify(body);
+        return this.tx(() => {
+            const now = this.q.stateBody.get(id, name);
+            if (now?.body === json)
+                return false;
+            this.q.putState.run(id, name, this.bump(id), json);
+            return true;
+        });
+    }
+    // Every session that has this state.
+    sessionsWith(name) {
+        return this.q.withState.all(name).map((r) => r.session);
+    }
+    state(id, name) {
+        const r = this.q.stateBody.get(id, name);
+        return r ? JSON.parse(r.body) : undefined;
+    }
+    // The rows written after version v, oldest first, at most limit of them.
+    rowsAfter(id, v, limit) {
+        const rows = [
+            ...this.q.itemsAfter.all(id, v, limit + 1).map(itemRow),
+            ...this.q.messagesAfter.all(id, v, limit + 1).map(messageRow),
+            ...this.q.statesAfter.all(id, v, limit + 1).map(stateRow),
+        ].sort((a, b) => a.v - b.v);
+        return { rows: rows.slice(0, limit), more: rows.length > limit };
+    }
+    // What a copy starting over needs: every message and state, and the newest items of the conversation
+    // shown now. older: the pos before which there are more items, null when there are none.
+    firstRows(id, items) {
+        const conv = this.session(id)?.conv ?? null;
+        const newest = conv ? this.q.itemsBefore.all(id, conv, Number.MAX_SAFE_INTEGER, items + 1) : [];
+        const kept = newest.slice(0, items).reverse();
+        const rows = [...kept.map(itemRow), ...this.q.allMessages.all(id).map(messageRow), ...this.q.allStates.all(id).map(stateRow)];
+        return { rows, older: newest.length > items ? kept[0].pos : null };
+    }
+    // The items of conv before pos, oldest first, at most limit of them; older: the pos before which there
+    // are more, null when there are none.
+    itemsBefore(id, conv, pos, limit) {
+        const got = this.q.itemsBefore.all(id, conv, pos, limit + 1);
+        const kept = got.slice(0, limit).reverse();
+        return { rows: kept.map(itemRow), older: got.length > limit ? kept[0].pos : null };
+    }
+    // Each session's version.
+    heads() {
+        return Object.fromEntries(this.q.heads.all().map((r) => [r.id, r.v]));
+    }
+    drop(id) {
+        this.tx(() => {
+            for (const q of [this.q.dropItems, this.q.dropMessages, this.q.dropStates, this.q.dropSession])
+                q.run(id);
+        });
+    }
+    close() {
+        this.db.close();
+    }
+}
+// Node has SQLite built in from 22.13 (and 22.5 behind a flag); without it there is no journal.
+export const sqliteAvailable = () => !!process.getBuiltinModule('node:sqlite');
+function open(file) {
+    const sqlite = process.getBuiltinModule('node:sqlite');
+    const db = new sqlite.DatabaseSync(file);
+    try {
+        db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA busy_timeout = 2000;');
+        db.exec(SCHEMA);
+        // What was said in each session: its owner's only, as everything else in the home is.
+        for (const f of [file, `${file}-wal`, `${file}-shm`])
+            if (existsSync(f))
+                chmodSync(f, 0o600);
+        const row = db.prepare('SELECT epoch FROM journal LIMIT 1').get();
+        if (row)
+            return { db, epoch: row.epoch };
+        const epoch = randomUUID();
+        db.prepare('INSERT INTO journal (epoch) VALUES (?)').run(epoch);
+        return { db, epoch };
+    }
+    catch (e) {
+        db.close();
+        throw e;
+    }
+}

package/dist/server/journal/derive.js

@@ -0,0 +1,50 @@
+export function taskList(items) {
+    let tasks = new Map();
+    for (const i of items) {
+        if (i.k === 'todos')
+            tasks = new Map(i.items.map((t, n) => [`todo-${n}`, t]));
+        if (i.k !== 'task')
+            continue;
+        if (i.status === 'deleted') {
+            tasks.delete(i.id);
+            continue;
+        }
+        const was = tasks.get(i.id);
+        const subject = i.subject ?? was?.subject;
+        if (subject)
+            tasks.set(i.id, { subject, status: i.status ?? was?.status ?? 'pending' });
+    }
+    return [...tasks.values()];
+}
+export function backgroundRunning(items) {
+    const tools = new Map();
+    const running = new Map();
+    for (const i of items) {
+        if (i.k === 'tool')
+            tools.set(i.id, i.summary);
+        if (i.k !== 'bg')
+            continue;
+        if (i.status === 'running')
+            running.set(i.id, { id: i.id, label: (i.tool && tools.get(i.tool)) || i.id });
+        else
+            running.delete(i.id);
+    }
+    return [...running.values()];
+}
+// How much of what Claude is writing is compared with what the transcript already has.
+const LIVE_PROBE = 40;
+// The screen shows Markdown drawn, the transcript has it written: both are compared without its marks.
+const flat = (s) => s.replace(/[\s*_`#>~|-]+/g, '');
+// The reply Claude is writing, unless the transcript has it already among the items since the last
+// message to Claude (then the reply shows, and the preview would be a second copy).
+export function liveShown(live, items) {
+    if (!live)
+        return null;
+    const probe = flat(live).slice(0, LIVE_PROBE);
+    for (let i = items.length - 1; i >= 0 && items[i].k !== 'user'; i--) {
+        const t = items[i];
+        if (t.k === 'text' && flat(t.text).includes(probe))
+            return null;
+    }
+    return live;
+}

package/dist/server/journal/index.js

@@ -0,0 +1,256 @@
+import { JournalDb } from './db.js';
+import { liveShown } from './derive.js';
+import { ingest, landFound, onTheirWay } from './ingest.js';
+const NOT_LANDED = new Set(['delivering', 'delivered', 'enqueued', 'unconfirmed']);
+// As many rows as one answer carries; a copy further behind asks again.
+const PAGE_ROWS = 500;
+// The items a copy starting over is given, and a page of older ones.
+const PAGE_ITEMS = 100;
+// What the Machine and the routes write to and read from the journal (db.ts), and who is told of new rows.
+export class Journal {
+    epoch;
+    db;
+    root;
+    log;
+    listeners = new Map();
+    // Per session listened to, the version its listeners were last told up to.
+    told = new Map();
+    headListeners = new Set();
+    running = new Map();
+    queued = new Map();
+    // The reply being written as the screen last read it, and the last one it showed, kept until the
+    // transcript grows: the screen stops showing it (a tool runs, the turn ends) before the transcript has it.
+    screenLive = new Map();
+    heldLive = new Map();
+    titled;
+    constructor(o) {
+        this.db = new JournalDb(o.file, o.log);
+        this.epoch = this.db.epoch;
+        this.root = o.root;
+        this.log = o.log;
+        this.titled = o.titled;
+        // A paste the server stopped in the middle of may have gone in: it is not pasted again.
+        for (const { session, m } of this.db.messagesIn('delivering'))
+            this.db.putMessage(session, { ...m, state: 'delivered' });
+        const notLanded = [...this.db.messagesIn('delivered'), ...this.db.messagesIn('enqueued'), ...this.db.messagesIn('unconfirmed')];
+        for (const session of new Set(notLanded.map((r) => r.session)))
+            landFound(this.db, session, this.log);
+        // An ask lives in the server that holds it (PermissionAsks, Codex's agent) and goes with it: none is open
+        // when a server starts, whatever the last one wrote.
+        for (const session of this.db.sessionsWith('asks'))
+            this.db.putState(session, 'asks', []);
+    }
+    // The messages sent to a session through WorkOffDesk (the outbox's storage), oldest change first.
+    messages(id) {
+        return this.db.messages(id);
+    }
+    putMessage(id, m) {
+        this.db.putMessage(id, m);
+        this.tell(id);
+    }
+    // A message as a row, to answer what was done to it with.
+    message(id, qid) {
+        return this.db.message(id, qid);
+    }
+    // workingSince: when Claude set out on what it is doing, by this Mac's clock.
+    status(id, status) {
+        const wrote = this.db.tx(() => {
+            const changed = this.db.putState(id, 'status', status);
+            const since = status === 'working' ? changed && this.db.putState(id, 'workingSince', Date.now()) : this.db.putState(id, 'workingSince', null);
+            return changed || since;
+        });
+        if (wrote)
+            this.tell(id);
+        if (status === 'idle' || status === 'ended')
+            this.notFoundAfterIngest(id);
+    }
+    screen(id, s) {
+        const { live, ...screen } = s;
+        this.screenLive.set(id, live);
+        if (live)
+            this.heldLive.set(id, live);
+        const shown = live ?? this.heldLive.get(id) ?? null;
+        const conv = this.db.session(id)?.conv;
+        const wrote = this.db.tx(() => {
+            const a = this.db.putState(id, 'screen', screen);
+            const b = this.db.putState(id, 'live', { text: shown && liveShown(shown, conv ? this.db.turnTexts(id, conv) : []) });
+            return a || b;
+        });
+        if (wrote)
+            this.tell(id);
+    }
+    // What the screen shows, for an agent that says itself what it is writing (Codex).
+    screenAsIs(id, screen) {
+        if (this.db.putState(id, 'screen', screen))
+            this.tell(id);
+    }
+    asks(id, asks) {
+        if (this.db.putState(id, 'asks', asks))
+            this.tell(id);
+    }
+    // How many terminals on the Mac have the session's conversation open in a Claude of their own, each one
+    // writing it beside the session's.
+    onMac(id, terminals) {
+        if (this.db.putState(id, 'onMac', terminals))
+            this.tell(id);
+    }
+    // Items an agent hands over itself (Codex), in order; one already in is left as it is. A user message that
+    // carries the id a message was sent under is that message: it landed.
+    putAgentItems(id, conv, entries) {
+        let wrote = false;
+        this.db.tx(() => {
+            if (this.db.session(id)?.conv !== conv) {
+                this.db.setConv(id, conv, 0);
+                this.db.putState(id, 'conv', { conv });
+                wrote = true;
+            }
+            let pos = this.db.lastPos(id, conv);
+            for (const e of entries) {
+                if (!this.db.putItem(id, conv, e.id, pos + 1, e.item))
+                    continue;
+                pos++;
+                wrote = true;
+                const m = e.clientId && this.db.messages(id).find((r) => r.id === e.clientId);
+                if (m && NOT_LANDED.has(m.state))
+                    this.db.putMessage(id, { ...m, state: 'landed', item: e.id });
+            }
+        });
+        if (wrote)
+            this.tell(id);
+    }
+    // The reply an agent is writing, as it says it; null once it is written.
+    live(id, text) {
+        if (this.db.putState(id, 'live', { text }))
+            this.tell(id);
+    }
+    // Takes in what the conversation has added. One run per session at a time; asked meanwhile, one more after.
+    ingest(id, conv) {
+        const waiting = this.queued.get(id);
+        if (waiting) {
+            waiting.conv = conv;
+            return waiting.p;
+        }
+        const before = this.running.get(id);
+        if (!before)
+            return this.run(id, conv);
+        const next = { conv, p: Promise.resolve() };
+        next.p = before.then(() => {
+            this.queued.delete(id);
+            return this.run(id, next.conv);
+        });
+        this.queued.set(id, next);
+        return next.p;
+    }
+    sync(id, epoch, v) {
+        const now = this.db.session(id)?.v ?? 0;
+        if (epoch !== this.epoch || v === 0 || v > now) {
+            const first = this.db.firstRows(id, PAGE_ITEMS);
+            return { epoch: this.epoch, v: now, rows: first.rows, more: false, reset: true, older: first.older };
+        }
+        const { rows, more } = this.db.rowsAfter(id, v, PAGE_ROWS);
+        return { epoch: this.epoch, v: more ? rows[rows.length - 1].v : now, rows, more };
+    }
+    older(id, conv, before) {
+        return this.db.itemsBefore(id, conv, before, PAGE_ITEMS);
+    }
+    // Calls cb with each change's rows: from is the version the last call ended at, to the one this ends at.
+    onRows(id, cb) {
+        let set = this.listeners.get(id);
+        if (!set) {
+            set = new Set();
+            this.listeners.set(id, set);
+            this.told.set(id, this.db.session(id)?.v ?? 0);
+        }
+        set.add(cb);
+        return () => {
+            set.delete(cb);
+            if (set.size || this.listeners.get(id) !== set)
+                return;
+            this.listeners.delete(id);
+            this.told.delete(id);
+        };
+    }
+    // Each session's version, for a device to tell whether the copy it kept is behind before asking.
+    heads() {
+        return this.db.heads();
+    }
+    // Calls cb after anything is written to any session.
+    onHead(cb) {
+        this.headListeners.add(cb);
+        return () => this.headListeners.delete(cb);
+    }
+    drop(id) {
+        this.db.drop(id);
+        this.screenLive.delete(id);
+        this.heldLive.delete(id);
+        for (const l of this.headListeners)
+            l();
+    }
+    close() {
+        this.db.close();
+    }
+    run(id, conv) {
+        const p = this.ingestNow(id, conv)
+            .catch((e) => this.log('journal ingest failed', { session: id, error: e instanceof Error ? e.message : String(e) }))
+            .finally(() => this.running.get(id) === p && this.running.delete(id));
+        this.running.set(id, p);
+        return p;
+    }
+    async ingestNow(id, conv) {
+        if (!this.root)
+            return;
+        const wrote = await ingest(this.db, id, conv, {
+            root: this.root,
+            live: (grew) => {
+                const live = this.screenLive.get(id) ?? null;
+                if (grew && !live)
+                    this.heldLive.delete(id);
+                return live ?? this.heldLive.get(id) ?? null;
+            },
+            log: this.log,
+            titled: (found, fresh) => this.titled?.(id, found, fresh),
+        });
+        if (wrote)
+            this.tell(id);
+    }
+    // A message delivered before the turn was over that what Claude took in does not have is unconfirmed:
+    // shown where it was sent, and said, so a change in how Claude Code records a prompt is seen rather
+    // than guessed at. Only those on their way when the turn ended: one delivered after is the next turn's.
+    notFoundAfterIngest(id) {
+        const waiting = onTheirWay(this.db, id).map((m) => m.id);
+        if (!waiting.length)
+            return;
+        void this.ingest(id, this.db.session(id)?.conv ?? null).then(() => {
+            let wrote = false;
+            for (const m of this.db.messages(id)) {
+                if ((m.state !== 'delivered' && m.state !== 'enqueued') || !waiting.includes(m.id))
+                    continue;
+                this.db.putMessage(id, { ...m, state: 'unconfirmed' });
+                this.log('message not found in the transcript', { session: id, qid: m.id, chars: m.text.length });
+                wrote = true;
+            }
+            if (wrote)
+                this.tell(id);
+        });
+    }
+    tell(id) {
+        for (const l of this.headListeners)
+            l();
+        const set = this.listeners.get(id);
+        if (!set?.size)
+            return;
+        let from = this.told.get(id) ?? 0;
+        for (;;) {
+            const { rows, more } = this.db.rowsAfter(id, from, PAGE_ROWS);
+            if (!rows.length)
+                return;
+            const to = more ? rows[rows.length - 1].v : (this.db.session(id)?.v ?? from);
+            this.told.set(id, to);
+            for (const cb of set)
+                cb(from, to, rows);
+            from = to;
+            if (!more)
+                return;
+        }
+    }
+}