syndes 0.1.0

package/dashboard/web/views/settings.js

@@ -0,0 +1,180 @@
+/**
+ * Settings — a page, laid out in two columns of grouped cards.
+ *
+ * Only keys the server's allowlist accepts are rendered. Showing a control the
+ * API would reject is worse than not showing it: the user changes it, the write
+ * fails, and they have no idea which half lied to them.
+ *
+ * Destructive operations are deliberately absent, with the reason stated on the
+ * page rather than left as a mystery.
+ */
+
+import { api } from '../api.js';
+import { h, card, replace, kv, modal } from '../ui.js';
+
+export async function render() {
+  const page = h('div.settings');
+  await paint(page);
+  return h('div.content--fit', { style: 'display:grid;min-height:0;overflow-y:auto;padding-right:4px' }, [page]);
+}
+
+async function paint(page) {
+  const data = await api.config();
+  const config = data.config;
+
+  const save = async (key, value) => {
+    await api.saveConfig({ [key]: value });
+    await paint(page);
+  };
+
+  const setLock = async (mode) => {
+    let pin;
+    if (mode === 'pin') {
+      pin = await modal({
+        title: 'Choose a PIN',
+        note: 'Four to twelve digits. You will be asked for it each time the dashboard opens.',
+        confirmLabel: 'Set PIN',
+        input: {
+          type: 'password',
+          inputmode: 'numeric',
+          maxlength: 12,
+          placeholder: '••••',
+          validate: (value) => (/^\d{4,12}$/.test(value) ? null : 'Four to twelve digits, numbers only.'),
+        },
+      });
+      if (pin === null) return;
+    }
+
+    try {
+      await api.setAuth(mode, pin);
+      window.location.reload();
+    } catch (error) {
+      await modal({
+        title: 'Could not change the lock',
+        note: error.body?.error ?? 'Something went wrong.',
+        confirmLabel: 'Close',
+        cancelLabel: 'Dismiss',
+      });
+    }
+  };
+
+  replace(page,
+    card('How the dashboard unlocks', [
+      options([
+        { id: 'open', name: 'Open', desc: 'No prompt. The link from the command line carries a one-time key.' },
+        {
+          id: 'system',
+          name: 'System',
+          desc: data.auth.system.available
+            ? `Ask ${data.auth.system.method}, falling back to your login password.`
+            : data.auth.system.reason,
+          disabled: !data.auth.system.available,
+        },
+        { id: 'pin', name: 'PIN', desc: 'Ask for a short PIN you choose.' },
+      ], data.auth.mode, setLock),
+      note('Changing this signs out every open dashboard session.'),
+    ]),
+
+    card('Privacy', [
+      options([
+        { id: 'redacted', name: 'Redacted', desc: 'Scrub secrets, keep the text. The default.' },
+        { id: 'full', name: 'Full', desc: 'Store prompts and paths exactly as written.' },
+        { id: 'metadata', name: 'Metadata', desc: 'Lengths and counts only. No text at all.' },
+      ], config.privacy, (value) => save('privacy', value)),
+      note('Redaction runs before anything is written. An append-only ledger has no take-backs, so this cannot be applied retroactively.'),
+    ]),
+
+    card('What is recorded', [
+      toggle('Tool calls', config.track.tools, (v) => save('track.tools', v), 'Most of the score is built from these.'),
+      toggle('Prompts', config.track.prompts, (v) => save('track.prompts', v)),
+      toggle('Permission requests', config.track.permissions, (v) => save('track.permissions', v)),
+      toggle('Tokens and cost', config.track.transcript, (v) => save('track.transcript', v), 'Read incrementally from the session transcript.'),
+      toggle('Git commits', config.track.git, (v) => save('track.git', v)),
+    ]),
+
+    card('Coaching', [
+      toggle('Enabled', config.coach.enabled, (v) => save('coach.enabled', v)),
+      toggle('Desktop notification', config.coach.channels.notification, (v) => save('coach.channels.notification', v)),
+      toggle('Card at session start', config.coach.channels.terminal, (v) => save('coach.channels.terminal', v)),
+      number('Most nudges per day', config.coach.maxPerDay, 1, 10, (v) => save('coach.maxPerDay', v)),
+      number('Cooldown per rule (hours)', config.coach.cooldownHours, 1, 168, (v) => save('coach.cooldownHours', v)),
+      note('Three dismissals mute a rule for good.'),
+    ]),
+
+    card('Storage', [
+      number('Keep records for (days)', config.retention.days, 0, 3650, (v) => save('retention.days', v), '0 means forever.'),
+      number('Compress segments after (days)', config.retention.gzipAfterDays, 0, 3650, (v) => save('retention.gzipAfterDays', v)),
+      note('Pruning appends a record naming what it removed before deleting anything, so the chain stays honest about its own gap.'),
+      h('div.kv', { style: 'margin-top:8px' }, [
+        kv('Location', data.paths.data),
+        kv('Hooks wired', `${data.hooks.count} events`),
+        kv('Notifications', data.notifications.available ? data.notifications.channel : 'unavailable here'),
+        kv('Unprocessed', String(data.spoolPending)),
+      ]),
+      h('div', { style: 'display:flex;gap:8px;flex-wrap:wrap;margin-top:4px' }, [
+        h('a.btn.btn--sm', { href: api.exportUrl('csv', 'all'), text: 'Export CSV' }),
+        h('a.btn.btn--sm', { href: api.exportUrl('json', 'all'), text: 'Export JSON' }),
+        h('a.btn.btn--sm', { href: api.exportUrl('ndjson', 'all'), text: 'Export NDJSON' }),
+      ]),
+    ]),
+
+    card('Only in the terminal', [
+      h('div.kv', {}, [
+        kv('Uninstall', 'syndes uninstall'),
+        kv('Delete all history', 'syndes uninstall --purge'),
+        kv('Rebuild metrics', 'syndes rebuild'),
+        kv('Check the chain', 'syndes verify --full'),
+      ]),
+      note('A web page that can delete your history is a web page that can be tricked into deleting it. These stay where they need a person at a keyboard.'),
+    ]),
+  );
+}
+
+// ── Controls ───────────────────────────────────────────────────────────────
+
+function options(list, current, onPick) {
+  return h('div', { style: 'display:grid;gap:8px' }, list.map((option) =>
+    h('button.optioncard', {
+      'aria-pressed': String(option.id === current),
+      disabled: option.disabled === true,
+      onclick: () => option.id !== current && !option.disabled && onPick(option.id),
+    }, [
+      h('div', {}, [
+        h('div.optioncard__name', { text: option.name }),
+        h('div.optioncard__desc', { text: option.desc }),
+      ]),
+      option.id === current ? h('span', { style: 'margin-left:auto;font-weight:700', text: '✓' }) : null,
+    ])));
+}
+
+function row(label, help, control) {
+  return h('div.setrow', {}, [
+    h('div', {}, [
+      h('div.setrow__label', { text: label }),
+      help ? h('div.setrow__help', { text: help }) : null,
+    ]),
+    h('div.setrow__control', {}, [control]),
+  ]);
+}
+
+function toggle(label, value, onChange, help = null) {
+  return row(label, help, h('button.btn.btn--sm', {
+    class: value ? 'btn--lime' : '',
+    style: 'min-width:58px;justify-content:center',
+    'aria-pressed': String(Boolean(value)),
+    text: value ? 'On' : 'Off',
+    onclick: () => onChange(!value),
+  }));
+}
+
+function number(label, value, min, max, onChange, help = null) {
+  return row(label, help, h('input.btn.btn--sm', {
+    type: 'number', value: String(value), min: String(min), max: String(max),
+    style: 'width:80px;text-align:right',
+    onchange: (event) => onChange(Number(event.target.value)),
+  }));
+}
+
+function note(text) {
+  return h('p', { style: 'font-size:11px;color:var(--ink-3);line-height:1.6;margin:0', text });
+}

package/ledger/append.mjs

@@ -0,0 +1,126 @@
+/**
+ * The only writer in the system. Called by the worker, under the lock.
+ *
+ * There is no update path and no delete path, because a ledger with an update
+ * path is a database. The single repair this module performs is recovering its
+ * own tip after a crash, and it does that by reading the chain rather than by
+ * trusting the cache.
+ */
+
+import { appendFileSync, writeFileSync, renameSync, existsSync } from 'node:fs';
+import { ensureDataDirs, tipFile, segmentFile } from '../runtime/paths.mjs';
+import { lastRecordOf } from '../runtime/jsonl.mjs';
+import { readJson } from '../runtime/config.mjs';
+import { ensureKey } from './keys.mjs';
+import { link } from './chain.mjs';
+import { draft, KIND } from './schema.mjs';
+import { dayOf, listDays, isSealed, computeSeal, writeSeal, fileForDay } from './segments.mjs';
+import { debug } from '../runtime/log.mjs';
+
+/**
+ * The chain tip, recovered from the chain itself rather than from the cache.
+ *
+ * state/tip.json is only a hint. If a worker was killed between writing records
+ * and writing the tip, the cache is behind — and reusing those seq numbers would
+ * fork the chain, which is the one corruption this design must not permit. So
+ * the newest segment's last record always wins.
+ *
+ * @returns {{seq: number, hash: string, day: string, torn: boolean}|null}
+ */
+export function loadTip() {
+  const cached = readJson(tipFile).data;
+  const days = listDays();
+  const newest = days[days.length - 1];
+
+  if (!newest) return cached ?? null;
+
+  const file = fileForDay(newest);
+  const { record, torn } = file ? lastRecordOf(file) : { record: null, torn: false };
+  if (!record || typeof record.seq !== 'number') return cached ?? null;
+
+  if (cached && cached.seq > record.seq) {
+    // The cache claims records the chain does not contain. Trust the chain.
+    debug('tip cache ahead of chain', cached.seq, '>', record.seq);
+  }
+  return { seq: record.seq, hash: record.hash, day: newest, torn };
+}
+
+function writeTip(tip) {
+  const staging = `${tipFile}.tmp`;
+  writeFileSync(staging, `${JSON.stringify(tip)}\n`);
+  renameSync(staging, tipFile); // atomic: never a half-written tip
+}
+
+/**
+ * Append drafts to the chain, in the order given.
+ *
+ * @param {object[]} drafts  from schema.draft()
+ * @returns {Promise<{written: number, from: number|null, to: number|null, tip: object|null}>}
+ */
+export async function append(drafts) {
+  if (!drafts.length) return { written: 0, from: null, to: null, tip: loadTip() };
+
+  ensureDataDirs();
+  const { key, keyId } = ensureKey();
+  let tip = loadTip();
+
+  const queue = [...drafts];
+  if (!tip && !existsSync(segmentFile(dayOf(queue[0].ts)))) {
+    queue.unshift(draft(KIND.GENESIS, {
+      ts: queue[0].ts,
+      data: { started: new Date(queue[0].ts).toISOString() },
+      key: keyId,
+    }));
+  }
+
+  let written = 0;
+  let from = null;
+
+  for (const run of groupByDay(queue)) {
+    // A day change closes the previous segment. The seal record is chained into
+    // the NEW day, so the closed day stays byte-identical to what was sealed.
+    if (tip?.day && tip.day !== run.day && !isSealed(tip.day)) {
+      const seal = await computeSeal(tip.day);
+      if (seal) {
+        const sealed = link(key, tip, draft(KIND.SEAL, { ts: run.records[0].ts, data: seal }));
+        appendFileSync(segmentFile(run.day), `${JSON.stringify(sealed)}\n`);
+        writeSeal(seal); // only after the record is durable
+        tip = { seq: sealed.seq, hash: sealed.hash, day: run.day };
+        written += 1;
+        if (from === null) from = sealed.seq;
+      }
+    }
+
+    const lines = [];
+    for (const record of run.records) {
+      const linked = link(key, tip, record);
+      lines.push(JSON.stringify(linked));
+      tip = { seq: linked.seq, hash: linked.hash, day: run.day };
+      if (from === null) from = linked.seq;
+      written += 1;
+    }
+
+    // One write per day-run: fewer syscalls, and O_APPEND keeps it a single
+    // atomic extend on POSIX. Only this process writes here, under the lock.
+    appendFileSync(segmentFile(run.day), `${lines.join('\n')}\n`);
+  }
+
+  writeTip(tip);
+  return { written, from, to: tip.seq, tip };
+}
+
+/** Convenience for callers with a single record. */
+export async function appendOne(record) {
+  return append([record]);
+}
+
+function groupByDay(records) {
+  const runs = [];
+  for (const record of records) {
+    const day = dayOf(record.ts);
+    const current = runs[runs.length - 1];
+    if (current && current.day === day) current.records.push(record);
+    else runs.push({ day, records: [record] });
+  }
+  return runs;
+}

package/ledger/chain.mjs

@@ -0,0 +1,53 @@
+/**
+ * The hash chain. hash = HMAC-SHA256(key, prev || "\n" || canonical(hashable)).
+ *
+ * The genesis record's `prev` is the zero digest. Every later record's `prev` is
+ * its predecessor's `hash`, and the chain continues ACROSS daily segments — the
+ * first record of a day carries the last hash of the previous day, so a deleted
+ * day is a visible break rather than a clean seam.
+ *
+ * This is what "immutable" reduces to in practice: any edit, deletion or
+ * back-date makes every subsequent hash wrong, and verify() names the exact seq
+ * where it stopped adding up.
+ */
+
+import { createHmac, createHash, timingSafeEqual } from 'node:crypto';
+import { canonical } from '../runtime/jsonl.mjs';
+import { hashable } from './schema.mjs';
+
+export const ZERO = '0'.repeat(64);
+
+export function computeHash(key, prev, record) {
+  return createHmac('sha256', key)
+    .update(prev)
+    .update('\n')
+    .update(canonical(hashable(record)))
+    .digest('hex');
+}
+
+/** Seal a draft into the chain. Returns a new object; the draft is not mutated. */
+export function link(key, tip, drafted) {
+  const prev = tip?.hash ?? ZERO;
+  const seq = (tip?.seq ?? -1) + 1;
+  const record = { ...drafted, seq, prev };
+  return { ...record, hash: computeHash(key, prev, record) };
+}
+
+/** Constant-time compare, because a hash check that leaks timing is theatre. */
+export function hashMatches(a, b) {
+  if (typeof a !== 'string' || typeof b !== 'string' || a.length !== b.length) return false;
+  return timingSafeEqual(Buffer.from(a, 'utf8'), Buffer.from(b, 'utf8'));
+}
+
+/**
+ * A segment root: the digest of every record hash in it, in order.
+ *
+ * Cheaper than a Merkle tree and sufficient for the claim being made — the seal
+ * proves the whole segment at once, and per-record proofs come from the chain
+ * itself, which every record already carries.
+ */
+export function segmentRoot(hashes) {
+  const digest = createHash('sha256');
+  for (const hash of hashes) digest.update(hash);
+  return digest.digest('hex');
+}

package/ledger/keys.mjs

@@ -0,0 +1,72 @@
+/**
+ * The chain key: 32 random bytes at keys/chain.key, mode 0600, made at install.
+ *
+ * HMAC rather than a bare SHA-256 digest, so forging a plausible chain requires
+ * reading the key and not merely owning a hash function everybody has.
+ *
+ * Keys are identified by a truncated digest of themselves and recorded at
+ * genesis and at every rotation, so verification checks each stretch of the
+ * chain against the key it was actually written with.
+ *
+ * On Windows the 0600 is a no-op. `doctor` says so plainly rather than printing
+ * a green check it did not earn.
+ */
+
+import { randomBytes, createHash } from 'node:crypto';
+import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync } from 'node:fs';
+import { join } from 'node:path';
+import { keysDir, chainKeyFile } from '../runtime/paths.mjs';
+
+export function keyIdOf(key) {
+  return createHash('sha256').update(key).digest('hex').slice(0, 16);
+}
+
+/** Create the key if it does not exist. @returns {{key: Buffer, keyId: string, created: boolean}} */
+export function ensureKey() {
+  mkdirSync(keysDir, { recursive: true, mode: 0o700 });
+
+  if (existsSync(chainKeyFile)) {
+    const key = readFileSync(chainKeyFile);
+    return { key, keyId: keyIdOf(key), created: false };
+  }
+
+  const key = randomBytes(32);
+  writeFileSync(chainKeyFile, key, { mode: 0o600 });
+  const keyId = keyIdOf(key);
+  writeFileSync(join(keysDir, `${keyId}.key`), key, { mode: 0o600 });
+  return { key, keyId, created: true };
+}
+
+export function loadKey() {
+  if (!existsSync(chainKeyFile)) return null;
+  const key = readFileSync(chainKeyFile);
+  return { key, keyId: keyIdOf(key) };
+}
+
+/**
+ * The key a given stretch of chain was written with.
+ *
+ * Archived under its own id at mint time, so a rotation never invalidates
+ * history: old records stay verifiable against the key that signed them.
+ */
+export function loadKeyById(keyId) {
+  const archived = join(keysDir, `${keyId}.key`);
+  if (existsSync(archived)) return readFileSync(archived);
+
+  const current = loadKey();
+  return current && current.keyId === keyId ? current.key : null;
+}
+
+/** Mint a new current key, keeping the old one readable. @returns {{keyId, previousKeyId}} */
+export function rotateKey() {
+  const previous = loadKey();
+  const key = randomBytes(32);
+  const keyId = keyIdOf(key);
+
+  writeFileSync(join(keysDir, `${keyId}.key`), key, { mode: 0o600 });
+  const staging = `${chainKeyFile}.new`;
+  writeFileSync(staging, key, { mode: 0o600 });
+  renameSync(staging, chainKeyFile); // atomic swap; never a window with no key
+
+  return { keyId, previousKeyId: previous?.keyId ?? null };
+}

package/ledger/read.mjs

@@ -0,0 +1,77 @@
+/**
+ * Streaming reads across segments, plain or gzipped.
+ *
+ * Everything is an async iterator. A heavy year of use is hundreds of megabytes
+ * and the dashboard must not need it resident to answer "how was last Tuesday".
+ */
+
+import { listDays, fileForDay, dayOf } from './segments.mjs';
+import { streamRecords, lastRecordOf } from '../runtime/jsonl.mjs';
+
+/** Every record, oldest first. */
+export async function* iterate() {
+  for (const day of listDays()) {
+    const file = fileForDay(day);
+    if (!file) continue;
+    yield* streamRecords(file);
+  }
+}
+
+/**
+ * Records with `from <= ts < to`.
+ *
+ * Segments are UTC and callers ask in local time, so the day window is widened
+ * by one on each side and the precise bound is applied per record.
+ */
+export async function* readRange(from, to) {
+  const firstDay = dayOf(from - 86_400_000);
+  const lastDay = dayOf(to + 86_400_000);
+
+  for (const day of listDays()) {
+    if (day < firstDay || day > lastDay) continue;
+    const file = fileForDay(day);
+    if (!file) continue;
+    for await (const record of streamRecords(file)) {
+      if (record.ts >= from && record.ts < to) yield record;
+    }
+  }
+}
+
+export async function* readDay(day) {
+  const file = fileForDay(day);
+  if (file) yield* streamRecords(file);
+}
+
+export async function readSession(sessionId) {
+  const out = [];
+  for await (const record of iterate()) {
+    if (record.session === sessionId) out.push(record);
+  }
+  return out;
+}
+
+/** The last n records. Walks segments backwards so a huge chain costs one file. */
+export async function tail(n = 50) {
+  const days = listDays().reverse();
+  const out = [];
+
+  for (const day of days) {
+    const file = fileForDay(day);
+    if (!file) continue;
+    const dayRecords = [];
+    for await (const record of streamRecords(file)) dayRecords.push(record);
+    out.unshift(...dayRecords.slice(Math.max(0, dayRecords.length - (n - out.length))));
+    if (out.length >= n) break;
+  }
+  return out.slice(-n);
+}
+
+/** Height and tip without reading the whole chain. */
+export function height() {
+  const days = listDays();
+  const newest = days[days.length - 1];
+  if (!newest) return { seq: -1, hash: null, days: 0 };
+  const file = fileForDay(newest);
+  const { record } = file ? lastRecordOf(file) : { record: null };
+  return { seq: record?.seq ?? -1, hash: record?.hash ?? null, days: days.length };
+}

package/ledger/retention.mjs

@@ -0,0 +1,104 @@
+/**
+ * Retention, without lying about it.
+ *
+ * Old segments may be gzipped, and may eventually be pruned — but pruning first
+ * appends a ledger.prune record naming the segments, their seq ranges and their
+ * seal roots, and only then removes the files. The chain therefore stays honest
+ * about its own gap, and verify() can tell "the user chose to drop 2024" from
+ * "someone deleted 2024".
+ *
+ * Default retention is forever. Deleting a user's history is opt-in.
+ */
+
+import { createReadStream, createWriteStream, unlinkSync, existsSync, statSync } from 'node:fs';
+import { createGzip } from 'node:zlib';
+import { pipeline } from 'node:stream/promises';
+import { listDays, fileForDay, isSealed, readSeal, segmentFile } from './segments.mjs';
+import { append } from './append.mjs';
+import { draft, KIND } from './schema.mjs';
+import { loadConfig } from '../runtime/config.mjs';
+import { debug } from '../runtime/log.mjs';
+
+const DAY_MS = 86_400_000;
+
+/** Compress sealed segments older than the threshold. Reads stay transparent. */
+export async function archive({ olderThanDays } = {}) {
+  const config = loadConfig();
+  const cutoffDays = olderThanDays ?? config.retention.gzipAfterDays;
+  if (!cutoffDays) return { archived: [] };
+
+  const cutoff = dayString(Date.now() - cutoffDays * DAY_MS);
+  const archived = [];
+
+  for (const day of listDays()) {
+    if (day >= cutoff) continue;
+    const plain = segmentFile(day);
+    if (!existsSync(plain) || !isSealed(day)) continue; // never compress an open day
+
+    try {
+      await pipeline(createReadStream(plain), createGzip({ level: 9 }), createWriteStream(`${plain}.gz`));
+      unlinkSync(plain);
+      archived.push(day);
+    } catch (error) {
+      debug('archive failed', day, error.message);
+      if (existsSync(`${plain}.gz`)) { try { unlinkSync(`${plain}.gz`); } catch { /* leave it */ } }
+    }
+  }
+  return { archived };
+}
+
+/**
+ * Delete segments older than the retention window — declaring it first.
+ *
+ * The prune record is appended and durable before a single byte is removed. If
+ * this process dies between the two, the chain over-declares a prune that did
+ * not happen, which verify reads as "nothing missing". Over-declaring is the
+ * safe direction; the other order would produce an unexplained hole.
+ */
+export async function prune({ olderThanDays, dryRun = false } = {}) {
+  const config = loadConfig();
+  const days = olderThanDays ?? config.retention.days;
+  if (!days) return { pruned: [], declared: false, reason: 'retention is set to forever' };
+
+  const cutoff = dayString(Date.now() - days * DAY_MS);
+  const doomed = listDays().filter((day) => day < cutoff && isSealed(day));
+  if (!doomed.length) return { pruned: [], declared: false, reason: 'nothing old enough' };
+
+  const ranges = [];
+  const roots = [];
+  for (const day of doomed) {
+    const seal = readSeal(day);
+    if (!seal) continue;
+    ranges.push([seal.first_seq, seal.last_seq]);
+    roots.push({ day, root: seal.root, count: seal.count });
+  }
+  if (dryRun) return { pruned: doomed, declared: false, ranges, roots, dryRun: true };
+
+  await append([draft(KIND.PRUNE, {
+    data: { days: doomed, ranges, roots, retentionDays: days, reason: 'retention' },
+  })]);
+
+  const removed = [];
+  for (const day of doomed) {
+    const file = fileForDay(day);
+    if (!file) continue;
+    try { unlinkSync(file); removed.push(day); } catch (error) { debug('prune failed', day, error.message); }
+  }
+  return { pruned: removed, declared: true, ranges, roots };
+}
+
+/** Bytes on disk, for `status` and the dashboard. */
+export function usage() {
+  let bytes = 0;
+  let segments = 0;
+  for (const day of listDays()) {
+    const file = fileForDay(day);
+    if (!file) continue;
+    try { bytes += statSync(file).size; segments += 1; } catch { /* raced with archive */ }
+  }
+  return { bytes, segments };
+}
+
+function dayString(ms) {
+  return new Date(ms).toISOString().slice(0, 10);
+}