synabun 1.3.0 → 2.0.0-beta

package/neural-interface/lib/backup-zip-reader.js

@@ -0,0 +1,262 @@
+import { createReadStream, createWriteStream, mkdirSync, renameSync, rmSync } from 'node:fs';
+import { open } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { Readable, Transform, Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { createInflateRaw, crc32 } from 'node:zlib';
+import { dirname } from 'node:path';
+
+/**
+ * Streaming ZIP reader for backup archives.
+ *
+ * AdmZip reads the whole archive into one Buffer, and Node refuses any file
+ * over 2 GiB (ERR_FS_FILE_TOO_LARGE), so every large backup failed its own
+ * verification and was deleted. This reader only buffers the central
+ * directory; entry data streams from disk through inflate. It trusts the
+ * central directory for sizes, so data descriptors (archiver writes them)
+ * need no special handling, and it reads ZIP64 records for archives, entries
+ * and offsets past 4 GiB.
+ */
+
+const SIG_EOCD = 0x06054b50;
+const SIG_ZIP64_LOCATOR = 0x07064b50;
+const SIG_ZIP64_EOCD = 0x06064b50;
+const SIG_CENTRAL = 0x02014b50;
+const SIG_LOCAL = 0x04034b50;
+const EOCD_MIN = 22;
+const EOCD_MAX_SCAN = EOCD_MIN + 0xffff;
+const MAX_CENTRAL_DIRECTORY_BYTES = 512 * 1024 * 1024;
+const MAX_MANIFEST_BYTES = 64 * 1024 * 1024;
+
+async function readAt(handle, position, length) {
+  const buffer = Buffer.alloc(length);
+  let offset = 0;
+  while (offset < length) {
+    const { bytesRead } = await handle.read(buffer, offset, length - offset, position + offset);
+    if (bytesRead === 0) throw new Error('Invalid ZIP: unexpected end of file');
+    offset += bytesRead;
+  }
+  return buffer;
+}
+
+function readUInt64(buffer, offset) {
+  const value = buffer.readBigUInt64LE(offset);
+  if (value > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('Invalid ZIP: 64-bit field out of range');
+  return Number(value);
+}
+
+async function locateCentralDirectory(handle, fileSize) {
+  if (fileSize < EOCD_MIN) throw new Error('Invalid ZIP: file is too small');
+  const scanLength = Math.min(fileSize, EOCD_MAX_SCAN);
+  const tailStart = fileSize - scanLength;
+  const tail = await readAt(handle, tailStart, scanLength);
+  let eocd = -1;
+  for (let index = tail.length - EOCD_MIN; index >= 0; index--) {
+    if (tail.readUInt32LE(index) === SIG_EOCD) { eocd = index; break; }
+  }
+  if (eocd < 0) throw new Error('Invalid ZIP: end of central directory not found');
+
+  let entryCount = tail.readUInt16LE(eocd + 10);
+  let size = tail.readUInt32LE(eocd + 12);
+  let offset = tail.readUInt32LE(eocd + 16);
+
+  // A ZIP64 locator sits immediately before the classic record whenever any
+  // of its fields overflowed; the classic fields then hold 0xFFFF sentinels.
+  const locatorAt = tailStart + eocd - 20;
+  if (locatorAt >= 0) {
+    const locator = await readAt(handle, locatorAt, 20);
+    if (locator.readUInt32LE(0) === SIG_ZIP64_LOCATOR) {
+      const zip64At = readUInt64(locator, 8);
+      const record = await readAt(handle, zip64At, 56);
+      if (record.readUInt32LE(0) !== SIG_ZIP64_EOCD) throw new Error('Invalid ZIP: ZIP64 end of central directory is corrupt');
+      entryCount = readUInt64(record, 32);
+      size = readUInt64(record, 40);
+      offset = readUInt64(record, 48);
+    }
+  }
+  if (size > MAX_CENTRAL_DIRECTORY_BYTES) throw new Error('Invalid ZIP: central directory is implausibly large');
+  if (offset + size > fileSize) throw new Error('Invalid ZIP: central directory is outside the file');
+  return { entryCount, size, offset };
+}
+
+function parseZip64Extra(extra, entry) {
+  let cursor = 0;
+  while (cursor + 4 <= extra.length) {
+    const id = extra.readUInt16LE(cursor);
+    const length = extra.readUInt16LE(cursor + 2);
+    const start = cursor + 4;
+    if (id === 0x0001) {
+      // Only the fields whose classic value overflowed are present, in order.
+      let field = start;
+      const take = () => {
+        if (field + 8 > start + length) throw new Error(`Invalid ZIP: truncated ZIP64 field for ${entry.name}`);
+        const value = readUInt64(extra, field);
+        field += 8;
+        return value;
+      };
+      if (entry.uncompressedSize === 0xffffffff) entry.uncompressedSize = take();
+      if (entry.compressedSize === 0xffffffff) entry.compressedSize = take();
+      if (entry.localHeaderOffset === 0xffffffff) entry.localHeaderOffset = take();
+      return;
+    }
+    cursor = start + length;
+  }
+}
+
+function parseCentralDirectory(buffer, expectedCount) {
+  const entries = [];
+  let cursor = 0;
+  while (cursor + 46 <= buffer.length) {
+    if (buffer.readUInt32LE(cursor) !== SIG_CENTRAL) break;
+    const flags = buffer.readUInt16LE(cursor + 8);
+    const nameLength = buffer.readUInt16LE(cursor + 28);
+    const extraLength = buffer.readUInt16LE(cursor + 30);
+    const commentLength = buffer.readUInt16LE(cursor + 32);
+    const nameStart = cursor + 46;
+    // Backups are written with ASCII/UTF-8 names; CP437 is not worth a table.
+    const name = buffer.toString('utf8', nameStart, nameStart + nameLength);
+    const entry = {
+      name,
+      flags,
+      method: buffer.readUInt16LE(cursor + 10),
+      crc32: buffer.readUInt32LE(cursor + 16),
+      compressedSize: buffer.readUInt32LE(cursor + 20),
+      uncompressedSize: buffer.readUInt32LE(cursor + 24),
+      localHeaderOffset: buffer.readUInt32LE(cursor + 42),
+      isDirectory: name.endsWith('/'),
+    };
+    parseZip64Extra(buffer.subarray(nameStart + nameLength, nameStart + nameLength + extraLength), entry);
+    entries.push(entry);
+    cursor = nameStart + nameLength + extraLength + commentLength;
+  }
+  // The 16-bit count wraps for >65535 entries without ZIP64; trust the walk
+  // unless it came up short of a count that cannot have wrapped.
+  if (entries.length < expectedCount) throw new Error('Invalid ZIP: central directory is truncated');
+  return entries;
+}
+
+/** Verify the entry's CRC-32 and size as bytes stream past. */
+function integrityCheck(entry) {
+  let crc = 0;
+  let bytes = 0;
+  return new Transform({
+    transform(chunk, _encoding, callback) {
+      bytes += chunk.length;
+      if (bytes > entry.uncompressedSize) {
+        callback(new Error(`Invalid ZIP: ${entry.name} is larger than declared`));
+        return;
+      }
+      crc = crc32(chunk, crc);
+      callback(null, chunk);
+    },
+    flush(callback) {
+      if (bytes !== entry.uncompressedSize) callback(new Error(`Invalid ZIP: ${entry.name} is truncated`));
+      else if ((crc >>> 0) !== entry.crc32) callback(new Error(`Invalid ZIP: CRC mismatch for ${entry.name}`));
+      else callback();
+    },
+  });
+}
+
+export async function openZipArchive(path) {
+  const handle = await open(path, 'r');
+  try {
+    const { size: fileSize } = await handle.stat();
+    const directory = await locateCentralDirectory(handle, fileSize);
+    const buffer = await readAt(handle, directory.offset, directory.size);
+    const entries = parseCentralDirectory(buffer, directory.entryCount);
+    const byName = new Map();
+    for (const entry of entries) if (!byName.has(entry.name)) byName.set(entry.name, entry);
+
+    async function dataRange(entry) {
+      const local = await readAt(handle, entry.localHeaderOffset, 30);
+      if (local.readUInt32LE(0) !== SIG_LOCAL) throw new Error(`Invalid ZIP: local header for ${entry.name} is corrupt`);
+      const start = entry.localHeaderOffset + 30 + local.readUInt16LE(26) + local.readUInt16LE(28);
+      if (start + entry.compressedSize > fileSize) throw new Error(`Invalid ZIP: ${entry.name} extends past the end of the file`);
+      return { start, length: entry.compressedSize };
+    }
+
+    return {
+      path,
+      size: fileSize,
+      entries,
+      getEntry: name => byName.get(name) || null,
+      /** Stream one entry's uncompressed bytes into `sink`, checking CRC and size. */
+      async pipeEntry(entry, sink) {
+        if (entry.flags & 0x1) throw new Error(`Unsupported ZIP: ${entry.name} is encrypted`);
+        if (entry.method !== 0 && entry.method !== 8) throw new Error(`Unsupported ZIP: ${entry.name} uses compression method ${entry.method}`);
+        const { start, length } = await dataRange(entry);
+        const stages = [length > 0
+          ? createReadStream(path, { start, end: start + length - 1, highWaterMark: 1024 * 1024 })
+          : Readable.from([])];
+        if (entry.method === 8) stages.push(createInflateRaw());
+        stages.push(integrityCheck(entry), sink);
+        await pipeline(stages);
+      },
+      close: () => handle.close(),
+    };
+  } catch (error) {
+    await handle.close().catch(() => {});
+    throw error;
+  }
+}
+
+export async function hashZipEntry(zip, entry) {
+  const digest = createHash('sha256');
+  await zip.pipeEntry(entry, new Writable({
+    write(chunk, _encoding, callback) { digest.update(chunk); callback(); },
+  }));
+  return digest.digest('hex');
+}
+
+export async function readZipEntry(zip, entry, { maxBytes = MAX_MANIFEST_BYTES } = {}) {
+  if (entry.uncompressedSize > maxBytes) throw new Error(`${entry.name} is too large to read into memory`);
+  const chunks = [];
+  await zip.pipeEntry(entry, new Writable({
+    write(chunk, _encoding, callback) { chunks.push(chunk); callback(); },
+  }));
+  return Buffer.concat(chunks);
+}
+
+/** Extract to a sibling temp file first so a failed CRC never leaves a partial target. */
+export async function extractZipEntry(zip, entry, targetPath) {
+  mkdirSync(dirname(targetPath), { recursive: true });
+  const temp = `${targetPath}.${process.pid}.restore-tmp`;
+  try {
+    await zip.pipeEntry(entry, createWriteStream(temp));
+    renameSync(temp, targetPath);
+  } catch (error) {
+    rmSync(temp, { force: true });
+    throw error;
+  }
+}
+
+/**
+ * Locate `<prefix>/manifest.json` and parse it. Only a top-level manifest
+ * counts: backed-up files can be named manifest.json too (a synced skill's
+ * global-skills/…/manifest.json), and they are written before the real one.
+ * Returns null when the archive has no manifest so callers can phrase their
+ * own error.
+ */
+export async function readBackupManifest(zip) {
+  const entry = zip.entries.find(item => !item.isDirectory && /^(?:[^/]+\/)?manifest\.json$/.test(item.name));
+  if (!entry) return null;
+  const prefix = entry.name.slice(0, -'manifest.json'.length).replace(/\/$/, '');
+  const manifest = JSON.parse((await readZipEntry(zip, entry)).toString('utf-8'));
+  return { entry, prefix, manifest };
+}
+
+export function backupEntryName(prefix, archivePath) {
+  return prefix ? `${prefix}/${archivePath}` : archivePath;
+}
+
+/** Stream every manifest-listed entry through sha256 and compare. */
+export async function verifyBackupChecksums(zip, { prefix, manifest, label = 'Backup verification failed' }) {
+  for (const [archivePath, expected] of Object.entries(manifest.checksums || {})) {
+    const entry = zip.getEntry(backupEntryName(prefix, archivePath));
+    if (!entry) throw new Error(`${label}: ${archivePath} is missing`);
+    let actual;
+    try { actual = `sha256:${await hashZipEntry(zip, entry)}`; }
+    catch (error) { throw new Error(`${label}: ${archivePath} is unreadable (${error.message})`); }
+    if (actual !== expected) throw new Error(`${label}: checksum mismatch for ${archivePath}`);
+  }
+}

package/neural-interface/lib/browser-assist-api.js

@@ -0,0 +1,41 @@
+/**
+ * Read-only browser-assist routes. Mounted under /api/browser after the
+ * request middleware, so deadlines, cancellation and the `actionStarted`
+ * stamp apply as they do to every other browser route. Nothing here marks an
+ * action as started: a caller may retry these freely.
+ *
+ * A Router factory rather than routes in server.js, so the route tests can
+ * mount the real thing beside their stubbed session map.
+ */
+import { Router } from 'express';
+import { runBrowserRequest, measureBrowserPhase } from './browser-execution.js';
+import { collectSemanticContext } from './browser-semantic-context.js';
+
+const PURPOSES = new Set(['navigation', 'page-state', 'empty-extractor', 'social', 'batch']);
+
+export function createBrowserAssistRouter({ browserSessions, getTargetPage, normalizeSelector = s => s }) {
+  const router = Router();
+
+  // POST /api/browser/sessions/:id/semantic-context  { tabId?, purpose?, scope?, social? }
+  router.post('/sessions/:id/semantic-context', async (req, res) => {
+    const session = browserSessions.get(req.params.id);
+    if (!session) return res.status(404).json({ error: 'Session not found' });
+    const routed = getTargetPage(session, req);
+    if (!routed) return res.status(404).json({ error: `Tab ${req.body?.tabId} not found in session` });
+    const { purpose, scope, social } = req.body || {};
+    try {
+      const context = await runBrowserRequest(req, () => measureBrowserPhase('semanticContext', () => collectSemanticContext(routed.page, {
+        purpose: PURPOSES.has(purpose) ? purpose : 'page-state',
+        scopeSel: typeof scope === 'string' && scope.trim() ? normalizeSelector(scope.trim()).slice(0, 300) : null,
+        social: social === true,
+      })), 4000);
+      res.json({ ok: true, context });
+    } catch (err) {
+      // Static text: a collector error can quote page content.
+      res.status(err?.code === 'BROWSER_REQUEST_CANCELLED' || err?.code === 'BROWSER_OPERATION_TIMEOUT' ? 408 : 500)
+        .json({ error: 'Semantic context unavailable.', code: err?.code || 'SEMANTIC_CONTEXT_FAILED' });
+    }
+  });
+
+  return router;
+}

package/neural-interface/lib/browser-console.js

@@ -0,0 +1,176 @@
+// ═══════════════════════════════════════════
+// SynaBun — browser console buffer (browser_console)
+// ═══════════════════════════════════════════
+//
+// A per-page ring buffer of console messages and uncaught page errors, so an
+// agent can read a page's errors through SynaBun's browser instead of opening
+// its own DevTools or Playwright. Attached wherever server.js registers a page
+// (the first tab of a session, new tabs, recovered tabs, their popups);
+// attaching twice is a no-op. A popup is not a tab of its session, so its
+// entries come back with the tab that opened it (browser_console on that tab),
+// each labelled with the popup's URL; the last MAX_POPUPS popups of a tab count.
+//
+// Playwright hands every console argument over as a JSHandle that stays alive
+// until the page navigates. Only the text is kept here, so the handles are
+// released at once: a long-lived tab of a chatty single-page app would
+// otherwise pin every logged object in the page and in this process.
+
+export const CONSOLE_BUFFER_LIMIT = 200;
+export const MAX_POPUPS = 5;
+const TEXT_LIMIT = 2000;
+const STACK_LIMIT = 4000;
+const LOCATION_URL_LIMIT = 300;
+// Popups of popups are read too, this deep.
+const POPUP_DEPTH = 3;
+// One order across every buffer, so a tab's entries and its popups' interleave as they happened.
+let arrival = 0;
+
+// Severity order; a read at one level includes every more severe one.
+export const CONSOLE_LEVELS = ['error', 'warning', 'info', 'log'];
+const RANK = { error: 0, warning: 1, info: 2, log: 3 };
+
+const buffers = new WeakMap();
+
+/** Playwright's console message type (log, debug, warning, assert, verbose…) as one of CONSOLE_LEVELS. */
+export function consoleLevel(type) {
+  const text = String(type || '').toLowerCase();
+  if (text === 'error' || text === 'assert') return 'error';
+  if (text === 'warning' || text === 'warn') return 'warning';
+  if (text === 'info') return 'info';
+  return 'log';
+}
+
+const clip = (text, max) => {
+  const value = String(text ?? '');
+  return value.length > max ? `${value.slice(0, max)}… (${value.length - max} more chars)` : value;
+};
+
+function formatLocation(location) {
+  if (!location || !location.url) return null;
+  const url = clip(location.url, LOCATION_URL_LIMIT);
+  const line = Number.isFinite(location.lineNumber) ? `:${location.lineNumber}` : '';
+  const column = line && Number.isFinite(location.columnNumber) ? `:${location.columnNumber}` : '';
+  return `${url}${line}${column}`;
+}
+
+function push(buffer, entry) {
+  buffer.seq += 1;
+  arrival += 1;
+  buffer.entries.push({ seq: buffer.seq, arrival, ...entry });
+  if (buffer.entries.length > buffer.limit) {
+    buffer.dropped += buffer.entries.length - buffer.limit;
+    buffer.entries.splice(0, buffer.entries.length - buffer.limit);
+  }
+}
+
+function pageUrl(page) {
+  try { return clip(page.url(), LOCATION_URL_LIMIT); } catch { return null; }
+}
+
+/**
+ * Start buffering `page`'s console and page errors (idempotent). Returns the
+ * buffer. `popup`: the page is a popup, so each entry carries its URL.
+ */
+export function attachConsoleBuffer(page, { limit = CONSOLE_BUFFER_LIMIT, now = Date.now, popup = false } = {}) {
+  if (!page || typeof page.on !== 'function') return null;
+  const existing = buffers.get(page);
+  if (existing) return existing;
+  const buffer = { entries: [], dropped: 0, seq: 0, limit, attachedAt: now(), popups: [] };
+  buffers.set(page, buffer);
+  const where = () => (popup ? { popup: pageUrl(page) } : {});
+  page.on('console', (message) => {
+    try {
+      const type = message.type();
+      push(buffer, {
+        kind: 'console',
+        level: consoleLevel(type),
+        type,
+        text: clip(message.text(), TEXT_LIMIT),
+        location: formatLocation(message.location()),
+        ...where(),
+        at: now(),
+      });
+    } catch { /* a message that cannot be read is skipped */ }
+    try { for (const handle of message.args()) handle.dispose().catch(() => {}); } catch {}
+  });
+  page.on('pageerror', (error) => {
+    push(buffer, {
+      kind: 'pageerror',
+      level: 'error',
+      message: clip(error?.message || String(error), TEXT_LIMIT),
+      stack: error?.stack ? clip(error.stack, STACK_LIMIT) : null,
+      ...where(),
+      at: now(),
+    });
+  });
+  // A popup a registered tab opens gets its own buffer from its first message,
+  // read with this tab's (popups are not tabs a caller can name).
+  page.on('popup', (child) => {
+    const childBuffer = attachConsoleBuffer(child, { limit, now, popup: true });
+    if (!childBuffer || buffer.popups.includes(childBuffer)) return;
+    buffer.popups.push(childBuffer);
+    if (buffer.popups.length > MAX_POPUPS) buffer.popups.splice(0, buffer.popups.length - MAX_POPUPS);
+  });
+  return buffer;
+}
+
+/** A buffer and its popups' buffers (and theirs, POPUP_DEPTH deep). */
+function withPopups(buffer, depth = 0, out = []) {
+  if (!buffer || out.includes(buffer)) return out;
+  out.push(buffer);
+  if (depth < POPUP_DEPTH) for (const child of buffer.popups || []) withPopups(child, depth + 1, out);
+  return out;
+}
+
+/** The buffer attached to `page`, or null. */
+export function consoleBufferOf(page) {
+  return (page && buffers.get(page)) || null;
+}
+
+/** `since` as epoch ms: a number, a numeric string, or an ISO date. NaN when unreadable. */
+export function parseConsoleSince(value) {
+  if (value === undefined || value === null || value === '') return null;
+  if (typeof value === 'number') return value;
+  const text = String(value).trim();
+  if (/^\d+(?:\.\d+)?$/.test(text)) return Number(text);
+  const parsed = Date.parse(text);
+  return Number.isNaN(parsed) ? NaN : parsed;
+}
+
+/**
+ * Read a page's buffer, with the entries of the popups it opened (each with
+ * `popup`: that popup's URL). `level` (error | warning | info | log | all)
+ * keeps that level and every more severe one; `since` (epoch ms) keeps entries
+ * after it; `limit` returns the newest N of what matched, oldest first; `clear`
+ * empties the buffers after reading.
+ */
+export function readConsoleBuffer(page, { level = 'all', since = null, limit = 50, clear = false } = {}) {
+  const buffer = consoleBufferOf(page);
+  if (!buffer) return { attached: false, entries: [], matched: 0, total: 0, dropped: 0 };
+  const wanted = level === 'all' || !level ? RANK.log : RANK[level];
+  if (wanted === undefined) throw new Error(`level must be one of all, ${CONSOLE_LEVELS.join(', ')}`);
+  const sources = withPopups(buffer);
+  const all = sources.flatMap((source) => source.entries).sort((a, b) => a.arrival - b.arrival);
+  const matching = all.filter((entry) => RANK[entry.level] <= wanted && (since === null || entry.at > since));
+  const cap = Math.max(1, Math.min(CONSOLE_BUFFER_LIMIT, Math.floor(Number(limit) || 50)));
+  const entries = matching.slice(-cap).map(({ seq, arrival: _arrival, at, ...rest }) => ({ ...rest, at: new Date(at).toISOString() }));
+  const popups = sources.length - 1;
+  const result = {
+    attached: true,
+    entries,
+    matched: matching.length,
+    total: all.length,
+    dropped: sources.reduce((sum, source) => sum + source.dropped, 0),
+    bufferingSince: new Date(buffer.attachedAt).toISOString(),
+    latest: all.length ? new Date(Math.max(...all.map((entry) => entry.at))).toISOString() : null,
+    ...(popups && { popups }),
+  };
+  if (clear) {
+    for (const source of sources) {
+      source.entries.length = 0;
+      source.dropped = 0;
+    }
+    result.cleared = true;
+  }
+  return result;
+}