synabun 1.3.0 → 2.0.0-beta

package/neural-interface/lib/browser-execution.js

@@ -0,0 +1,241 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { randomUUID } from 'node:crypto';
+import { performance } from 'node:perf_hooks';
+
+const requests = new AsyncLocalStorage();
+export const browserV2Enabled = () => process.env.SYNABUN_BROWSER_V2 !== '0';
+
+export async function measureBrowserPhase(name, work) {
+  const state = requests.getStore();
+  const start = performance.now();
+  try { return await work(); }
+  finally {
+    if (state) {
+      state.phases[name] = (state.phases[name] || 0) + performance.now() - start;
+      state.counts[name] = (state.counts[name] || 0) + 1;
+    }
+  }
+}
+
+export function assertBrowserRequestActive(req) {
+  const state = req?.browserRequest || requests.getStore();
+  if (state && (state.signal.aborted || Date.now() >= state.deadline)) {
+    const error = new Error(state.actionStarted
+      ? 'Browser request expired after execution began; outcome is uncertain. Inspect the page before retrying.'
+      : 'Browser request cancelled before execution.');
+    error.code = 'BROWSER_REQUEST_CANCELLED';
+    throw error;
+  }
+}
+
+export function browserActionTimeout(req, fallback = 5000) {
+  assertBrowserRequestActive(req);
+  const state = req?.browserRequest || requests.getStore();
+  return state ? Math.max(1, Math.min(fallback, state.deadline - Date.now())) : fallback;
+}
+
+export function markBrowserActionStarted(req) {
+  assertBrowserRequestActive(req);
+  const state = req?.browserRequest || requests.getStore();
+  if (state) state.actionStarted = true;
+}
+
+/** Bound probes/queues without replaying their underlying operation. Every later
+ * mutation still checks request liveness, since Promise.race cannot cancel CDP. */
+export async function runBrowserRequest(req, work, fallback = 10000) {
+  assertBrowserRequestActive(req);
+  const state = req?.browserRequest || requests.getStore();
+  let timer, abort;
+  try {
+    return await Promise.race([
+      Promise.resolve().then(work),
+      new Promise((_, reject) => {
+        const fail = () => reject(Object.assign(new Error('Browser operation deadline exceeded; inspect state before retrying.'), { code: 'BROWSER_OPERATION_TIMEOUT' }));
+        timer = setTimeout(fail, Math.max(1, Math.min(fallback, state ? state.deadline - Date.now() : fallback)));
+        abort = fail;
+        state?.signal.addEventListener('abort', abort, { once: true });
+        if (state?.signal.aborted) abort();
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+    if (abort) state?.signal.removeEventListener('abort', abort);
+  }
+}
+
+export function cancellableBrowserDelay(ms, req) {
+  assertBrowserRequestActive(req);
+  const state = req?.browserRequest || requests.getStore();
+  return new Promise((resolve, reject) => {
+    const cleanup = () => state?.signal.removeEventListener('abort', abort);
+    const abort = () => {
+      clearTimeout(timer);
+      cleanup();
+      try { assertBrowserRequestActive(req); } catch (error) { reject(error); }
+    };
+    const timer = setTimeout(() => { cleanup(); resolve(); }, ms);
+    state?.signal.addEventListener('abort', abort, { once: true });
+    if (state?.signal.aborted) abort();
+  });
+}
+
+/** Metadata only: never logs URLs, selectors, form values, scripts or response bodies. */
+export function browserRequestMiddleware(req, res, next) {
+  const controller = new AbortController();
+  const start = performance.now();
+  const supplied = req.get('X-Synabun-Request-Id');
+  const requestId = /^[\w-]{1,80}$/.test(supplied || '') ? supplied : randomUUID();
+  const requestedDeadline = Number(req.get('X-Synabun-Deadline'));
+  const deadline = Number.isFinite(requestedDeadline) && requestedDeadline > 0
+    ? Math.min(requestedDeadline, Date.now() + 120_000) : Date.now() + 120_000;
+  const state = { requestId, deadline, signal: controller.signal, actionStarted: false, phases: {}, counts: {} };
+  req.browserRequest = state;
+  res.setHeader('X-Synabun-Request-Id', requestId);
+  let responseBytes = 0;
+  let estimatedTextTokens = 0;
+  const send = res.send.bind(res);
+  res.send = (body) => {
+    if (typeof body === 'string' || Buffer.isBuffer(body)) responseBytes = Buffer.byteLength(body);
+    return send(body);
+  };
+  const json = res.json.bind(res);
+  res.json = (body) => {
+    if (res.headersSent || res.destroyed) return res;
+    if (body?.error) {
+      body = { ...body, actionStarted: state.actionStarted, ...(state.actionStarted ? { outcome: 'uncertain' } : {}) };
+      if (!state.actionStarted && body.error === 'Session not found') body.code = 'SESSION_NOT_FOUND';
+      if (!state.actionStarted && /^Tab .* not found in session$/.test(body.error)) body.code = 'TAB_NOT_FOUND';
+    }
+    if (process.env.SYNABUN_BROWSER_METRICS === '1' && body && typeof body === 'object') {
+      const { data: _data, image: _image, screenshot: _screenshot, ...textFields } = body;
+      estimatedTextTokens = Math.ceil(JSON.stringify(textFields).length / 4);
+    }
+    const began = performance.now();
+    const result = json(body);
+    state.phases.serialization = performance.now() - began;
+    return result;
+  };
+  const timer = setTimeout(() => {
+    controller.abort();
+    if (!res.headersSent && !res.destroyed) {
+      res.status(408).json({ error: state.actionStarted
+        ? 'Browser deadline exceeded after execution began; outcome uncertain. Inspect before retrying.'
+        : 'Browser deadline exceeded before execution.', code: 'BROWSER_DEADLINE_EXCEEDED' });
+    }
+  }, Math.max(1, deadline - Date.now()));
+  const abort = () => controller.abort();
+  req.once('aborted', abort);
+  let logged = false;
+  const finish = () => {
+    clearTimeout(timer);
+    req.removeListener('aborted', abort);
+    if (!res.writableFinished) controller.abort();
+    if (!logged && process.env.SYNABUN_BROWSER_METRICS === '1') {
+      logged = true;
+      const operation = String(req.route?.path || req.path).split('/').pop();
+      console.error('[browser-metric]', JSON.stringify({
+        requestId, operation: /^[\w-]+$/.test(operation) ? operation : 'session',
+        durationMs: performance.now() - start, phases: state.phases, counts: state.counts,
+        responseBytes, estimatedTextTokens, status: res.statusCode,
+        actionStarted: state.actionStarted, cancelled: controller.signal.aborted,
+      }));
+    }
+  };
+  res.once('finish', finish);
+  res.once('close', finish);
+  requests.run(state, () => {
+    if (deadline <= Date.now()) {
+      controller.abort();
+      return res.status(408).json({ error: 'Browser deadline exceeded before execution.', code: 'BROWSER_DEADLINE_EXCEEDED' });
+    }
+    next();
+  });
+}
+
+/** Session-wide write queue: cancelled waiters never execute or consume a write slot. */
+export function createEngagementScheduler({ now = Date.now, random = Math.random, delay = cancellableBrowserDelay } = {}) {
+  const states = new Map();
+  return {
+    clear(key) { states.delete(key); },
+    async run(key, req, execute) {
+      const state = states.get(key) || { tail: Promise.resolve(), lastWriteAt: null };
+      states.set(key, state);
+      const predecessor = state.tail;
+      let release;
+      state.tail = new Promise(resolve => { release = resolve; });
+      try {
+        await predecessor;
+        assertBrowserRequestActive(req);
+        const gap = 30_000 + Math.floor(random() * 60_001);
+        const remaining = state.lastWriteAt === null ? 0 : Math.max(0, state.lastWriteAt + gap - now());
+        if (remaining) await measureBrowserPhase('intentionalPacing', () => delay(remaining, req));
+        assertBrowserRequestActive(req);
+        // Time from completion conservatively includes preparation and uncertain
+        // attempts. A wedged probe must not permanently hold the session queue.
+        try { return await runBrowserRequest(req, execute, 120_000); }
+        finally {
+          const request = req?.browserRequest || requests.getStore();
+          if (!request || request.actionStarted) state.lastWriteAt = now();
+        }
+      } finally { release(); }
+    },
+  };
+}
+
+const ENGAGE_WRITE_RE = /tweetButton|data-testid=["']?(like|unlike|retweet|unretweet)\b|-follow\b|:has-text\(["'](Follow|Seguir|Segui|Folgen|Suivre)["']\)/i;
+
+/**
+ * The selector-string half of "is this an engagement write". The write queue
+ * keys on the selector text, so `a:has-text("Follow")` queues even when the
+ * resolved element says nothing; anything that must never queue checks both.
+ */
+export function isEngagementWriteSelector(selector) {
+  return typeof selector === 'string' && ENGAGE_WRITE_RE.test(selector);
+}
+
+const X_HOST_RE = /^(?:[a-z0-9-]+\.)*(?:x|twitter)\.com$/i;
+function isXPage(url) {
+  try { return X_HOST_RE.test(new URL(String(url)).hostname); } catch { return false; }
+}
+
+/**
+ * Cmd/Ctrl+Enter in an X composer publishes without clicking tweetButton, which
+ * skips the publish gate and the write queue. Loops used it when the button was
+ * covered by an overlay; the /press route refuses it on loop-owned tabs.
+ */
+export function isXShortcutPublish(key, url) {
+  return typeof key === 'string'
+    && /(?:^|\+)(?:Meta|Control|ControlOrMeta|Ctrl|Cmd|Command)\+(?:.*\+)?Enter$/i.test(key.trim())
+    && isXPage(url);
+}
+
+const X_SCRIPT_TARGET_RE = /tweetButton|confirmationSheetConfirm|["'[=](?:like|unlike|retweet|unretweet)["'\]]|-follow\b/i;
+const X_SCRIPT_ACTION_RE = /\.click\s*\(|dispatchEvent\s*\(|requestSubmit\s*\(|\.submit\s*\(/;
+
+/**
+ * A browser_evaluate script that clicks a publish/like/repost/follow control on X
+ * is the other gate bypass (the "JS .click() on an intercepted tweetButton" trick).
+ * Reads that only query those elements stay allowed.
+ */
+export function isXScriptedWrite(script, url) {
+  return typeof script === 'string'
+    && X_SCRIPT_TARGET_RE.test(script)
+    && X_SCRIPT_ACTION_RE.test(script)
+    && isXPage(url);
+}
+
+/** Read semantics from the resolved element, including ref-based targets. */
+export async function engagementTargetSelector(target) {
+  return target.evaluate(el => {
+    for (let node = el; node; node = node.parentElement) {
+      const testid = node.getAttribute('data-testid') || '';
+      const label = (node.getAttribute('aria-label') || node.textContent || '').trim();
+      if (/^(tweetButton(?:Inline)?|like|unlike|retweet|unretweet)$/.test(testid) || /-follow$/.test(testid)) {
+        return '[data-testid="' + testid + '"]';
+      }
+      if ((node.tagName === 'BUTTON' || node.getAttribute('role') === 'button')
+        && /^(Follow|Seguir|Segui|Folgen|Suivre)$/.test(label)) return 'button:has-text("Follow")';
+    }
+    return '';
+  });
+}

package/neural-interface/lib/browser-extraction.js

@@ -0,0 +1,191 @@
+// One server operation owns each scroll/readiness/extract cycle. Nothing here
+// retries a scroll after an uncertain browser failure.
+export const EXTRACTION_TIMEOUT_MS = 15_000;
+
+const identity = (item, keys) => {
+  for (const key of keys) if (item[key] !== null && item[key] !== undefined && item[key] !== '') return `${key}:${item[key]}`;
+  return JSON.stringify(item);
+};
+
+function extractionError(message, code) {
+  return Object.assign(new Error(message), { code });
+}
+
+/** Apply the output budget to complete items, after deduplication, preserving JSON. */
+export function budgetExtractionItems(items, { fields, maxItems = 50, maxChars } = {}) {
+  const selected = [];
+  let chars = 2; // JSON array brackets
+  let reason;
+  for (const item of items) {
+    if (selected.length >= maxItems) { reason = 'max_items'; break; }
+    const value = fields ? Object.fromEntries(fields.filter(key => Object.hasOwn(item, key)).map(key => [key, item[key]])) : item;
+    const size = JSON.stringify(value).length + (selected.length ? 1 : 0);
+    if (maxChars !== undefined && chars + size > maxChars) { reason = 'max_chars'; break; }
+    selected.push(value);
+    chars += size;
+  }
+  return { items: selected, truncated: selected.length < items.length, budgetReason: reason };
+}
+
+/**
+ * Expressions are the same trusted extraction scripts accepted by browser_evaluate.
+ * runtime supports an HTTP disconnect signal and the request's absolute deadline.
+ */
+export async function runBrowserExtraction(page, script, opts = {}, runtime = {}) {
+  if (typeof script !== 'string' || !script.trim()) throw new Error('script required');
+  const scrolls = Math.min(Math.max(Math.trunc(opts.scrolls ?? 0), 0), 10);
+  const maxItems = Math.max(1, Math.trunc(opts.maxItems || 50));
+  const dedupeKeys = Array.isArray(opts.dedupeKeys) ? opts.dedupeKeys : [];
+  const timeoutMs = Math.min(Math.max(opts.timeoutMs || EXTRACTION_TIMEOUT_MS, 1), 30_000);
+  const deadline = Math.min(Date.now() + timeoutMs, runtime.deadline ?? Infinity);
+  const signal = runtime.signal;
+  const measure = runtime.measure || ((_name, fn) => fn());
+  const collected = new Map();
+  let raw = 0;
+  let scrollsUsed = 0;
+  let stopReason = 'scroll_limit';
+  let failureReason;
+  let dryRounds = 0;
+  let bonusRounds = 0;
+
+  function active() {
+    if (signal?.aborted) throw extractionError('Extraction cancelled; collected items are preserved.', 'cancelled');
+    if (Date.now() >= deadline) throw extractionError('Extraction deadline reached; collected items are preserved.', 'deadline');
+  }
+  async function bounded(name, fn) {
+    active();
+    let timer;
+    let abort;
+    try {
+      return await Promise.race([
+        measure(name, fn),
+        new Promise((_, reject) => {
+          timer = setTimeout(() => reject(extractionError('Extraction deadline reached; collected items are preserved.', 'deadline')), Math.max(1, deadline - Date.now()));
+          abort = () => reject(extractionError('Extraction cancelled; collected items are preserved.', 'cancelled'));
+          signal?.addEventListener('abort', abort, { once: true });
+          if (signal?.aborted) abort();
+        }),
+      ]);
+    } finally {
+      clearTimeout(timer);
+      signal?.removeEventListener('abort', abort);
+    }
+  }
+  function merge(items) {
+    if (!Array.isArray(items)) throw new Error('Extractor must return an array of objects.');
+    raw += items.length;
+    const before = collected.size;
+    for (const item of items) {
+      if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error('Extractor must return an array of objects.');
+      const key = identity(item, dedupeKeys);
+      // Map replacement retains discovery order while refreshing counts/text
+      // when a previously observed identity finishes hydration or changes.
+      collected.set(key, item);
+    }
+    return collected.size - before;
+  }
+
+  try {
+    let items = await bounded('extraction', () => page.evaluate(script));
+    for (let round = 0; ; round++) {
+      active();
+      const added = merge(items);
+      const budget = budgetExtractionItems([...collected.values()], opts);
+      if (budget.budgetReason) { stopReason = budget.budgetReason; break; }
+      if (collected.size >= maxItems) { stopReason = 'max_items'; break; }
+      if (opts.minItems && collected.size >= opts.minItems) { stopReason = 'min_items'; break; }
+      if (opts.scrollIfEmpty && round === 0 && collected.size === 0) bonusRounds = 1;
+      dryRounds = added ? 0 : dryRounds + 1;
+      if (dryRounds >= 2) { stopReason = 'end_of_feed'; break; }
+      if (round >= scrolls + bonusRounds) break;
+
+      const distance = (opts.scrollDistance ?? 1200) * (opts.scrollDirection ?? 1);
+      const scrollTarget = opts.scrollTarget || 'window';
+      const loadingBefore = await bounded('action', () => page.evaluate(({ scrollTarget, distance }) => {
+        const target = (0, eval)(scrollTarget);
+        if (!target || typeof target.scrollBy !== 'function') throw new Error('Scroll container not found.');
+        const root = target === window ? document : target;
+        const busy = '[aria-busy="true"], [role="progressbar"]';
+        const loading = !!(root.matches?.(busy) || root.querySelector(busy));
+        target.scrollBy(0, distance);
+        return loading;
+      }, { scrollTarget, distance }));
+      scrollsUsed++;
+      // Poll inside the renderer, not over HTTP. Reuse the first newly visible
+      // extraction so virtualized items cannot disappear between wait and read.
+      const waitMs = Math.min(opts.settleMs ?? 3000, Math.max(1, deadline - Date.now()));
+      try {
+        items = await bounded('readiness', async () => {
+          const handle = await page.waitForFunction(state => {
+            const { script, dedupeKeys, seen, scrollTarget } = state;
+            const items = (0, eval)(script);
+            if (!Array.isArray(items)) throw new Error('Extractor must return an array of objects.');
+            const known = new Set(seen);
+            const hasNew = items.some(item => {
+              if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error('Extractor must return an array of objects.');
+              let key;
+              for (const field of dedupeKeys) {
+                if (item[field] !== null && item[field] !== undefined && item[field] !== '') { key = `${field}:${item[field]}`; break; }
+              }
+              return !known.has(key ?? JSON.stringify(item));
+            });
+            const target = (0, eval)(scrollTarget);
+            const root = target === window ? document : target;
+            if (!root) throw new Error('Scroll container disappeared while waiting.');
+            const busy = '[aria-busy="true"], [role="progressbar"]';
+            const loading = !!(root.matches?.(busy) || root.querySelector(busy));
+            // Playwright reuses this argument between predicate polls. Remember
+            // loading that begins after scroll dispatch as well as before it.
+            state.loadingBefore ||= loading;
+            const loadingEnded = state.loadingBefore && !loading;
+            return !loading && (hasNew || loadingEnded) ? { items } : false;
+          }, { script, dedupeKeys, seen: [...collected.keys()], scrollTarget, loadingBefore }, { timeout: waitMs, polling: 50 });
+          try { return (await handle.jsonValue()).items; } finally { await handle.dispose(); }
+        });
+      } catch (error) {
+        if (error.name !== 'TimeoutError') throw error;
+        // Read once at the readiness boundary, but never accept known loading
+        // skeletons merely because the readiness timer elapsed.
+        const finalRead = await bounded('extraction', () => page.evaluate(({ script, scrollTarget }) => {
+          const target = (0, eval)(scrollTarget);
+          const root = target === window ? document : target;
+          if (!root) throw new Error('Scroll container disappeared while waiting.');
+          const busy = '[aria-busy="true"], [role="progressbar"]';
+          const loading = !!(root.matches?.(busy) || root.querySelector(busy));
+          return { loading, items: loading ? [] : (0, eval)(script) };
+        }, { script, scrollTarget }));
+        if (finalRead.loading) throw extractionError('Feed is still loading after the readiness deadline; collected items are preserved.', 'readiness_timeout');
+        items = finalRead.items;
+      }
+    }
+  } catch (error) {
+    stopReason = ['deadline', 'cancelled', 'readiness_timeout'].includes(error.code) ? error.code : 'extraction_error';
+    failureReason = error.message;
+  }
+
+  const budget = budgetExtractionItems([...collected.values()], opts);
+  return {
+    ...budget, raw, scrollsUsed, stopReason: budget.budgetReason || stopReason,
+    partial: !!failureReason && collected.size > 0,
+    ...(failureReason ? { failureReason, ...(collected.size === 0 ? { error: failureReason } : {}) } : {}),
+  };
+}
+
+/** Scoped HTML reader: clone before cleaning, and retain link source URLs. */
+export async function readBrowserContentHtml(page, selector) {
+  const target = page.locator(selector || 'body');
+  if (selector && await target.count() !== 1) throw new Error('Content selector must match exactly one element.');
+  return target.evaluate((element, scoped) => {
+    const clone = element.cloneNode(true);
+    const noise = scoped
+      ? 'script,style,noscript,iframe'
+      : 'nav,header,footer,aside,[role="navigation"],[role="banner"],[role="contentinfo"],[role="complementary"],.sidebar,.nav,.footer,.header,.advertisement,.ad,.ads,[class*="cookie"],[class*="popup"],[class*="modal"],[class*="overlay"],script,style,noscript,iframe';
+    clone.querySelectorAll(noise).forEach(node => node.remove());
+    clone.querySelectorAll('a[href],img[src]').forEach(node => {
+      const attr = node.tagName === 'A' ? 'href' : 'src';
+      try { node.setAttribute(attr, new URL(node.getAttribute(attr), document.baseURI).href); } catch { /* Keep unresolvable links verbatim. */ }
+    });
+    const main = scoped ? clone : (clone.querySelector('main') || clone.querySelector('article') || clone.querySelector('[role="main"]') || clone);
+    return scoped ? main.outerHTML : main.innerHTML;
+  }, !!selector);
+}