poltertab 1.4.0 → 1.5.0-beta.1

package/mcp-server/config.js

@@ -0,0 +1,85 @@
+// Every tunable the Node half has, in one place, because several of them are
+// only correct relative to each other and that relationship was previously
+// unwritten — three copies of a 35s timeout in one file, a port in five.
+//
+// ── The timeout budget ──────────────────────────────────────────────────────
+//
+// A DOM command crosses four processes, and each layer waits on the one below:
+//
+//   MCP server   COMMAND_TIMEOUT_MS         35s   ← must be the largest
+//     extension  navigation wait            30s   + 500ms settle  (background.js)
+//       frame    postToFrame                10s                   (background.js)
+//         page   ELEMENT_WAIT_MS             3s                   (content_script.js)
+//
+// The invariant: **each layer's budget must exceed the sum of what it waits
+// on.** Break it and a slow-but-successful page load surfaces at the top as a
+// timeout, which the caller can only answer by guessing whether to retry — the
+// single most expensive wrong answer this system can give, because the retry
+// re-runs a scrape that already worked.
+//
+// The extension's half of these numbers cannot be required from here: it runs
+// in Chrome, there is no build step, and MV3 forbids remote code. They are
+// mirrored at the top of background.js with a pointer back to this comment. If
+// you change one, change both.
+
+const os = require("os");
+const path = require("path");
+
+// Anything the user accumulates lives outside the package. Under a global npm
+// install __dirname resolves inside node_modules/poltertab/, so site memory
+// written next to the code is destroyed by the next `npm update -g` — the
+// upgrade would read as amnesia. Downloads have it worse: output_file exists to
+// keep large payloads out of the context window, and burying them in
+// node_modules makes them hard to find and just as easy to lose.
+//
+// POLTERTAB_HOME exists so the test suite can point this somewhere disposable
+// instead of writing into the real one.
+const POLTERTAB_HOME =
+  process.env.POLTERTAB_HOME || path.join(os.homedir(), ".poltertab");
+
+// Port: MCP_BROWSER_WS_PORT env, then --port CLI arg, then the default.
+const DEFAULT_WS_PORT = 7822;
+function resolvePort(argv = process.argv, env = process.env) {
+  const i = argv.indexOf("--port");
+  if (i !== -1 && argv[i + 1]) return parseInt(argv[i + 1], 10);
+  if (env.MCP_BROWSER_WS_PORT) return parseInt(env.MCP_BROWSER_WS_PORT, 10);
+  return DEFAULT_WS_PORT;
+}
+
+module.exports = {
+  POLTERTAB_HOME,
+  MEMORY_DIR: path.join(POLTERTAB_HOME, "navigation_memory"),
+  DOWNLOADS_DIR: path.join(POLTERTAB_HOME, "downloads"),
+
+  DEFAULT_WS_PORT,
+  resolvePort,
+
+  // See the budget above before changing this.
+  COMMAND_TIMEOUT_MS: 35000,
+
+  // Dead-connection detection. A socket that misses a ping/pong round is
+  // terminated so the extension's own reconnect can take over.
+  HEARTBEAT_MS: 60000,
+
+  // The capture buffer is a bounded tail, not a log. Both caps exist to stop a
+  // chatty tab from turning the bridge into a memory leak: the server holds
+  // response bodies for tabs it may never be asked about again.
+  NETWORK_TTL_MS: 5 * 60 * 1000,
+  NETWORK_GC_INTERVAL_MS: 60000,
+  NETWORK_MAX_REQUESTS: 500,
+  NETWORK_MAX_BODY_BYTES: 1024 * 1024,
+
+  // Promotion race: every orphaned Secondary tries to bind the freed port at
+  // once, so they stagger themselves rather than all failing but one.
+  PROMOTION_DELAY_MS: 500,
+  PROMOTION_JITTER_MS: 2000,
+
+  // More concurrent agents than this sharing one browser stops being useful and
+  // starts being a way to lose track of which tab belongs to whom.
+  MAX_SECONDARIES: 5,
+
+  // Lazy-loaded content arrives after the scroll returns. Long enough for the
+  // XHR it triggered to land in the capture buffer, short enough that a
+  // scroll-and-read loop is not dominated by waiting.
+  SMART_SCROLL_SETTLE_MS: 2000,
+};

package/mcp-server/extract-all.js

@@ -0,0 +1,159 @@
+// browser_extract_all: paginate a site and return records, so the model stops
+// being the for-loop.
+//
+// sendCommand is injected rather than required, so this module has no opinion
+// about whether it is talking to an extension directly or proxying through a
+// Primary node — and so the suite can drive it with a scripted transport.
+
+const { isBlank } = require("./output.js");
+
+// --- The pagination loop ---
+//
+// This exists so the model stops being the for-loop. It supplies the spec once
+// and receives records; every page in between costs a browser round-trip
+// instead of a model round-trip.
+//
+// Every halt condition below is a case where continuing would produce data
+// that looks complete and is not, so each one names itself in the result. A
+// paused run costs far less than a confidently wrong dataset.
+async function extractAll(sendCommand, args) {
+  const {
+    url_template,
+    record,
+    fields,
+    anchor,
+    key,
+    limit = 200,
+    offset = 0,
+    start_page = 1,
+    max_pages = 50,
+    fill_tolerance = 0.5,
+    max_text,
+    session,
+  } = args;
+
+  if (!url_template.includes("{page}")) {
+    throw new Error("url_template must contain a {page} placeholder");
+  }
+
+  const spec = { record, fields, anchor, max_text, session, probe: true };
+  const target = offset + limit;
+  const rows = [];
+  const seen = new Set();
+  const warnings = [];
+  const pages = [];
+
+  let baseline = null;
+  let stopped_because = "max_pages";
+  let page = start_page;
+  let fetched = 0;
+
+  const keyOf = (row) =>
+    key ? String(row[key] ?? "") : JSON.stringify(row);
+
+  while (fetched < max_pages) {
+    const url = url_template.replace("{page}", String(page));
+    const nav = await sendCommand("navigate", { url, session });
+    fetched++;
+
+    const res = await sendCommand("extract", spec);
+    const pageRows = (res && res.rows) || [];
+    pages.push({
+      page,
+      url: nav && nav.url,
+      found: res ? res.records_found : 0,
+      kept: pageRows.length,
+      dropped: res ? res.dropped : 0,
+    });
+    if (res && res.warnings && res.warnings.length) {
+      warnings.push(`page ${page}: ${res.warnings.join("; ")}`);
+    }
+
+    if (!pageRows.length) {
+      stopped_because = "empty_page";
+      break;
+    }
+
+    // A site that ignores an unrecognised page param answers every request with
+    // page 1. Identical content reads as real data, which is how a "next 100"
+    // silently becomes the same 12 records eight times over.
+    const fresh = pageRows.filter((r) => !seen.has(keyOf(r)));
+    if (!fresh.length) {
+      stopped_because = "duplicate_page";
+      warnings.push(
+        `page ${page} returned only records already seen — pagination is not advancing`,
+      );
+      break;
+    }
+
+    // Fill rates against the first page. A spec learned on page 1 degrades
+    // quietly later: variant card layouts, a column that stops being populated.
+    // Halting beats emitting rows that are 40% empty.
+    const ratios = {};
+    for (const [name, n] of Object.entries(res.fill_rates || {})) {
+      ratios[name] = n / pageRows.length;
+    }
+    if (!baseline) {
+      baseline = ratios;
+    } else if (fill_tolerance > 0) {
+      const collapsed = Object.keys(baseline).filter(
+        (name) =>
+          baseline[name] >= 0.5 &&
+          ratios[name] < baseline[name] * fill_tolerance,
+      );
+      if (collapsed.length) {
+        stopped_because = "fill_rate_deviation";
+        warnings.push(
+          `page ${page}: ${collapsed
+            .map(
+              (n) =>
+                `${n} ${(ratios[n] * 100).toFixed(0)}% vs baseline ${(baseline[n] * 100).toFixed(0)}%`,
+            )
+            .join(", ")} — page layout likely differs from the learned spec`,
+        );
+        break;
+      }
+    }
+
+    for (const r of fresh) {
+      seen.add(keyOf(r));
+      rows.push(r);
+    }
+
+    if (rows.length >= target) {
+      stopped_because = "limit_reached";
+      break;
+    }
+    page++;
+  }
+
+  const sliced = rows.slice(offset, offset + limit);
+  if (offset && fetched) {
+    warnings.push(
+      `offset ${offset} was reached by fetching from page ${start_page}; pass start_page to skip pages instead of re-reading them`,
+    );
+  }
+
+  // extract reports fill as counts, so extract_all does too — the per-page
+  // baseline is a fraction because it is compared across pages of differing
+  // size, and carries the unit in its name rather than looking like a count.
+  const fill_rates = {};
+  for (const name of Object.keys(fields)) {
+    fill_rates[name] = sliced.filter((r) => !isBlank(r[name])).length;
+  }
+
+  return {
+    rows: sliced,
+    count: sliced.length,
+    collected: rows.length,
+    pages_fetched: fetched,
+    last_page: page,
+    stopped_because,
+    fill_rates,
+    baseline_fill_ratios: baseline,
+    pages,
+    warnings,
+  };
+}
+
+module.exports = { extractAll };