poltertab 1.6.0-beta.1 → 1.6.0-beta.2

package/mcp-server/extract-all.js

@@ -6,6 +6,7 @@
 // Primary node — and so the suite can drive it with a scripted transport.
 
 const { isBlank } = require("./output.js");
+const { collapsed, ratiosOf } = require("./recipes.js");
 
 // --- The pagination loop ---
 //
@@ -89,22 +90,19 @@ async function extractAll(sendCommand, args) {
     // Fill rates against the first page. A spec learned on page 1 degrades
     // quietly later: variant card layouts, a column that stops being populated.
     // Halting beats emitting rows that are 40% empty.
-    const ratios = {};
-    for (const [name, n] of Object.entries(res.fill_rates || {})) {
-      ratios[name] = n / pageRows.length;
-    }
+    //
+    // The staleness rule itself lives in recipes.js, because a stored recipe's
+    // baseline has to be judged by exactly the same threshold as a paginated
+    // run's.
+    const ratios = ratiosOf(res);
     if (!baseline) {
       baseline = ratios;
-    } else if (fill_tolerance > 0) {
-      const collapsed = Object.keys(baseline).filter(
-        (name) =>
-          baseline[name] >= 0.5 &&
-          ratios[name] < baseline[name] * fill_tolerance,
-      );
-      if (collapsed.length) {
+    } else {
+      const stale = collapsed(baseline, ratios, fill_tolerance);
+      if (stale.length) {
         stopped_because = "fill_rate_deviation";
         warnings.push(
-          `page ${page}: ${collapsed
+          `page ${page}: ${stale
             .map(
               (n) =>
                 `${n} ${(ratios[n] * 100).toFixed(0)}% vs baseline ${(baseline[n] * 100).toFixed(0)}%`,

package/mcp-server/index.js

@@ -20,7 +20,19 @@ const {
   recordSelector,
   noteSelectorFail,
   selectorKey,
+  pathPattern,
+  listRecipeSummaries,
 } = require("./memory.js");
+const {
+  hydrate,
+  observe,
+  noteStep,
+  noteNavigation,
+  redactValue,
+  hintFor,
+  describeRecipes,
+  RECIPES_NOTE,
+} = require("./recipes.js");
 const { extractAll } = require("./extract-all.js");
 const bridge = require("./bridge.js");
 const OWN_VERSION = require("./../package.json").version;
@@ -98,6 +110,65 @@ function pageFor(args) {
   return lastPage;
 }
 
+// The step buffer's key, resolved by exactly the same rule as pageFor: an
+// explicit session or tab gets its own and never falls through to the
+// process-global one. Without that, two concurrent tasks in two tabs borrow
+// each other's preamble — the O13 failure, one layer up.
+function targetKeyFor(args) {
+  const a = args || {};
+  if (a.session) return `session:${a.session}`;
+  if (a.tabId != null) return `tab:${a.tabId}`;
+  return "last";
+}
+
+// Actions worth remembering as a recipe's preamble: the facet click, the
+// "100 per page", the scroll that loaded the list.
+const RECORDED = new Set(["click", "fill", "hover", "scroll", "smart_scroll"]);
+
+// Bookkeeping for an action that has already happened, so it can never throw
+// at the caller. A fill's value goes through redactValue and nowhere else: the
+// store is a plaintext file in the user's home directory.
+function noteStepFor(action, args, targetKey, page, result) {
+  try {
+    const { value, redacted } = redactValue(
+      action,
+      args,
+      result && result.fingerprint,
+    );
+    noteStep(targetKey, {
+      action,
+      selector: (args && args.selector) || null,
+      path: page && page.path,
+      host: page && page.host,
+      // Nothing at all for a redacted fill, not the null redactValue reports:
+      // a stored `value: null` reads back as "this field was filled with
+      // nothing", which is a different step from the one that happened.
+      value: redacted ? undefined : value,
+      redacted,
+    });
+  } catch (_) {
+    // A preamble we failed to record is a recipe with fewer steps, not a
+    // failed action.
+  }
+}
+
+// Recall has to be implicit — the tool surface is already 23 schemas on every
+// request — so a navigate says what has been learned about where it landed.
+// One memory read, and nothing at all on a host with no recipes.
+function onNavigate(args, targetKey, result) {
+  try {
+    const dest = pageOf(result && result.url);
+    if (!dest) return result;
+    noteNavigation(targetKey, dest.host);
+    const known = listRecipeSummaries(dest.host, pathPattern(dest.path));
+    if (!known.length) return result;
+    return { ...result, recipes_available: hintFor(known) };
+  } catch (_) {
+    // The navigation succeeded; a missing hint is the lesser evil.
+    return result;
+  }
+}
+
 const handleToolCall = async (request) => {
   const { name, arguments: args } = request.params;
 
@@ -151,6 +222,16 @@ const handleToolCall = async (request) => {
       const scrollResult = await bridge.sendCommand("scroll", args || {});
       bridge.noteActiveTab(scrollResult && scrollResult.tabId);
 
+      // The scroll that loaded the list is part of the preamble a recipe
+      // replays, and this branch returns before the generic path can see it.
+      noteStepFor(
+        "smart_scroll",
+        args,
+        targetKeyFor(args),
+        pageFor(args),
+        scrollResult,
+      );
+
       // Wait for network requests to arrive (lazy loading)
       await new Promise((r) => setTimeout(r, SMART_SCROLL_SETTLE_MS));
 
@@ -187,12 +268,18 @@ const handleToolCall = async (request) => {
     if (action === "get_site_memory") {
       const host = args.hostname || args.domain || args.url;
       if (!host) throw new Error("Missing 'hostname' parameter");
-      // The agent only wants its own notes; the selectors map is internal
-      // plumbing for self-healing and would just be noise here.
+      // The agent's own notes, plus what it worked out about extracting the
+      // site. The selectors map stays out: it is internal plumbing for
+      // self-healing and would just be noise here.
+      const memory = readMemory(host);
+      const payload = { notes: memory.notes };
+      const learned = describeRecipes(memory.recipes);
+      if (learned.length) {
+        payload.recipes = learned;
+        payload.recipes_note = RECIPES_NOTE;
+      }
       return {
-        content: [
-          { type: "text", text: JSON.stringify(readMemory(host).notes, null, 2) },
-        ],
+        content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
       };
     }
 
@@ -207,12 +294,24 @@ const handleToolCall = async (request) => {
 
     // Loops in the server, not in the model. One tool call covers every page.
     if (action === "extract_all") {
-      const payload = await extractAll(bridge.sendCommand, args || {});
       const opts = args || {};
+      const targetKey = targetKeyFor(opts);
+      // The page a recipe belongs to comes from the template, not from the tab:
+      // this tool drives its own navigation, so the tab can be anywhere. The
+      // inner sendCommand("extract", spec) calls always carry an explicit spec
+      // and so are never hydrated.
+      const page = pageOf(String(opts.url_template || "").replace("{page}", "1"));
+      const ctx = hydrate("extract_all", opts, page, targetKey);
+
+      const payload = await extractAll(bridge.sendCommand, opts);
+      const patch = ctx
+        ? observe({ action, args: opts, result: payload, ctx, targetKey })
+        : null;
+      const out = patch ? { ...payload, ...patch } : payload;
 
       if (opts.output_file) {
-        const written = writeOutput(opts.output_file, payload, payload.rows);
-        const { rows, pages, ...rest } = payload;
+        const written = writeOutput(opts.output_file, out, out.rows);
+        const { rows, pages, ...rest } = out;
         const summary = {
           ...rest,
           ...written,
@@ -227,7 +326,7 @@ const handleToolCall = async (request) => {
       }
 
       return {
-        content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
+        content: [{ type: "text", text: JSON.stringify(out, null, 2) }],
       };
     }
 
@@ -237,8 +336,13 @@ const handleToolCall = async (request) => {
     // by the action itself — see memory.selectorKey.
     const healable = action === "click" || action === "fill";
     const selector = args && args.selector;
-    const page = healable && selector ? pageFor(args) : null;
-    const key = page ? selectorKey(action, page.path, selector) : null;
+    // Resolved for every action, not just a healable click: the recipe layer
+    // is keyed by the same page the selector store is.
+    const page = pageFor(args);
+    const key =
+      healable && selector && page
+        ? selectorKey(action, page.path, selector)
+        : null;
     if (key && !args.fingerprint) {
       // Reading the store must never stop a click that would otherwise work.
       try {
@@ -249,6 +353,17 @@ const handleToolCall = async (request) => {
       }
     }
 
+    // Recipes live in the generic path on purpose: hydrate a missing extract
+    // spec before the command goes out, judge what came back after. A second
+    // `if (action === "extract")` branch would need its own copy of
+    // output_file, tab tracking and page learning. Throwing before sendCommand
+    // is deliberate too — an extract with no record selector is not a cheap
+    // mistake, it returns the page's whole text as one row.
+    const targetKey = targetKeyFor(args);
+    // targetKey, not just the page: hydrate matches a missing `variant`
+    // against the steps this server watched the caller issue on this target.
+    const ctx = hydrate(action, args, page, targetKey);
+
     let result;
     try {
       result = await bridge.sendCommand(action, args || {});
@@ -279,17 +394,52 @@ const handleToolCall = async (request) => {
       // The action succeeded; losing the bookkeeping is the lesser evil.
     }
 
+    // Recorded only once the browser has confirmed the action, so a stored
+    // preamble never replays a click that never landed.
+    if (RECORDED.has(action)) {
+      noteStepFor(action, args, targetKey, page, result);
+    }
+
+    // browser_fill echoes back the value it typed. Keeping a password out of
+    // the store is no use if the echo puts it in the model's context and from
+    // there into the transcript, which is the exposure the redaction denylist
+    // exists to close. Only a fill redactValue calls sensitive is touched — a
+    // plain fill echoing its value is worth reading. The extension still sends
+    // the value over the local WebSocket, so this narrows where a secret lands
+    // rather than making the fill secret end to end.
+    if (action === "fill" && result && typeof result === "object") {
+      try {
+        const { redacted } = redactValue(action, args, result.fingerprint);
+        if (redacted && "value" in result) {
+          result = { ...result, value: "(value withheld)" };
+        }
+      } catch (_) {
+        // Same contract as noteStepFor: the fill has already happened.
+      }
+    }
+
+    if (action === "navigate") result = onNavigate(args, targetKey, result);
+
+    // Patched before the output_file branch below: a run that writes to disk
+    // is exactly the run where the model cannot see the rows for itself, so
+    // losing used_recipe/stale there would hide a replay entirely.
+    const patch = ctx
+      ? observe({ action, args, result, ctx, targetKey })
+      : null;
+    if (patch) result = { ...result, ...patch };
+
     // Any read tool can send its payload to disk. Placed after tab tracking so
     // taking the file path does not cost the session its tab bookkeeping.
     if (args && args.output_file && result && typeof result === "object") {
       const rows = rowsOf(result);
       const written = writeOutput(args.output_file, result, rows);
+      const summary = summarizeOutput(result, rows, written);
       return {
         content: [
           {
             type: "text",
             text: JSON.stringify(
-              summarizeOutput(result, rows, written),
+              patch ? { ...summary, ...patch } : summary,
               null,
               2,
             ),

package/mcp-server/memory.js

@@ -69,21 +69,40 @@ function memoryFile(rawHost) {
 // bound; selectors likewise, evicting the least-recently-used first.
 const NOTES_CAP = 100;
 const SELECTORS_CAP = 200;
+// A learned recipe carries a whole extract spec plus replay steps per variant,
+// so it is orders of magnitude larger than a selector entry — cap it far
+// tighter, and count across path patterns because the file size is what the
+// cap actually protects.
+const RECIPES_CAP = 20;
+// And per recipe: a site with many facet combinations learns one variant per
+// combination under a single name, and evictRecipes counts recipes rather than
+// variants, so without this one recipe on one path pattern grows without
+// bound. Eight is enough slices of one task to be worth keeping.
+const VARIANTS_CAP = 8;
 // A selector that keeps missing even with its fingerprint has drifted past
 // recognition — stop trusting it rather than relocating against a dead signature.
 const MAX_FAILS = 3;
 
 // Normalize any on-disk shape — including the original bare array — to the
-// current { notes, selectors } form. Old files upgrade in place on the next
-// write, so there is no migration step.
+// current { notes, selectors, recipes } form. Old files upgrade in place on the
+// next write, so there is no migration step.
 function normalize(raw) {
-  if (Array.isArray(raw)) return { notes: raw, selectors: {} };
+  if (Array.isArray(raw)) return { notes: raw, selectors: {}, recipes: {} };
   return {
     notes: raw && Array.isArray(raw.notes) ? raw.notes : [],
     selectors:
       raw && raw.selectors && typeof raw.selectors === "object"
         ? raw.selectors
         : {},
+    // An array here would survive the typeof check and then hand out numeric
+    // recipe names from Object.keys, so reject anything but a plain object.
+    recipes:
+      raw &&
+      raw.recipes &&
+      typeof raw.recipes === "object" &&
+      !Array.isArray(raw.recipes)
+        ? raw.recipes
+        : {},
   };
 }
 
@@ -124,9 +143,13 @@ function writeMemory(rawHost, data) {
 // The selector store is bookkeeping for an action that already happened, so a
 // failed write must never surface as a failed click — the agent would retry and
 // double-submit. saveMemory stays strict: it IS the user's action.
+//
+// Whatever fn returned comes back out, so noteRecipeFail can report what it
+// deleted. A throw reads as undefined, which is the right answer either way: a
+// write that failed deleted nothing.
 function bestEffort(fn) {
   try {
-    fn();
+    return fn();
   } catch (_) {
     // Read-only or full home: keep the healing we cannot persist to ourselves.
   }
@@ -196,6 +219,179 @@ function evictSelectors(selectors) {
     .forEach((k) => delete selectors[k]);
 }
 
+// ── Learned extraction recipes ──────────────────────────────────────────────
+//
+// A recipe is what the agent worked out about a listing page: the record
+// selector and field map that produced clean rows, and — per variant, a slice
+// of the same task with its own filters — the steps that got there and the
+// fill rate to expect next time. Keyed by path pattern → recipe name →
+// variant, so /jobs/12345 and /jobs/99 share what was learned once.
+
+const UUID_SEGMENT =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+// Volatile path segments have to collapse or every record page teaches a recipe
+// that nothing ever reads back: /jobs/12345 is the same page shape as /jobs/99,
+// and a recipe learned on one is what makes the other free.
+function pathPattern(rawPath) {
+  let p = rawPath == null ? "" : String(rawPath);
+
+  // Callers pass a pathname, but what the page reported is often the full URL.
+  if (p.includes("://")) {
+    try {
+      p = new URL(p).pathname;
+    } catch {
+      // Not parseable as a URL; the query/hash strip below still applies.
+    }
+  }
+  p = p.split("#")[0].split("?")[0];
+
+  const segments = p
+    .split("/")
+    .filter(Boolean)
+    .map((seg) => {
+      if (/^\d+$/.test(seg)) return ":n";
+      if (UUID_SEGMENT.test(seg)) return ":id";
+      // A long run of letters and digits with no separator is an opaque id.
+      // Slugs are excluded by the separator rule on purpose: "senior-engineer-2024"
+      // names a page shape, and collapsing it would merge unrelated pages into
+      // one recipe.
+      if (seg.length >= 12 && /\d/.test(seg) && /^[a-z0-9]+$/i.test(seg))
+        return ":id";
+      return seg;
+    });
+
+  return segments.length ? `/${segments.join("/")}` : "/";
+}
+
+// A recipe is as fresh as its freshest variant: one dead filter combination
+// must not make the whole learned spec look stale to eviction.
+function recipeLastOk(recipe) {
+  const variants = recipe && recipe.variants ? recipe.variants : {};
+  return Object.keys(variants).reduce(
+    (max, k) => Math.max(max, (variants[k] && variants[k].lastOk) || 0),
+    0,
+  );
+}
+
+function getRecipes(rawHost, pattern) {
+  return readMemory(rawHost).recipes[pattern] || {};
+}
+
+function getRecipe(rawHost, pattern, name) {
+  return getRecipes(rawHost, pattern)[name] || null;
+}
+
+// Same contract as recordSelector: the extract that taught us this already
+// returned its rows, so a failed write must not surface as a failed extract —
+// the agent would re-run the whole crawl.
+function putRecipe(rawHost, pattern, name, recipe) {
+  if (!recipe) return;
+  bestEffort(() => {
+    const data = readMemory(rawHost);
+    const bucket = data.recipes[pattern] || (data.recipes[pattern] = {});
+    const existing = bucket[name];
+    bucket[name] = {
+      extract: recipe.extract || (existing && existing.extract),
+      // Merge: variants of one task are learned one run at a time, so writing
+      // the second must not forget the first.
+      variants: {
+        ...(existing && existing.variants),
+        ...(recipe.variants || {}),
+      },
+    };
+    evictVariants(bucket[name].variants);
+    evictRecipes(data.recipes);
+    writeMemory(rawHost, data);
+  });
+}
+
+// What was deleted, so the caller can say so: a recipe that vanished under the
+// agent is a recipe it will expect to be there next time. Undefined when the
+// write failed — see bestEffort.
+function noteRecipeFail(rawHost, pattern, name, variant) {
+  return bestEffort(() => {
+    const data = readMemory(rawHost);
+    const bucket = data.recipes[pattern];
+    const recipe = bucket && bucket[name];
+    const entry = recipe && recipe.variants && recipe.variants[variant];
+    if (!entry) return { variant: false, recipe: false, fails: 0 };
+    entry.failCount = (entry.failCount || 0) + 1;
+    const dropped = { variant: false, recipe: false, fails: entry.failCount };
+    if (entry.failCount >= MAX_FAILS) {
+      delete recipe.variants[variant];
+      dropped.variant = true;
+      // Unwind the empty levels above it, or the file accumulates husks that
+      // every later read has to page in and every hint has to filter out.
+      if (!Object.keys(recipe.variants).length) {
+        delete bucket[name];
+        dropped.recipe = true;
+      }
+      if (!Object.keys(bucket).length) delete data.recipes[pattern];
+    }
+    writeMemory(rawHost, data);
+    return dropped;
+  });
+}
+
+// Oldest slice of one task first, by the same rule as evictSelectors. Keyed on
+// lastOk rather than on insertion order because the variant nothing has
+// replayed in weeks is the one worth losing.
+function evictVariants(variants) {
+  const keys = Object.keys(variants || {});
+  if (keys.length <= VARIANTS_CAP) return;
+  keys
+    .sort(
+      (a, b) =>
+        ((variants[a] && variants[a].lastOk) || 0) -
+        ((variants[b] && variants[b].lastOk) || 0),
+    )
+    .slice(0, keys.length - VARIANTS_CAP)
+    .forEach((k) => delete variants[k]);
+}
+
+// Bound the store the way evictSelectors does, but across path patterns: a
+// crawler that walks a thousand listing URLs would otherwise learn a recipe per
+// pattern and grow one host's file without limit.
+function evictRecipes(recipes) {
+  const entries = [];
+  for (const pattern of Object.keys(recipes))
+    for (const name of Object.keys(recipes[pattern]))
+      entries.push({
+        pattern,
+        name,
+        lastOk: recipeLastOk(recipes[pattern][name]),
+      });
+  if (entries.length <= RECIPES_CAP) return;
+  entries
+    .sort((a, b) => a.lastOk - b.lastOk)
+    .slice(0, entries.length - RECIPES_CAP)
+    .forEach(({ pattern, name }) => {
+      delete recipes[pattern][name];
+      if (!Object.keys(recipes[pattern]).length) delete recipes[pattern];
+    });
+}
+
+// What a recipe offers, without what it costs: the hint that tells an agent a
+// recipe exists is worth nothing if quoting it costs as much context as the
+// extract it saves, so the field map and the replay steps stay on disk.
+function listRecipeSummaries(rawHost, pattern) {
+  const bucket = getRecipes(rawHost, pattern);
+  return Object.keys(bucket).map((name) => {
+    const recipe = bucket[name];
+    const fields = recipe && recipe.extract && recipe.extract.fields;
+    return {
+      name,
+      fields:
+        fields && typeof fields === "object" && !Array.isArray(fields)
+          ? Object.keys(fields)
+          : [],
+      variants: recipe && recipe.variants ? Object.keys(recipe.variants) : [],
+      lastOk: recipeLastOk(recipe),
+    };
+  });
+}
+
 module.exports = {
   memoryFile,
   selectorKey,
@@ -204,4 +400,12 @@ module.exports = {
   getSelector,
   recordSelector,
   noteSelectorFail,
+  RECIPES_CAP,
+  VARIANTS_CAP,
+  pathPattern,
+  getRecipes,
+  getRecipe,
+  putRecipe,
+  noteRecipeFail,
+  listRecipeSummaries,
 };