poltertab 1.4.0 → 1.5.0-beta.2

package/mcp-server/memory.js

@@ -0,0 +1,81 @@
+// Site memory: the notes an agent leaves itself about a domain, keyed by
+// hostname. The key arrives from a model, so it is untrusted input rather than
+// a filename — see memoryFile.
+
+const fs = require("fs");
+const path = require("path");
+const { MEMORY_DIR } = require("./config.js");
+
+fs.mkdirSync(MEMORY_DIR, { recursive: true });
+
+// Installs predating the move kept memory beside the code. Copy it forward once
+// so an upgrade does not look like the agent forgot everything it learned.
+// Never overwrite: if both sides have a note for a domain, the one already in
+// the new location is the live one.
+(() => {
+  const legacy = path.join(__dirname, "navigation_memory");
+  if (legacy === MEMORY_DIR || !fs.existsSync(legacy)) return;
+  let copied = 0;
+  for (const name of fs.readdirSync(legacy)) {
+    const to = path.join(MEMORY_DIR, name);
+    if (!name.endsWith(".json") || fs.existsSync(to)) continue;
+    try {
+      fs.copyFileSync(path.join(legacy, name), to);
+      copied++;
+    } catch (_) {
+      // A read-only or half-removed legacy dir is not worth failing startup.
+    }
+  }
+  if (copied) {
+    console.error(
+      `[PolterTab MCP] Migrated ${copied} site memory file(s) to ${MEMORY_DIR}`,
+    );
+  }
+})();
+
+// Site memory is keyed by hostname, and that key arrives from a model — so it
+// is untrusted input rather than a filename. Two failures this closes: a note
+// saved under kw.com was invisible to a lookup for www.kw.com (the same site),
+// and the raw value was interpolated straight into a path, so "../.." reached
+// outside MEMORY_DIR.
+function memoryFile(rawHost) {
+  let host = String(rawHost).trim().toLowerCase();
+
+  // The parameter is also documented as accepting `url`, so a full URL turning
+  // up here is expected rather than a caller mistake.
+  if (host.includes("/")) {
+    try {
+      host = new URL(host.includes("://") ? host : `https://${host}`).hostname;
+    } catch {
+      host = host.split("/")[0];
+    }
+  }
+
+  host = host.replace(/[^a-z0-9.-]/g, "").replace(/^\.+/, "");
+  if (!host) throw new Error(`Not a usable hostname: ${rawHost}`);
+
+  // Existing notes live under whichever spelling first created them — the store
+  // already holds both kw.com.json and www.linkedin.com.json — so try the
+  // variants before concluding this is a new file.
+  const bare = host.replace(/^www\./, "");
+  for (const name of [bare, host, `www.${bare}`]) {
+    const p = path.join(MEMORY_DIR, `${name}.json`);
+    if (fs.existsSync(p)) return p;
+  }
+  return path.join(MEMORY_DIR, `${bare}.json`);
+}
+
+function readMemory(rawHost) {
+  const file = memoryFile(rawHost);
+  return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, "utf8")) : [];
+}
+
+function saveMemory(rawHost, obstacle, solution) {
+  const file = memoryFile(rawHost);
+  const data = readMemory(rawHost);
+  data.push({ obstacle, solution, timestamp: Date.now() });
+  fs.writeFileSync(file, JSON.stringify(data, null, 2));
+  return data.length;
+}
+
+module.exports = { memoryFile, readMemory, saveMemory };

package/mcp-server/output.js

@@ -0,0 +1,103 @@
+// Writing a payload to disk instead of into the context window, and the CSV /
+// JSONL encoders that make the file the deliverable rather than something the
+// agent has to re-type.
+
+const fs = require("fs");
+const path = require("path");
+const { DOWNLOADS_DIR } = require("./config.js");
+
+// --- Output handling ---
+//
+// The raw payload must never be the default path into the context window. Two
+// costs dominated a 100-record scrape and neither was the data: envelopes
+// several lines of JSON deep to carry one short string, and the agent then
+// re-typing every record by hand into a file. Writing straight to disk removes
+// both.
+
+function toCsv(rows) {
+  if (!rows.length) return "";
+  const cols = [...new Set(rows.flatMap((r) => Object.keys(r)))];
+  const cell = (v) => {
+    if (v === null || v === undefined) return "";
+    const s = Array.isArray(v)
+      ? v.join(" | ")
+      : typeof v === "object"
+        ? JSON.stringify(v) // beats a column of "[object Object]"
+        : String(v);
+    // Quote only when it would otherwise break the row, and double any
+    // embedded quote — the two mistakes that produce a file which imports
+    // silently and wrongly.
+    return /[",\r\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+  };
+  return [
+    cols.join(","),
+    ...rows.map((r) => cols.map((c) => cell(r[c])).join(",")),
+  ].join("\n");
+}
+
+const isBlank = (v) =>
+  v === null || v === undefined || v === "" || (Array.isArray(v) && !v.length);
+
+// `rows` is passed separately when the payload has a records array: .jsonl and
+// .csv are formats for records, not for envelopes.
+function writeOutput(name, payload, rows) {
+  const requested = String(name);
+  const safeName = path.basename(requested);
+  const parts = safeName.split(".");
+  const ext = parts.length > 1 ? `.${parts.pop()}` : "";
+  const base = parts.join(".");
+  const finalName = `${base}_${Date.now()}${ext}`;
+
+  fs.mkdirSync(DOWNLOADS_DIR, { recursive: true });
+  const safePath = path.join(DOWNLOADS_DIR, finalName);
+
+  let body;
+  if (rows && ext === ".jsonl") {
+    body = rows.map((r) => JSON.stringify(r)).join("\n");
+  } else if (rows && ext === ".csv") {
+    body = toCsv(rows);
+  } else {
+    body = JSON.stringify(payload, null, 2);
+  }
+
+  fs.writeFileSync(safePath, body);
+  const out = { file: safePath, bytes: Buffer.byteLength(body) };
+
+  // Output stays inside DOWNLOADS_DIR: this path comes from a model, and a tool
+  // that writes to an arbitrary absolute path is a different capability than
+  // one that saves a scrape. But relocating without a word is how a caller ends
+  // up looking for a file that was never going to be there.
+  if (safeName !== requested) {
+    out.note = `output_file is confined to ${DOWNLOADS_DIR} — "${requested}" was written as ${path.basename(safePath)}, not to the path requested.`;
+  }
+  return out;
+}
+
+// What comes back inline when the payload went to disk: enough to know the call
+// worked and the shape is right, and nothing more.
+function summarizeOutput(payload, rows, written) {
+  const summary = { ...written };
+  if (rows) {
+    summary.rows = rows.length;
+    summary.fields = rows.length ? Object.keys(rows[0]) : [];
+    summary.sample = rows.slice(0, 2);
+    for (const k of ["fill_rates", "dropped", "warnings", "stopped_because", "pages_fetched"]) {
+      if (payload && payload[k] !== undefined) summary[k] = payload[k];
+    }
+  } else if (Array.isArray(payload)) {
+    summary.items = payload.length;
+    summary.sample = payload.slice(0, 2);
+  } else if (payload && typeof payload === "object") {
+    summary.keys = Object.keys(payload);
+  }
+  return summary;
+}
+
+// Payload shapes that carry a records array worth writing as jsonl/csv.
+function rowsOf(payload) {
+  if (payload && Array.isArray(payload.rows)) return payload.rows;
+  if (Array.isArray(payload)) return payload;
+  return null;
+}
+
+module.exports = { toCsv, isBlank, writeOutput, summarizeOutput, rowsOf };

package/mcp-server/tools.js

@@ -0,0 +1,439 @@
+// The tool surface: what the model can see and call. Data only — every handler
+// lives in index.js. Kept apart because this is the file that changes whenever
+// a capability is added or a description is reworded, and it was 27% of a
+// 1,600-line index.js that also held the transport, the writers and the loop.
+
+// Every page-facing tool accepts the same two targeting parameters and, if it
+// reads anything back, the same sink. They were copy-pasted into all 23 schemas,
+// which is how `session` ended up described on browser_navigate and nowhere else
+// — fifteen tools offered the model a bare string with no hint what it does.
+const TARGET = {
+  tabId: {
+    type: "number",
+    description: "Target a specific tab. Omit to use the last tab navigated.",
+  },
+  session: {
+    type: "string",
+    description:
+      "Named session (tab) to run against. Created on first use by browser_navigate, so a loop can keep one tab instead of opening hundreds.",
+  },
+};
+
+// One description, because writeOutput's behaviour is uniform: records become
+// .jsonl/.csv when the extension asks for them, everything else is JSON.
+const SINK = {
+  output_file: {
+    type: "string",
+    description:
+      "Filename to write the payload under ~/.poltertab/downloads/, returning only a summary (row count, field names, fill rates, two sample records). .jsonl and .csv are written in those formats when the payload carries records, anything else as JSON. A path is reduced to its basename — output cannot be written elsewhere.",
+  },
+};
+
+// Define tools
+const BROWSER_TOOLS = [
+  {
+    name: "browser_navigate",
+    description: "Navigate to any URL in the browser",
+    inputSchema: {
+      type: "object",
+      properties: {
+        url: { type: "string" },
+        ...TARGET,
+      },
+      required: ["url"],
+    },
+  },
+  {
+    name: "browser_click",
+    description: "Click an element on the page",
+    inputSchema: {
+      type: "object",
+      properties: {
+        selector: { type: "string" },
+        ...TARGET,
+      },
+      required: ["selector"],
+    },
+  },
+  {
+    name: "browser_fill",
+    description: "Fill an input field with text",
+    inputSchema: {
+      type: "object",
+      properties: {
+        selector: { type: "string" },
+        value: { type: "string" },
+        submit: { type: "boolean" },
+        ...TARGET,
+      },
+      required: ["selector", "value"],
+    },
+  },
+  {
+    name: "browser_scrape",
+    description:
+      "Scrape the page or specific elements. For repeating records (cards, rows, listings) use browser_extract instead — it keeps fields grouped per record.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        selector: { type: "string" },
+        attribute: {
+          type: "string",
+          description:
+            "Attribute to read, or 'text' for the element's text. 'href'/'src' come back absolute.",
+        },
+        multiple: { type: "boolean" },
+        fields: {
+          type: "array",
+          items: {
+            type: "string",
+            enum: ["meta", "jsonld", "links", "headings", "bodyText"],
+          },
+          description:
+            "Full-page scrape only (no selector): which parts to return. title and url are always included. ['meta','jsonld'] is the cheap structured-data path — og: tags and schema.org blobs without 50KB of body text. Omit for everything.",
+        },
+        max_text: {
+          type: "number",
+          description: "Max characters of text per element (default 500)",
+        },
+        ...SINK,
+        ...TARGET,
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_extract",
+    description:
+      "Extract repeating records (cards, rows, listings) with fields grouped per record. Fields resolve INSIDE each record root and a missing field yields null instead of shifting later records' values. Returns fill rates and warns when a field is empty inside the record scope but matches page-wide (record boundary too narrow).",
+    inputSchema: {
+      type: "object",
+      properties: {
+        record: {
+          type: "string",
+          description:
+            "Selector for the repeating container, e.g. '.agent-card'. Verify with browser_snapshot that it encloses ALL the fields you want — the element that looks like the card often excludes siblings holding some of them.",
+        },
+        fields: {
+          type: "object",
+          description:
+            "Map of field name -> {sel, get, many, strip}. sel is relative to the record root; omit it (or use '.') for the root itself. get: 'text' (default) | 'href' | 'src' | any attribute name. many: true collects all matches into an array. strip removes a leading prefix, e.g. 'tel:'.",
+          additionalProperties: {
+            type: "object",
+            properties: {
+              sel: { type: "string" },
+              get: { type: "string" },
+              many: { type: "boolean" },
+              strip: { type: "string" },
+            },
+          },
+        },
+        anchor: {
+          type: "string",
+          description:
+            "Name of the field that is always present on a real record (usually the detail-page link). Records missing it are dropped as placeholders and counted in 'dropped'.",
+        },
+        max_text: {
+          type: "number",
+          description: "Max characters per text field (default 500)",
+        },
+        probe: {
+          type: "boolean",
+          description:
+            "Page-wide re-check of any field that came back entirely empty (default true)",
+        },
+        ...SINK,
+        ...TARGET,
+      },
+      required: ["record", "fields"],
+    },
+  },
+  {
+    name: "browser_extract_all",
+    description:
+      "Paginate and extract in one call, with no model round-trip per page. Takes browser_extract's spec plus a URL template, walks pages, dedups on a key, and halts on: limit reached, empty page, a page whose records repeat an earlier page's (the trap where ignored page-size params silently return page 1 again), fill rates collapsing against page 1's baseline, or max_pages. Always reports which condition fired and returns everything collected so far.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        url_template: {
+          type: "string",
+          description:
+            "URL with a {page} placeholder, e.g. 'https://www.kw.com/agents?page={page}'",
+        },
+        record: { type: "string" },
+        fields: { type: "object" },
+        anchor: { type: "string" },
+        key: {
+          type: "string",
+          description:
+            "Field name to dedup on — use a stable per-record identifier such as the detail URL. Falls back to whole-row equality.",
+        },
+        limit: {
+          type: "number",
+          description: "Stop after this many records (default 200)",
+        },
+        offset: {
+          type: "number",
+          description:
+            "Skip this many records from the start of the stream. Pages are re-fetched to reach the offset; pass start_page to skip cheaply.",
+        },
+        start_page: { type: "number", description: "First page (default 1)" },
+        max_pages: {
+          type: "number",
+          description: "Hard guard on pages fetched (default 50)",
+        },
+        fill_tolerance: {
+          type: "number",
+          description:
+            "Halt when a field well-populated on the baseline page falls below this fraction of it (default 0.5). 0 disables the check.",
+        },
+        max_text: { type: "number" },
+        ...SINK,
+        // Session only, deliberately: this tool drives its own navigation from
+        // url_template, so a tabId would be accepted and then ignored.
+        session: TARGET.session,
+      },
+      required: ["url_template", "record", "fields"],
+    },
+  },
+  {
+    name: "browser_screenshot",
+    description:
+      "Capture visible tab as base64 PNG. This action will focus the target tab.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...TARGET,
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_scroll",
+    description: "Scroll the page",
+    inputSchema: {
+      type: "object",
+      properties: {
+        direction: {
+          type: "string",
+          enum: ["up", "down", "left", "right", "top", "bottom"],
+        },
+        amount: { type: "number" },
+        selector: { type: "string" },
+        ...TARGET,
+      },
+      required: ["direction"],
+    },
+  },
+  {
+    name: "browser_hover",
+    description: "Hover over an element",
+    inputSchema: {
+      type: "object",
+      properties: {
+        selector: { type: "string" },
+        ...TARGET,
+      },
+      required: ["selector"],
+    },
+  },
+  {
+    name: "browser_get_text",
+    description: "Get text content of an element",
+    inputSchema: {
+      type: "object",
+      properties: {
+        selector: { type: "string" },
+        max_text: {
+          type: "number",
+          description:
+            "Max characters (default 10000). The result flags it when text was cut.",
+        },
+        ...SINK,
+        ...TARGET,
+      },
+      required: ["selector"],
+    },
+  },
+  {
+    name: "browser_get_title",
+    description: "Get page title and URL",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...TARGET,
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_get_url",
+    description: "Get page URL",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...TARGET,
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_snapshot",
+    description:
+      "Get a snapshot of the DOM. Large pages run to tens of KB — narrow it with interactive_only/max_nodes, or send it to output_file, before spending the context on it.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        interactive_only: {
+          type: "boolean",
+          description: "Only clickable/typable elements",
+        },
+        max_nodes: {
+          type: "number",
+          description: "Cap on nodes returned (default 400)",
+        },
+        max_depth: { type: "number" },
+        ...SINK,
+        ...TARGET,
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_get_network_state",
+    description:
+      "Get captured raw JSON data (GraphQL/XHR) for the current tab, bypassing DOM virtualization",
+    inputSchema: {
+      type: "object",
+      properties: {
+        ...TARGET,
+        clear: {
+          type: "boolean",
+          description: "Clear the buffer after reading (default: true)",
+        },
+        ...SINK,
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_set_intercept_patterns",
+    description:
+      "Set URL substrings to determine which network requests are captured in the MAIN world.",
+    inputSchema: {
+      type: "object",
+      properties: {
+        patterns: {
+          type: "array",
+          items: { type: "string" },
+          description:
+            "Array of substrings (e.g. ['graphql', '/api/v1/comments'])",
+        },
+        ...TARGET,
+      },
+      required: ["patterns"],
+    },
+  },
+  {
+    name: "browser_smart_scroll",
+    description:
+      "Scroll the page and wait for new network data to load (handles lazy-loading)",
+    inputSchema: {
+      type: "object",
+      properties: {
+        direction: { type: "string", enum: ["up", "down"] },
+        amount: { type: "number" },
+        ...TARGET,
+      },
+      required: ["direction"],
+    },
+  },
+  {
+    name: "browser_session_create",
+    description: "Create or track a named browser session (tab)",
+    inputSchema: {
+      type: "object",
+      properties: {
+        name: { type: "string" },
+        url: { type: "string" },
+      },
+      required: ["name"],
+    },
+  },
+  {
+    name: "browser_session_switch",
+    description: "Switch to a different tracked session",
+    inputSchema: {
+      type: "object",
+      properties: {
+        name: { type: "string" },
+      },
+      required: ["name"],
+    },
+  },
+  {
+    name: "browser_session_list",
+    description: "List all tracked sessions",
+    inputSchema: {
+      type: "object",
+      properties: {},
+      required: [],
+    },
+  },
+  {
+    name: "browser_session_close",
+    description: "Close a tracked session",
+    inputSchema: {
+      type: "object",
+      properties: {
+        name: { type: "string" },
+      },
+      required: ["name"],
+    },
+  },
+  {
+    name: "browser_session_context",
+    description: "Get context info for the current active session",
+    inputSchema: {
+      type: "object",
+      properties: {},
+      required: [],
+    },
+  },
+  {
+    name: "browser_get_site_memory",
+    description:
+      "Get navigation memory, obstacles, and fixes for a specific website domain",
+    inputSchema: {
+      type: "object",
+      properties: {
+        hostname: {
+          type: "string",
+          description: "e.g., 'www.linkedin.com' or 'x.com'",
+        },
+        domain: { type: "string", description: "Alias for hostname" },
+      },
+      required: [],
+    },
+  },
+  {
+    name: "browser_save_site_memory",
+    description:
+      "Save a new navigation memory, obstacle, or fix for a specific website domain",
+    inputSchema: {
+      type: "object",
+      properties: {
+        hostname: { type: "string", description: "e.g., 'www.linkedin.com'" },
+        domain: { type: "string", description: "Alias for hostname" },
+        obstacle: {
+          type: "string",
+          description: "What broke or was difficult?",
+        },
+        solution: { type: "string", description: "How did you solve it?" },
+      },
+      required: ["obstacle", "solution"],
+    },
+  },
+];
+
+module.exports = { BROWSER_TOOLS, TARGET, SINK };

package/mcp-server/update-check.js

@@ -196,7 +196,29 @@ function skew(serverVersion, extensionVersion) {
 
 // --- the one-line notice ---------------------------------------------------
 
-const EXTENSION_URL = "https://github.com/abhinav162/poltertab/releases/latest";
+const RELEASES = "https://github.com/abhinav162/poltertab/releases";
+
+// Which extension build to send someone to.
+//
+// It must match the server that drives it: mismatched halves are the exact
+// failure the rest of this module exists to detect, so a link that manufactures
+// one is worse than no link. GitHub's /releases/latest deliberately skips
+// prereleases — so telling a 1.5.0-beta.1 install to "get the latest release"
+// hands it a 1.4.0 extension, and the skew warning then fires on a fresh
+// install that followed the instructions exactly. Worse, that was also the
+// remediation advice printed *for* skew, so the fix re-created the fault.
+//
+// Ask for the release matching the version in hand. Callers pass whichever
+// version is right for what they are saying: the running server for "your
+// halves disagree", the version being upgraded to for "an update is available".
+// /releases/latest survives only as the fallback for a build whose version we
+// cannot parse — a dev checkout — where a tag URL would 404.
+function extensionUrl(version) {
+  const v = String(version || "")
+    .trim()
+    .replace(/^v/, "");
+  return parse(v) ? `${RELEASES}/tag/v${v}` : `${RELEASES}/latest`;
+}
 
 // Surfaced once per server process on a tool response, because it is the only
 // channel the user reliably reads: doctor and the popup both require them to
@@ -207,7 +229,7 @@ function notice({ current, latest, updateAvailable, skew: sk }) {
     parts.push(
       `The Chrome extension (${sk.extension}) is older than this server (${sk.server}) — ` +
         `browser commands may fail in ways that look like missing elements. ` +
-        `Reload it from ${EXTENSION_URL}`,
+        `Reload it from ${extensionUrl(sk.server)}`,
     );
   } else if (sk && sk.kind === "server-behind") {
     parts.push(
@@ -218,7 +240,8 @@ function notice({ current, latest, updateAvailable, skew: sk }) {
   if (updateAvailable && latest) {
     parts.push(
       `PolterTab ${latest} is available (running ${current}). ` +
-        `Update with: npm update -g poltertab && poltertab setup`,
+        `Update with: npm update -g poltertab && poltertab setup, ` +
+        `then reload the extension from ${extensionUrl(latest)}`,
     );
   }
   if (!parts.length) return null;
@@ -236,7 +259,7 @@ module.exports = {
   skew,
   notice,
   disabled,
-  EXTENSION_URL,
+  extensionUrl,
   CACHE_FILE,
   STATE_FILE,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "poltertab",
-  "version": "1.4.0",
+  "version": "1.5.0-beta.2",
   "description": "Let any AI assistant drive and scrape your existing Chrome profile — no headless browser, no complex setup.",
   "bin": {
     "poltertab": "./bin/poltertab.js"
@@ -17,7 +17,6 @@
     "skills/",
     "assets/",
     "!mcp-server/test/",
-    "!mcp-server/test-client.js",
     "!mcp-server/downloads/",
     "!mcp-server/navigation_memory/"
   ],