loom-code 0.1.1__py3-none-any.whl

loom_code/browse/observe.py

@@ -0,0 +1,249 @@
+"""Observe a page: find interactive elements, tag them with stable ids,
+serialize a compact view for the LLM.
+
+The JS below runs in the page and:
+  1. Finds interactive elements — native controls (a/button/input/
+     select/textarea/summary), ARIA roles (button/link/checkbox/combobox/
+     menuitem/tab/option/switch/radio), and contenteditable. Plus a
+     visibility + size filter so we don't list hidden/zero-area noise.
+  2. Tags each with ``data-loom-id="N"`` (stable handle for acting).
+  3. Returns ``[{id, tag, role, label, value, kind}]``.
+
+The Python side turns that into lines the model reads:
+    [12] textbox "Where from?"  (value="Kathmandu")
+    [13] button "Search"
+``kind`` ("input"|"button"|"link"|"select") helps the model pick the
+right action.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+# JS: tag + collect interactive elements. Returns a JSON-able list.
+# Kept as one expression-returning function so page.evaluate gets the
+# array back directly.
+_COLLECT_JS = r"""
+() => {
+  const INTERACTIVE_ROLES = new Set([
+    "button","link","checkbox","combobox","menuitem","tab","option",
+    "switch","radio","textbox","searchbox","slider","spinbutton",
+  ]);
+  const NATIVE = new Set(["A","BUTTON","INPUT","SELECT","TEXTAREA","SUMMARY"]);
+  const out = [];
+  let id = 0;
+
+  const isVisible = (el) => {
+    const r = el.getBoundingClientRect();
+    if (r.width < 4 || r.height < 4) return false;
+    const s = window.getComputedStyle(el);
+    if (s.display === "none" || s.visibility === "hidden" || s.opacity === "0")
+      return false;
+    return true;
+  };
+
+  // Resolve an element's human label as robustly as possible — this is
+  // what lets the model understand "which field is which". Order matters:
+  // explicit accessible name first, then associated <label>, then nearby
+  // text, then attributes.
+  const clean = (s) => (s || "").replace(/\s+/g, " ").trim().slice(0, 90);
+  const labelFor = (el) => {
+    // 1. Explicit accessible name.
+    const aria = el.getAttribute("aria-label");
+    if (aria) return clean(aria);
+    // 2. aria-labelledby → referenced element(s) text.
+    const lb = el.getAttribute("aria-labelledby");
+    if (lb) {
+      const txt = lb.split(/\s+/)
+        .map((id) => document.getElementById(id))
+        .filter(Boolean)
+        .map((n) => n.innerText || n.textContent || "")
+        .join(" ");
+      if (clean(txt)) return clean(txt);
+    }
+    // 3. <label for=id> or wrapping <label>.
+    if (el.id) {
+      const lab = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
+      if (lab && clean(lab.innerText)) return clean(lab.innerText);
+    }
+    const wrapLab = el.closest("label");
+    if (wrapLab && clean(wrapLab.innerText)) return clean(wrapLab.innerText);
+    // 4. placeholder / title.
+    const ph = el.getAttribute("placeholder");
+    if (ph) return clean(ph);
+    const title = el.getAttribute("title");
+    if (title) return clean(title);
+    // 5. The element's own visible text (good for buttons/links).
+    const own = clean(el.innerText || el.textContent || "");
+    if (own) return own;
+    // 6. For inputs with no label yet: the nearest preceding text node /
+    //    sibling label-ish element (handles "Where from?" rendered as a
+    //    separate span above the input).
+    let p = el.previousElementSibling;
+    for (let i = 0; i < 3 && p; i++) {
+      const t = clean(p.innerText || p.textContent || "");
+      if (t && t.length < 40) return t;
+      p = p.previousElementSibling;
+    }
+    // 7. Last resort: name / id attribute.
+    const name = el.getAttribute("name") || el.getAttribute("id") || "";
+    return clean(name);
+  };
+
+  const kindFor = (el, role) => {
+    const t = el.tagName;
+    if (t === "INPUT" || t === "TEXTAREA" || role === "textbox" ||
+        role === "searchbox" || role === "combobox" ||
+        el.isContentEditable) return "input";
+    if (t === "SELECT") return "select";
+    if (t === "A" || role === "link") return "link";
+    return "button";
+  };
+
+  const isInteractive = (el) => {
+    if (NATIVE.has(el.tagName)) return true;
+    const role = el.getAttribute("role");
+    if (role && INTERACTIVE_ROLES.has(role)) return true;
+    if (el.isContentEditable) return true;
+    // Cursor:pointer is a strong custom-button signal.
+    if (window.getComputedStyle(el).cursor === "pointer" &&
+        (el.onclick || el.getAttribute("jsaction"))) return true;
+    return false;
+  };
+
+  const all = document.querySelectorAll("*");
+  for (const el of all) {
+    try {
+      if (!isInteractive(el)) continue;
+      if (!isVisible(el)) continue;
+      const role = el.getAttribute("role") || "";
+      el.setAttribute("data-loom-id", String(id));
+      out.push({
+        id,
+        tag: el.tagName.toLowerCase(),
+        role,
+        kind: kindFor(el, role),
+        label: labelFor(el),
+        value: (el.value != null ? String(el.value) : "").slice(0, 60),
+      });
+      id++;
+    } catch (e) { /* skip problematic node */ }
+  }
+  return out;
+}
+"""
+
+
+# JS: extract the page's VISIBLE TEXT content — what a human reads
+# (prices, listings, results, headings). Unlike observe (which lists
+# CLICKABLE elements), this is for READING outcomes. Walks visible text
+# nodes, collapses whitespace, dedupes, and keeps meaningful lines.
+_READ_TEXT_JS = r"""
+() => {
+  const isVisible = (el) => {
+    if (!el) return false;
+    const s = window.getComputedStyle(el);
+    if (s.display === "none" || s.visibility === "hidden" ||
+        s.opacity === "0") return false;
+    const r = el.getBoundingClientRect();
+    return r.width > 1 && r.height > 1;
+  };
+  // Skip script/style/nav-chrome-heavy containers.
+  const SKIP = new Set(["SCRIPT","STYLE","NOSCRIPT","SVG","PATH"]);
+  const out = [];
+  const seen = new Set();
+  const walker = document.createTreeWalker(
+    document.body, NodeFilter.SHOW_TEXT, null);
+  let n;
+  while ((n = walker.nextNode())) {
+    const t = (n.nodeValue || "").replace(/\s+/g, " ").trim();
+    if (!t || t.length < 2) continue;
+    const p = n.parentElement;
+    if (!p || SKIP.has(p.tagName)) continue;
+    if (!isVisible(p)) continue;
+    if (seen.has(t)) continue;
+    seen.add(t);
+    out.push(t);
+    if (out.length > 400) break;  // safety cap
+  }
+  return out;
+}
+"""
+
+
+async def read_text(page: Any) -> str:
+    """Return the page's visible text content (for reading prices /
+    results / listings). Compact + capped so it stays affordable."""
+    try:
+        lines: list[str] = await page.evaluate(_READ_TEXT_JS)
+    except Exception as exc:  # noqa: BLE001
+        return f"(could not read page text: {exc})"
+    title = url = ""
+    try:
+        title = await page.title()
+        url = page.url
+    except Exception:  # noqa: BLE001
+        pass
+    body = "\n".join(lines)
+    # Hard char cap so a huge results page can't blow the budget.
+    if len(body) > 8000:
+        body = body[:8000] + "\n… (truncated — ask to read more / scroll)"
+    return f"PAGE: {title}\nURL: {url}\n\n{body}"
+
+
+async def observe(page: Any) -> tuple[list[dict[str, Any]], str]:
+    """Tag + collect interactive elements; return (elements, rendered).
+
+    ``rendered`` is the compact text the LLM reads. ``elements`` is the
+    structured list (also drives the selector map / id validity)."""
+    try:
+        elements: list[dict[str, Any]] = await page.evaluate(_COLLECT_JS)
+    except Exception as exc:  # noqa: BLE001 — never break the run
+        return [], f"(could not read the page: {exc})"
+
+    title = ""
+    url = ""
+    try:
+        title = await page.title()
+        url = page.url
+    except Exception:  # noqa: BLE001
+        pass
+
+    lines: list[str] = [f"PAGE: {title}", f"URL: {url}", ""]
+    if not elements:
+        lines.append("(no interactive elements found — the page may still "
+                     "be loading; try page_observe again, or page_check to "
+                     "see it visually.)")
+        return elements, "\n".join(lines)
+
+    # COST CONTROL: a page can have 800+ interactive elements (nav, footer
+    # links, ad chrome) — re-sending all of them every turn is the token
+    # blowup. Prioritise the elements that matter (inputs/comboboxes +
+    # buttons with real labels) and cap the rendered list. The full
+    # ``elements`` list (with ids) is still returned for acting; we only
+    # trim what the LLM READS.
+    PRIORITY = {"input": 0, "select": 1, "button": 2, "link": 3}
+
+    def _score(e: dict[str, Any]) -> tuple[int, int]:
+        kind = e.get("kind", "link")
+        has_label = 0 if (e.get("label") or "").strip() else 1
+        return (PRIORITY.get(kind, 9), has_label)
+
+    ranked = sorted(elements, key=_score)
+    CAP = 60
+    shown = ranked[:CAP]
+    # Keep them in id order within the shown set so the list reads
+    # naturally.
+    shown.sort(key=lambda e: e["id"])
+    for e in shown:
+        label = e.get("label") or "(no label)"
+        val = e.get("value") or ""
+        val_part = f'  (value="{val}")' if val else ""
+        lines.append(f'[{e["id"]}] {e["kind"]} "{label}"{val_part}')
+    if len(elements) > len(shown):
+        lines.append(
+            f"… (+{len(elements) - len(shown)} more elements not shown — "
+            "mostly nav/footer. If you need one that isn't listed, say which "
+            "and I'll surface it.)"
+        )
+    return elements, "\n".join(lines)

loom_code/browse/session.py

@@ -0,0 +1,96 @@
+"""A single headed browser session for loom-code's ``/computer`` mode.
+
+One ``BrowserSession`` owns one visible Chromium window for the life of a
+REPL session. The agent's page_* tools all act through it. It is created
+lazily on the first navigation and torn down on mode-off / exit.
+
+Reliability design (learned from our Google-Flights failures + how
+browser-use stays robust):
+
+* **Stable element handles.** Instead of ephemeral snapshot refs (which
+  die the instant the DOM changes — the ``Ref e145 not found`` error),
+  every interactive element is tagged in the live DOM with a
+  ``data-loom-id="N"`` attribute. Acting later re-selects by that
+  attribute (``[data-loom-id="N"]``), so Playwright re-resolves the
+  element FRESH on every action — no stale handles. The id rides on the
+  element through re-renders unless the element itself is recreated.
+
+* **One persistent page.** The session keeps the active page so observe →
+  act → observe operates on the same evolving page, like a human.
+
+This module is intentionally dependency-light: Playwright only (already
+installed for the project). No browser-use, no extra packages.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class BrowserSession:
+    """Lazily-launched headed Chromium + the active page. Single-page for
+    now (the active tab); multi-tab can come later."""
+
+    def __init__(self, *, headless: bool = False) -> None:
+        self._headless = headless
+        self._pw: Any = None
+        self._browser: Any = None
+        self._context: Any = None
+        self._page: Any = None
+        # index -> True; the live truth lives in the DOM (data-loom-id),
+        # this just tracks the highest id assigned this observe pass.
+        self._max_id: int = 0
+
+    async def start(self) -> None:
+        """Launch the browser if not already running. Idempotent."""
+        if self._page is not None:
+            return
+        from playwright.async_api import async_playwright
+
+        self._pw = await async_playwright().start()
+        # Headed = the user watches it work (the whole point of /computer).
+        self._browser = await self._pw.chromium.launch(
+            headless=self._headless,
+            args=["--start-maximized"],
+        )
+        # A real-ish context: viewport=None lets the window size drive it
+        # (with --start-maximized), and a normal UA reduces trivial
+        # bot-walls.
+        self._context = await self._browser.new_context(
+            viewport=None,
+            user_agent=(
+                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/124.0.0.0 Safari/537.36"
+            ),
+        )
+        self._page = await self._context.new_page()
+
+    @property
+    def page(self) -> Any:
+        if self._page is None:
+            raise RuntimeError("browser not started — call start() first")
+        return self._page
+
+    async def goto(self, url: str) -> None:
+        await self.start()
+        if not url.startswith(("http://", "https://")):
+            url = "https://" + url
+        await self._page.goto(
+            url, wait_until="domcontentloaded", timeout=45000
+        )
+
+    async def close(self) -> None:
+        """Tear everything down. Safe to call multiple times."""
+        for closer in (
+            getattr(self._context, "close", None),
+            getattr(self._browser, "close", None),
+            getattr(self._pw, "stop", None),
+        ):
+            if closer is not None:
+                try:
+                    await closer()
+                except Exception:  # noqa: BLE001 — best-effort teardown
+                    pass
+        self._pw = self._browser = self._context = self._page = None
+        self._max_id = 0

loom_code/browse/verify.py

@@ -0,0 +1,194 @@
+"""Verify page state by reading the live DOM.
+
+loomflow has no image-to-model (vision) input yet, so instead of a
+screenshot+vision check we read the ACTUAL values the page holds — the
+text content of the inputs/fields visible on screen — and return them so
+the agent can confirm "did Delhi land in the origin field?". This catches
+the wrong-field / didn't-stick failures without needing vision: the
+field's real ``value`` is ground truth.
+
+It also saves a screenshot to disk so the user (and a future vision
+upgrade) can inspect what the page looked like.
+"""
+
+from __future__ import annotations
+
+import base64
+from pathlib import Path
+from typing import Any
+
+# Set-of-Marks: draw a numbered box on each interactive element (using
+# the data-loom-id tags observe() already placed) so a screenshot the
+# model SEES is annotated with the SAME [ids] it acts on. Returns a
+# teardown function name so we can remove the overlay after the shot.
+_DRAW_MARKS_JS = r"""
+() => {
+  const old = document.getElementById('__loom_marks__');
+  if (old) old.remove();
+  const layer = document.createElement('div');
+  layer.id = '__loom_marks__';
+  layer.style.cssText =
+    'position:fixed;inset:0;z-index:2147483647;pointer-events:none;';
+  const COLORS = ['#e6194B','#3cb44b','#4363d8','#f58231','#911eb4',
+                  '#469990','#9A6324','#800000','#808000','#000075'];
+  let n = 0;
+  for (const el of document.querySelectorAll('[data-loom-id]')) {
+    const r = el.getBoundingClientRect();
+    if (r.width < 4 || r.height < 4) continue;
+    if (r.bottom < 0 || r.top > window.innerHeight) continue;  // off-screen
+    const id = el.getAttribute('data-loom-id');
+    const c = COLORS[n % COLORS.length]; n++;
+    const box = document.createElement('div');
+    box.style.cssText =
+      `position:fixed;left:${r.left}px;top:${r.top}px;width:${r.width}px;`+
+      `height:${r.height}px;border:2px solid ${c};box-sizing:border-box;`;
+    const tag = document.createElement('div');
+    tag.textContent = id;
+    tag.style.cssText =
+      `position:fixed;left:${r.left}px;top:${Math.max(0,r.top-14)}px;`+
+      `background:${c};color:#fff;font:bold 11px monospace;padding:0 3px;`+
+      `line-height:14px;`;
+    layer.appendChild(box); layer.appendChild(tag);
+  }
+  document.body.appendChild(layer);
+}
+"""
+
+_REMOVE_MARKS_JS = (
+    "() => { const m = document.getElementById('__loom_marks__'); "
+    "if (m) m.remove(); }"
+)
+
+
+async def screenshot_b64(page: Any, marks: bool = True) -> str | None:
+    """Base64 PNG of the current viewport. With ``marks`` (default), draw
+    the numbered Set-of-Marks overlay first so the model sees the [ids],
+    then remove it. Returns None on failure."""
+    drew = False
+    try:
+        if marks:
+            await page.evaluate(_DRAW_MARKS_JS)
+            drew = True
+        png = await page.screenshot(type="png")
+        return base64.b64encode(png).decode("ascii")
+    except Exception:  # noqa: BLE001
+        return None
+    finally:
+        if drew:
+            try:
+                await page.evaluate(_REMOVE_MARKS_JS)
+            except Exception:  # noqa: BLE001
+                pass
+
+# Snapshot the values of every labelled input/field + recent prices, so a
+# "what's currently entered / shown" question is answerable from the DOM.
+_STATE_JS = r"""
+() => {
+  const clean = (s) => (s || "").replace(/\s+/g, " ").trim();
+  const fields = [];
+  for (const el of document.querySelectorAll(
+        "input, textarea, select, [role=combobox], [contenteditable]")) {
+    const r = el.getBoundingClientRect();
+    if (r.width < 4 || r.height < 4) continue;
+    const label = clean(
+      el.getAttribute("aria-label") ||
+      el.getAttribute("placeholder") ||
+      el.getAttribute("name") || "");
+    const val = clean(el.value || el.getAttribute("value") ||
+                      el.innerText || "");
+    if (label || val) fields.push({ label, value: val.slice(0, 60) });
+  }
+  return { title: document.title, url: location.href, fields };
+}
+"""
+
+
+async def verify(page: Any, question: str, model: str | None = None) -> str:
+    """Return the page's current title/URL + the values held in its
+    fields, so the agent can answer ``question`` from real DOM state.
+
+    (``model`` is accepted for API stability / a future vision upgrade
+    but unused — loomflow has no image input yet.)"""
+    try:
+        state = await page.evaluate(_STATE_JS)
+    except Exception as exc:  # noqa: BLE001
+        return f"could not read page state ({exc}); try page_observe."
+
+    # Best-effort screenshot to disk for the user / future vision.
+    try:
+        shot = Path.home() / ".loom-code" / "last_page.png"
+        shot.parent.mkdir(parents=True, exist_ok=True)
+        await page.screenshot(path=str(shot), type="png")
+    except Exception:  # noqa: BLE001
+        pass
+
+    lines = [
+        f"Checking: {question}",
+        f"PAGE: {state.get('title', '')}",
+        f"URL: {state.get('url', '')}",
+        "Current field values:",
+    ]
+    fields = state.get("fields") or []
+    if not fields:
+        lines.append("  (no field values found)")
+    for f in fields:
+        lbl = f.get("label") or "(unlabelled)"
+        val = f.get("value") or ""
+        lines.append(f'  {lbl}: "{val}"')
+    lines.append(
+        "\n→ Compare the values above to what you intended. If a field has "
+        "the WRONG value (e.g. origin shows your location, not what you "
+        "typed), fix it: re-observe, clear that field, type again, and for "
+        "autocompletes CLICK the matching suggestion."
+    )
+    return "\n".join(lines)
+
+
+async def look(page: Any, question: str, model: str | None = None) -> str:
+    """VISION: screenshot the page (with Set-of-Marks [id] overlay) and
+    ask the session's multimodal model ``question`` about it. This is how
+    the agent SEES the page — reads prices/results, confirms layout,
+    disambiguates fields — instead of inferring from DOM text alone.
+
+    Sends the image via loomflow's new image input
+    (metadata['_loom_images']) so it works with whichever model the user
+    runs (Claude or GPT — both multimodal). Degrades to a DOM read if no
+    model / vision is available."""
+    b64 = await screenshot_b64(page, marks=True)
+    if b64 is None:
+        return await verify(page, question, model)
+    if not model:
+        return (
+            "no model available for page_look — using DOM values:\n\n"
+            + await verify(page, question, model)
+        )
+    try:
+        from loomflow import Agent
+    except Exception:  # noqa: BLE001
+        return await verify(page, question, model)
+    prompt = (
+        "You are looking at a screenshot of a web page. Numbered colored "
+        "boxes mark interactive elements (the number is the element id you "
+        "can act on). Answer concisely and concretely. If asked for prices/"
+        "results, read the actual values you see and list them.\n\n"
+        f"Question: {question}"
+    )
+    try:
+        probe = Agent(prompt, model=model)
+        result = await probe.run(
+            "(see attached screenshot)",
+            metadata={
+                "_loom_images": [
+                    {"data": b64, "media_type": "image/png"}
+                ]
+            },
+        )
+        out = getattr(result, "output", None) or str(result)
+        return str(out).strip() or "(model returned nothing)"
+    except Exception as exc:  # noqa: BLE001
+        # Vision failed (model not multimodal / API issue) → DOM fallback.
+        return (
+            f"page_look vision unavailable ({type(exc).__name__}); "
+            "using DOM values instead:\n\n"
+            + await verify(page, question, model)
+        )