agentatlas 0.1.0__tar.gz

agentatlas-0.1.0/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: agentatlas
+Version: 0.1.0
+Summary: Shared browser interaction schema registry for AI agents. Reduces LLM token usage by 80-100% on known sites.
+License: MIT
+Project-URL: Homepage, https://github.com/yourusername/agentatlas
+Project-URL: Issues, https://github.com/yourusername/agentatlas/issues
+Keywords: ai,agents,browser-automation,llm,playwright
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: supabase>=2.6.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: playwright>=1.44.0
+Requires-Dist: openai>=1.109.1
+Requires-Dist: playwright-stealth>=1.0.6
+
+# AgentAtlas
+
+**Shared browser interaction schema registry for AI agents.**
+
+Reduces LLM token usage by 80-100% on known sites by storing and sharing site interaction schemas across all users.
+
+## How it works
+```
+First user  → LLM learns the site → saved to shared registry
+Every user after → 0 tokens, instant response
+```
+
+## Benchmark results (real data)
+
+| | Without AgentAtlas | With AgentAtlas |
+|---|---|---|
+| Tokens | 2,597 | 0-445 |
+| Cost | $0.018 | $0.000-$0.002 |
+| Time | 19s | 0.2-12s |
+| Real URLs | ❌ | ✅ |
+
+**82.9% token reduction** when LLM still needed. **100% reduction** for repeat workflows.
+
+## Install
+```bash
+pip install agentatlas
+playwright install chromium
+```
+
+## Usage
+```python
+from agentatlas.atlas import Atlas
+
+atlas = Atlas()
+
+# Get schema for any site
+# Found in registry → 0 tokens
+# New site → learns once, saves for everyone
+schema = await atlas.get_schema(
+    site="greenhouse.io",
+    url="https://boards.greenhouse.io/anthropic"
+)
+
+# Pass compact schema to YOUR LLM
+# 150-500 tokens instead of 50,000
+print(schema.elements)
+print(schema.tokens_used)  # 0 if registry hit
+print(schema.source)       # "registry" or "llm_learned"
+```
+
+## Environment variables
+```bash
+SUPABASE_URL=your_supabase_url
+SUPABASE_SERVICE_ROLE_KEY=your_key
+OPENAI_API_KEY=your_key
+```
+
+## The flywheel
+```
+More developers use AgentAtlas
+        ↓
+More new sites get learned automatically
+        ↓
+Registry grows → higher hit rate
+        ↓
+Less tokens burned across the whole network
+        ↓
+Cheaper + faster → more developers adopt
+```
+
+## License
+
+MIT

agentatlas-0.1.0/README.md

@@ -0,0 +1,73 @@
+# AgentAtlas
+
+**Shared browser interaction schema registry for AI agents.**
+
+Reduces LLM token usage by 80-100% on known sites by storing and sharing site interaction schemas across all users.
+
+## How it works
+```
+First user  → LLM learns the site → saved to shared registry
+Every user after → 0 tokens, instant response
+```
+
+## Benchmark results (real data)
+
+| | Without AgentAtlas | With AgentAtlas |
+|---|---|---|
+| Tokens | 2,597 | 0-445 |
+| Cost | $0.018 | $0.000-$0.002 |
+| Time | 19s | 0.2-12s |
+| Real URLs | ❌ | ✅ |
+
+**82.9% token reduction** when LLM still needed. **100% reduction** for repeat workflows.
+
+## Install
+```bash
+pip install agentatlas
+playwright install chromium
+```
+
+## Usage
+```python
+from agentatlas.atlas import Atlas
+
+atlas = Atlas()
+
+# Get schema for any site
+# Found in registry → 0 tokens
+# New site → learns once, saves for everyone
+schema = await atlas.get_schema(
+    site="greenhouse.io",
+    url="https://boards.greenhouse.io/anthropic"
+)
+
+# Pass compact schema to YOUR LLM
+# 150-500 tokens instead of 50,000
+print(schema.elements)
+print(schema.tokens_used)  # 0 if registry hit
+print(schema.source)       # "registry" or "llm_learned"
+```
+
+## Environment variables
+```bash
+SUPABASE_URL=your_supabase_url
+SUPABASE_SERVICE_ROLE_KEY=your_key
+OPENAI_API_KEY=your_key
+```
+
+## The flywheel
+```
+More developers use AgentAtlas
+        ↓
+More new sites get learned automatically
+        ↓
+Registry grows → higher hit rate
+        ↓
+Less tokens burned across the whole network
+        ↓
+Cheaper + faster → more developers adopt
+```
+
+## License
+
+MIT

agentatlas-0.1.0/agentatlas/__init__.py

@@ -0,0 +1,4 @@
+from agentatlas.atlas import Atlas, SiteSchema
+
+__all__ = ["Atlas", "SiteSchema"]
+__version__ = "0.1.0"

agentatlas-0.1.0/agentatlas/atlas.py

@@ -0,0 +1,403 @@
+"""
+atlas.py — The complete AgentAtlas SDK
+
+Flow:
+  1. Developer calls get_schema(site, url)
+  2. Check DB → found? return immediately (0 tokens)
+  3. Not found? → crawl page → LLM labels elements → save to DB → return
+  4. Every developer after gets it free
+
+This is the flywheel.
+"""
+
+import re
+import json
+import os
+import asyncio
+from dataclasses import dataclass
+from urllib.parse import urlparse
+from dotenv import load_dotenv
+from playwright.async_api import async_playwright
+try:
+    from playwright_stealth import stealth_async
+except ImportError:
+    stealth_async = None
+from openai import OpenAI
+from agentatlas.supabase_client import get_supabase
+
+load_dotenv()
+
+
+# ─────────────────────────────────────────────────────────────
+# What gets returned to the developer
+# ─────────────────────────────────────────────────────────────
+@dataclass
+class SiteSchema:
+    site: str
+    url: str
+    route_key: str
+    status: str           # "found" | "learned" | "not_found"
+    confidence: float
+    elements: dict        # what the developer's LLM reads
+    source: str           # "registry" | "llm_learned" | "not_found"
+    tokens_used: int      # 0 if registry hit, >0 if LLM had to learn it
+    message: str          # human readable explanation
+
+
+# ─────────────────────────────────────────────────────────────
+# Main SDK class
+# ─────────────────────────────────────────────────────────────
+class Atlas:
+    def __init__(self):
+        self.sb     = get_supabase()
+        self.client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+    # ─────────────────────────────────────────────
+    # PUBLIC: main entry point
+    # ─────────────────────────────────────────────
+    async def get_schema(self, site: str, url: str) -> SiteSchema:
+        """
+        Get the interaction schema for a site/page.
+
+        If found in registry  → returns immediately, 0 tokens
+        If not found          → crawls page, learns schema,
+                                saves to DB for future users,
+                                returns schema
+        """
+
+        print(f"\n[AgentAtlas] Looking up: {site}")
+
+        # ── STEP 1: check registry ──────────────────────────
+        schema = await self._fetch_from_registry(site, url)
+        if schema:
+            print(f"[AgentAtlas] ✅ Registry hit — 0 tokens used")
+            return SiteSchema(
+                site=site, url=url,
+                route_key=schema["route_key"],
+                status="found",
+                confidence=schema["confidence"],
+                elements=schema["elements"],
+                source="registry",
+                tokens_used=0,
+                message="Schema found in registry. No LLM used."
+            )
+
+        # ── STEP 2: not found → crawl + learn ───────────────
+        print(f"[AgentAtlas] ⚠ Not in registry. Crawling and learning...")
+        learned = await self._learn_site(site, url)
+
+        if not learned:
+            return SiteSchema(
+                site=site, url=url,
+                route_key="unknown",
+                status="not_found",
+                confidence=0.0,
+                elements={},
+                source="not_found",
+                tokens_used=0,
+                message="Could not learn site. Page may be blocked or empty."
+            )
+
+        # ── STEP 3: save to DB for future users ─────────────
+        await self._save_to_registry(site, url, learned)
+        print(f"[AgentAtlas] 💾 Saved to registry — next user gets this free")
+
+        return SiteSchema(
+            site=site, url=url,
+            route_key=learned["route_key"],
+            status="learned",
+            confidence=0.6,
+            elements=learned["elements"],
+            source="llm_learned",
+            tokens_used=learned["tokens_used"],
+            message=f"Schema learned and saved. Tokens used: {learned['tokens_used']}. Free for all future users."
+        )
+
+
+    # ─────────────────────────────────────────────
+    # PRIVATE: fetch from registry
+    # ─────────────────────────────────────────────
+    async def _fetch_from_registry(self, site: str, url: str) -> dict | None:
+        # look up site
+        site_row = self.sb.table("sites")\
+            .select("id")\
+            .eq("domain", site)\
+            .limit(1).execute().data
+        if not site_row:
+            return None
+        site_id = site_row[0]["id"]
+
+        # match url to route
+        routes = self.sb.table("page_routes")\
+            .select("id, route_key, path_pattern")\
+            .eq("site_id", site_id)\
+            .execute().data
+        matched = self._match_route(url, routes)
+        if not matched:
+            return None
+
+        # fetch best active playbook
+        playbooks = self.sb.table("playbooks")\
+            .select("payload, confidence")\
+            .eq("site_id", site_id)\
+            .eq("route_id", matched["id"])\
+            .eq("status", "active")\
+            .order("confidence", desc=True)\
+            .limit(1).execute().data
+        if not playbooks:
+            return None
+
+        payload = playbooks[0]["payload"]
+        elements = self._build_elements(payload)
+        if not elements:
+            return None
+
+        return {
+            "route_key" : matched["route_key"],
+            "confidence": playbooks[0]["confidence"],
+            "elements"  : elements,
+        }
+
+
+    # ─────────────────────────────────────────────
+    # PRIVATE: crawl page + ask LLM to label
+    # ─────────────────────────────────────────────
+    async def _learn_site(self, site: str, url: str) -> dict | None:
+        # crawl the page
+        elements_raw = await self._crawl_page(url)
+        if not elements_raw or len(elements_raw) < 10:
+            print(f"[AgentAtlas] ❌ Too few elements ({len(elements_raw) if elements_raw else 0}) — page may be blocked")
+            return None
+
+        print(f"[AgentAtlas] 🔍 Crawled {len(elements_raw)} elements — asking LLM to label...")
+
+        # ask LLM to label elements
+        prompt = f"""
+You are building a browser automation schema for {site}.
+Below are raw DOM elements from the page at: {url}
+
+Label the KEY interactive elements found on this page.
+Be specific about what each element is for.
+
+Return ONLY valid JSON — no markdown, no explanation:
+{{
+  "route_key": "job_list | job_detail | search | product | home | other",
+  "elements": {{
+    "element_purpose": {{
+      "type": "css | role | text | aria_label | data_testid",
+      "selector": "the actual selector value",
+      "confidence": 0.0
+    }}
+  }}
+}}
+
+Rules:
+- Only include elements you are confident about (confidence >= 0.5)
+- Use stable selectors (aria-label, data-testid, name attr, semantic text)
+- Avoid fragile selectors (hashed CSS classes like _abc123)
+- Include 3-8 elements maximum — only the important ones
+
+DOM elements:
+{json.dumps(elements_raw[:80], indent=1)}
+"""
+
+        response = self.client.chat.completions.create(
+            model="gpt-4o",
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0,
+            response_format={"type": "json_object"},
+        )
+
+        tokens_used = response.usage.total_tokens
+        raw         = response.choices[0].message.content.strip()
+        labeled     = json.loads(raw)
+
+        print(f"[AgentAtlas] 🤖 LLM labeled {len(labeled.get('elements', {}))} elements ({tokens_used} tokens)")
+
+        return {
+            "route_key"  : labeled.get("route_key", "unknown"),
+            "elements"   : labeled.get("elements", {}),
+            "tokens_used": tokens_used,
+            "raw_payload": labeled,
+        }
+
+
+    # ─────────────────────────────────────────────
+    # PRIVATE: save learned schema to DB
+    # ─────────────────────────────────────────────
+    async def _save_to_registry(self, site: str, url: str, learned: dict):
+        try:
+            # upsert site
+            self.sb.table("sites").upsert(
+                {"domain": site, "display_name": site},
+                on_conflict="domain"
+            ).execute()
+
+            site_row = self.sb.table("sites")\
+                .select("id").eq("domain", site)\
+                .limit(1).execute().data
+            site_id = site_row[0]["id"]
+
+            # detect route from url path
+            path      = urlparse(url).path or "/"
+            route_key = learned.get("route_key", "unknown")
+
+            # upsert route
+            self.sb.table("page_routes").upsert(
+                {
+                    "site_id"     : site_id,
+                    "route_key"   : route_key,
+                    "path_pattern": f"^{re.escape(path)}",
+                    "example_url" : url,
+                },
+                on_conflict="site_id,route_key"
+            ).execute()
+
+            route_row = self.sb.table("page_routes")\
+                .select("id")\
+                .eq("site_id", site_id)\
+                .eq("route_key", route_key)\
+                .limit(1).execute().data
+            route_id = route_row[0]["id"]
+
+            # upsert task (generic)
+            self.sb.table("tasks").upsert(
+                {"task_key": "generic_extract", "description": "Generic extraction task"},
+                on_conflict="task_key"
+            ).execute()
+
+            task_row = self.sb.table("tasks")\
+                .select("id").eq("task_key", "generic_extract")\
+                .limit(1).execute().data
+            task_id = task_row[0]["id"]
+
+            # build locators payload from labeled elements
+            locators = {}
+            for purpose, info in learned.get("elements", {}).items():
+                if info.get("confidence", 0) >= 0.5:
+                    locators[purpose] = [{
+                        "type"      : info.get("type"),
+                        "value"     : info.get("selector"),
+                        "priority"  : 1,
+                        "confidence": info.get("confidence"),
+                    }]
+
+            # upsert playbook
+            self.sb.table("playbooks").upsert(
+                {
+                    "site_id"    : site_id,
+                    "route_id"   : route_id,
+                    "task_id"    : task_id,
+                    "variant_key": "desktop_enUS_loggedout",
+                    "version"    : 1,
+                    "status"     : "active",
+                    "confidence" : 0.6,
+                    "ttl_days"   : 14,
+                    "payload"    : {
+                        "locators"          : locators,
+                        "fingerprint_source": "llm_learned",
+                        "source_url"        : url,
+                    },
+                },
+                on_conflict="site_id,route_id,task_id,variant_key,version"
+            ).execute()
+
+        except Exception as e:
+            print(f"[AgentAtlas] ⚠ Save failed: {e}")
+
+
+    # ─────────────────────────────────────────────
+    # PRIVATE: crawl a page with Playwright
+    # ─────────────────────────────────────────────
+    async def _crawl_page(self, url: str) -> list[dict]:
+        try:
+            async with async_playwright() as p:
+                browser = await p.chromium.launch(
+                    headless=False,
+                    args=["--disable-blink-features=AutomationControlled","--no-sandbox"]
+                )
+                context = await browser.new_context(
+                    user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+                    viewport={"width": 1280, "height": 800},
+                    locale="en-US",
+                    timezone_id="America/New_York",
+                )
+                page = await context.new_page()
+
+                if stealth_async:
+                    await stealth_async(page)
+                    print(f"[AgentAtlas] 🥷 Stealth mode active")
+
+                await page.route("**/*.{png,jpg,jpeg,gif,svg,woff,woff2}", lambda r: r.abort())
+
+                try:
+                    await page.goto(url, wait_until="networkidle", timeout=30000)
+                except Exception:
+                    pass
+
+                for wait_ms in [2000, 2000, 2000]:
+                    await page.wait_for_timeout(wait_ms)
+                    count = await page.evaluate("() => document.querySelectorAll('a, button').length")
+                    print(f"[AgentAtlas] ⏳ Elements so far: {count}")
+                    if count > 10:
+                        break
+
+                elements = await page.evaluate("""
+                    () => {
+                        const sel = 'a, button, input, select, textarea, h1, h2, h3, [role=button], [role=link], [role=listitem]'
+                        return Array.from(document.querySelectorAll(sel))
+                            .filter(el => el.offsetParent !== null)
+                            .slice(0, 120)
+                            .map(el => ({
+                                tag:         el.tagName.toLowerCase(),
+                                role:        el.getAttribute('role'),
+                                aria_label:  el.getAttribute('aria-label'),
+                                placeholder: el.getAttribute('placeholder'),
+                                data_testid: el.getAttribute('data-testid'),
+                                name:        el.getAttribute('name'),
+                                text:        el.innerText?.trim().slice(0, 100),
+                                href:        el.href || null,
+                                type:        el.type || null,
+                            }))
+                    }
+                """)
+                await browser.close()
+                return elements
+        except Exception as e:
+            print(f"[AgentAtlas] Crawl error: {e}")
+            return []
+
+    # ─────────────────────────────────────────────
+    # PRIVATE: match url to stored route pattern
+    # ─────────────────────────────────────────────
+    def _match_route(self, url: str, routes: list) -> dict | None:
+        try:
+            path = urlparse(url).path
+        except Exception:
+            path = url
+        for route in routes:
+            try:
+                if re.search(route["path_pattern"], path):
+                    return route
+            except Exception:
+                continue
+        return routes[0] if routes else None
+
+
+    # ─────────────────────────────────────────────
+    # PRIVATE: build clean elements dict from payload
+    # ─────────────────────────────────────────────
+    def _build_elements(self, payload: dict) -> dict:
+        locators = payload.get("locators", {})
+        elements = {}
+        for purpose, locs in locators.items():
+            if not locs:
+                continue
+            best = sorted(locs, key=lambda x: x.get("priority", 99))[0]
+            elements[purpose] = {
+                "type"      : best.get("type"),
+                "selector"  : best.get("value"),
+                "confidence": best.get("confidence", 0.5),
+            }
+        return elements
+

agentatlas-0.1.0/agentatlas/crawler.py

@@ -0,0 +1,164 @@
+"""
+crawler.py - v2
+Same as before but with better wait strategy for JS-heavy / bot-protected sites.
+"""
+
+import asyncio
+import json
+import os
+from dotenv import load_dotenv
+from playwright.async_api import async_playwright
+from openai import OpenAI
+from agentatlas.supabase_client import get_supabase
+
+load_dotenv()
+
+SEED_TARGETS = [
+    {"site": "greenhouse.io",       "url": "https://boards.greenhouse.io/anthropic",        "route_key": "job_list"},
+    {"site": "lever.co",            "url": "https://jobs.lever.co/netflix",                 "route_key": "job_list"},
+    {"site": "ashbyhq.com",         "url": "https://jobs.ashbyhq.com/openai",               "route_key": "job_list"},
+    {"site": "smartrecruiters.com", "url": "https://www.smartrecruiters.com/Spotify/jobs",  "route_key": "job_list"},
+]
+
+async def extract_elements(url: str) -> list[dict]:
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+            viewport={"width": 1280, "height": 800},
+            locale="en-US",
+            timezone_id="America/New_York",
+        )
+        page = await context.new_page()
+
+        # Block images/fonts to speed up load
+        await page.route("**/*.{png,jpg,jpeg,gif,svg,woff,woff2}", lambda r: r.abort())
+
+        try:
+            await page.goto(url, wait_until="domcontentloaded", timeout=30000)
+        except Exception:
+            pass  # continue even if timeout, grab what loaded
+
+        # Wait progressively — up to 8 seconds for JS content to render
+        for wait_ms in [2000, 3000, 3000]:
+            await page.wait_for_timeout(wait_ms)
+            count = await page.evaluate("() => document.querySelectorAll('a, button').length")
+            if count > 10:
+                break
+
+        elements = await page.evaluate("""
+            () => {
+                const sel = 'a, button, input, select, textarea, h1, h2, h3, [role=button], [role=link], [role=listitem]'
+                return Array.from(document.querySelectorAll(sel))
+                    .filter(el => el.offsetParent !== null)
+                    .slice(0, 120)
+                    .map(el => ({
+                        tag:         el.tagName.toLowerCase(),
+                        role:        el.getAttribute('role'),
+                        aria_label:  el.getAttribute('aria-label'),
+                        placeholder: el.getAttribute('placeholder'),
+                        data_testid: el.getAttribute('data-testid'),
+                        name:        el.getAttribute('name'),
+                        text:        el.innerText?.trim().slice(0, 100),
+                        href:        el.href || null,
+                        type:        el.type || null,
+                        class_hint:  el.className?.toString().slice(0, 60),
+                    }))
+            }
+        """)
+
+        await browser.close()
+        print(f"  → {len(elements)} elements found on {url}")
+
+        if len(elements) < 10:
+            print(f"  ⚠ Low element count — likely bot-blocked or JS-heavy. Skipping LLM labeling.")
+            return []
+
+        return elements
+
+def label_elements(elements: list[dict], site: str, route_key: str) -> dict:
+    client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+    prompt = f"""
+You are building a browser automation schema for {site} ({route_key} page).
+Below are raw DOM elements extracted from the page.
+
+Your job: identify the KEY elements needed for these tasks:
+- extract_job_list: find job title links, location text, pagination
+- extract_job_detail: find title h1, description block, apply button
+- start_application: find the apply/submit button or link
+
+Return ONLY valid JSON in this exact shape, no explanation, no markdown:
+{{
+  "job_title_links":   {{ "type": "...", "value": "...", "confidence": 0.0 }},
+  "location_text":     {{ "type": "...", "value": "...", "confidence": 0.0 }},
+  "apply_button":      {{ "type": "...", "value": "...", "confidence": 0.0 }},
+  "job_heading":       {{ "type": "...", "value": "...", "confidence": 0.0 }},
+  "description_block": {{ "type": "...", "value": "...", "confidence": 0.0 }}
+}}
+
+For "type" use one of: css, role, text, aria_label, data_testid, placeholder
+For "value" use the actual selector value (e.g. "a[data-mapped='true']" or "Apply now")
+
+DOM elements:
+{json.dumps(elements[:80], indent=1)}
+"""
+
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": prompt}],
+        temperature=0,
+        response_format={"type": "json_object"},
+    )
+
+    return json.loads(response.choices[0].message.content.strip())
+
+def update_playbook_payload(site_domain: str, route_key: str, labeled: dict):
+    sb = get_supabase()
+
+    site = sb.table("sites").select("id").eq("domain", site_domain).limit(1).execute().data
+    if not site:
+        print(f"  ⚠ Site not found: {site_domain}")
+        return
+    site_id = site[0]["id"]
+
+    route = sb.table("page_routes").select("id").eq("site_id", site_id).eq("route_key", route_key).limit(1).execute().data
+    if not route:
+        print(f"  ⚠ Route not found: {route_key}")
+        return
+    route_id = route[0]["id"]
+
+    locators = {}
+    for purpose, locator in labeled.items():
+        if locator.get("confidence", 0) >= 0.4:
+            locators[purpose] = [
+                {"type": locator["type"], "value": locator["value"], "priority": 1, "confidence": locator["confidence"]}
+            ]
+
+    playbooks = sb.table("playbooks").select("id, payload").eq("site_id", site_id).eq("route_id", route_id).execute().data
+    for pb in playbooks:
+        updated_payload = {**pb["payload"], "locators": locators, "fingerprint_source": "crawled"}
+        sb.table("playbooks").update({
+            "payload": updated_payload,
+            "confidence": 0.6,
+            "status": "active",
+        }).eq("id", pb["id"]).execute()
+
+    print(f"  ✅ Updated {len(playbooks)} playbook(s) for {site_domain}/{route_key}")
+
+async def main():
+    for target in SEED_TARGETS:
+        print(f"\n🔍 Crawling {target['site']} — {target['url']}")
+        try:
+            elements = await extract_elements(target["url"])
+            if not elements:
+                print(f"  ⏭ Skipped — not enough elements to label reliably")
+                continue
+            labeled = label_elements(elements, target["site"], target["route_key"])
+            print(f"  GPT-4o labeled {len(labeled)} purposes")
+            update_playbook_payload(target["site"], target["route_key"], labeled)
+        except Exception as e:
+            print(f"  ❌ Failed: {e}")
+
+if __name__ == "__main__":
+    asyncio.run(main())

agentatlas-0.1.0/agentatlas/executor.py

@@ -0,0 +1,53 @@
+"""
+executor.py
+Takes a playbook from Supabase and executes it deterministically.
+No LLM calls. Handles scrolling for lazy-loaded content.
+"""
+
+import asyncio
+from playwright.async_api import async_playwright
+
+async def scroll_to_bottom(page, max_scrolls: int = 10):
+    """Scroll down to trigger lazy-loaded content."""
+    for _ in range(max_scrolls):
+        prev_height = await page.evaluate("document.body.scrollHeight")
+        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
+        await page.wait_for_timeout(800)
+        new_height = await page.evaluate("document.body.scrollHeight")
+        if new_height == prev_height:
+            break  # nothing more to load
+
+async def execute_job_list(url: str, selector: str) -> list[dict]:
+    async with async_playwright() as p:
+        browser = await p.chromium.launch(headless=True)
+        context = await browser.new_context(
+            user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36",
+            viewport={"width": 1280, "height": 800},
+        )
+        page = await context.new_page()
+        await page.goto(url, wait_until="networkidle", timeout=30000)
+        await page.wait_for_timeout(2000)
+
+        # Scroll to load all jobs
+        await scroll_to_bottom(page)
+
+        elements = await page.query_selector_all(selector)
+        jobs = []
+        for el in elements:
+            raw_text = (await el.inner_text()).strip()
+            href     = await el.get_attribute("href")
+
+            # Split "Job Title\n\nLocation" into separate fields
+            parts    = [p.strip() for p in raw_text.split("\n") if p.strip()]
+            title    = parts[0] if parts else raw_text
+            location = parts[1] if len(parts) > 1 else ""
+
+            if title and href:
+                jobs.append({
+                    "title"   : title,
+                    "location": location,
+                    "url"     : href if href.startswith("http") else f"https://boards.greenhouse.io{href}"
+                })
+
+        await browser.close()
+        return jobs

agentatlas-0.1.0/agentatlas/schemas.py

@@ -0,0 +1,149 @@
+ATS_SITES = [
+    {"domain": "greenhouse.io", "display_name": "Greenhouse", "requires_login": False, "anti_bot_hint": "low/medium"},
+    {"domain": "lever.co", "display_name": "Lever", "requires_login": False, "anti_bot_hint": "low/medium"},
+    {"domain": "myworkdayjobs.com", "display_name": "Workday Jobs", "requires_login": False, "anti_bot_hint": "medium/high"},
+    {"domain": "icims.com", "display_name": "iCIMS", "requires_login": False, "anti_bot_hint": "medium"},
+    {"domain": "taleo.net", "display_name": "Taleo", "requires_login": False, "anti_bot_hint": "medium/high"},
+    {"domain": "smartrecruiters.com", "display_name": "SmartRecruiters", "requires_login": False, "anti_bot_hint": "low/medium"},
+    {"domain": "ashbyhq.com", "display_name": "Ashby", "requires_login": False, "anti_bot_hint": "low/medium"},
+]
+
+TASKS = [
+    {
+        "task_key": "extract_job_list",
+        "description": "Extract a list of jobs from a job listing page.",
+        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
+        "output_schema": {
+            "type": "object",
+            "properties": {
+                "jobs": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {"title": {"type": "string"}, "location": {"type": "string"}, "url": {"type": "string"}},
+                        "required": ["title", "url"],
+                    },
+                }
+            },
+            "required": ["jobs"],
+        },
+    },
+    {
+        "task_key": "extract_job_detail",
+        "description": "Extract structured job detail from a job detail page.",
+        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
+        "output_schema": {
+            "type": "object",
+            "properties": {
+                "title": {"type": "string"},
+                "company": {"type": "string"},
+                "location": {"type": "string"},
+                "description_text": {"type": "string"},
+                "apply_url": {"type": "string"},
+            },
+            "required": ["title", "description_text"],
+        },
+    },
+    {
+        "task_key": "start_application",
+        "description": "Navigate from job detail page to application form page (or external apply).",
+        "input_schema": {"type": "object", "properties": {"url": {"type": "string"}}, "required": ["url"]},
+        "output_schema": {"type": "object", "properties": {"application_url": {"type": "string"}}, "required": ["application_url"]},
+    },
+]
+
+ROUTES_BY_DOMAIN = {
+    "greenhouse.io": [
+        {"route_key": "job_list", "path_pattern": r"^/[^/]+/?$"},
+        {"route_key": "job_detail", "path_pattern": r"^/[^/]+/jobs/\d+.*$"},
+    ],
+    "lever.co": [
+        {"route_key": "job_list", "path_pattern": r"^/[^/]+/?$"},
+        {"route_key": "job_detail", "path_pattern": r"^/[^/]+/[^/]+/?$"},
+    ],
+    "myworkdayjobs.com": [
+        {"route_key": "job_list", "path_pattern": r"^/[^/]+/[^/]+/.*$"},
+        {"route_key": "job_detail", "path_pattern": r"^/[^/]+/[^/]+/job/.*$"},
+    ],
+    "icims.com": [
+        {"route_key": "job_list", "path_pattern": r"^/jobs/.*$"},
+        {"route_key": "job_detail", "path_pattern": r"^/jobs/\d+/.*$"},
+    ],
+    "taleo.net": [
+        {"route_key": "job_list", "path_pattern": r"^/careersection/.*$"},
+        {"route_key": "job_detail", "path_pattern": r"^/careersection/.*?/jobdetail\.ftl.*$"},
+    ],
+    "smartrecruiters.com": [
+        {"route_key": "job_list", "path_pattern": r"^/[^/]+/jobs.*$"},
+        {"route_key": "job_detail", "path_pattern": r"^/[^/]+/job/.*$"},
+    ],
+    "ashbyhq.com": [
+        {"route_key": "job_list", "path_pattern": r"^/[^/]+/jobs.*$"},
+        {"route_key": "job_detail", "path_pattern": r"^/[^/]+/jobs/.*$"},
+    ],
+}
+
+def default_playbook_payload(task_key: str) -> dict:
+    """
+    Starter payload template (generic). You’ll refine per ATS platform later.
+    """
+    if task_key == "extract_job_list":
+        return {
+            "context": {"viewport": "desktop", "locale": "en-US", "logged_in": False},
+            "fingerprint": {"dom_hash": None, "anchors": ["Jobs", "Apply", "Location"]},
+            "preconditions": [{"type": "cookie_banner", "action": "accept_if_present"}],
+            "locators": {
+                "job_links": [
+                    {"type": "role", "value": "link", "priority": 1},
+                    {"type": "heuristic", "value": "links_that_look_like_job_details()", "priority": 2},
+                ]
+            },
+            "extraction": [{"field": "jobs", "from": "job_links", "transform": "extract_title_location_url"}],
+            "validation": [{"field": "jobs", "op": "min_items", "value": 1}],
+            "network": None,
+        }
+
+    if task_key == "extract_job_detail":
+        return {
+            "context": {"viewport": "desktop", "locale": "en-US", "logged_in": False},
+            "fingerprint": {"dom_hash": None, "anchors": ["Job Description", "Responsibilities", "Apply"]},
+            "preconditions": [{"type": "cookie_banner", "action": "accept_if_present"}],
+            "locators": {
+                "title": [
+                    {"type": "role", "value": "heading", "priority": 1},
+                    {"type": "css", "value": "h1", "priority": 2},
+                ],
+                "description": [{"type": "heuristic", "value": "main_content_text_block()", "priority": 1}],
+                "apply_link": [
+                    {"type": "text", "value": "Apply", "priority": 1},
+                    {"type": "role", "value": "link[name~='Apply']", "priority": 2},
+                    {"type": "role", "value": "button[name~='Apply']", "priority": 3},
+                ],
+            },
+            "extraction": [
+                {"field": "title", "from": "title.text"},
+                {"field": "description_text", "from": "description.text"},
+                {"field": "apply_url", "from": "apply_link.href", "optional": True},
+            ],
+            "validation": [{"field": "title", "op": "non_empty"}, {"field": "description_text", "op": "min_len", "value": 200}],
+            "network": None,
+        }
+
+    if task_key == "start_application":
+        return {
+            "context": {"viewport": "desktop", "locale": "en-US", "logged_in": False},
+            "fingerprint": {"dom_hash": None, "anchors": ["Apply", "Submit", "Continue"]},
+            "preconditions": [{"type": "cookie_banner", "action": "accept_if_present"}],
+            "locators": {
+                "apply_link": [
+                    {"type": "text", "value": "Apply", "priority": 1},
+                    {"type": "role", "value": "link[name~='Apply']", "priority": 2},
+                    {"type": "role", "value": "button[name~='Apply']", "priority": 3},
+                ]
+            },
+            "extraction": [{"field": "application_url", "from": "apply_link.href_or_navigation_url"}],
+            "validation": [{"field": "application_url", "op": "regex", "value": r"^https?://"}],
+            "network": None,
+        }
+
+    return {}

agentatlas-0.1.0/agentatlas/seed_registry.py

@@ -0,0 +1,100 @@
+from dotenv import load_dotenv
+from agentatlas.supabase_client import get_supabase
+from agentatlas.schemas import ATS_SITES, TASKS, ROUTES_BY_DOMAIN, default_playbook_payload
+
+load_dotenv()
+
+def get_id_by_unique(sb, table: str, col: str, value: str):
+    res = sb.table(table).select("id").eq(col, value).limit(1).execute()
+    data = res.data or []
+    return data[0]["id"] if data else None
+
+def main():
+    sb = get_supabase()
+
+    # 1) Upsert tasks
+    print("Seeding tasks...")
+    for t in TASKS:
+        sb.table("tasks").upsert(
+            {
+                "task_key": t["task_key"],
+                "description": t.get("description"),
+                "input_schema": t.get("input_schema", {}),
+                "output_schema": t.get("output_schema", {}),
+            },
+            on_conflict="task_key",
+        ).execute()
+
+    # 2) Upsert sites, routes, playbooks
+    print("Seeding sites, routes, playbooks...")
+    for s in ATS_SITES:
+        sb.table("sites").upsert(
+            {
+                "domain": s["domain"],
+                "display_name": s.get("display_name"),
+                "base_url": s.get("base_url"),
+                "requires_login": s.get("requires_login", False),
+                "anti_bot_hint": s.get("anti_bot_hint"),
+                "notes": s.get("notes"),
+            },
+            on_conflict="domain",
+        ).execute()
+
+        site_id = get_id_by_unique(sb, "sites", "domain", s["domain"])
+        if not site_id:
+            raise RuntimeError(f"Failed to fetch site_id for domain={s['domain']}")
+
+        routes = ROUTES_BY_DOMAIN.get(s["domain"], [])
+        for r in routes:
+            sb.table("page_routes").upsert(
+                {
+                    "site_id": site_id,
+                    "route_key": r["route_key"],
+                    "path_pattern": r["path_pattern"],
+                    "example_url": r.get("example_url"),
+                },
+                on_conflict="site_id,route_key",
+            ).execute()
+
+            route_id = (
+                sb.table("page_routes")
+                .select("id")
+                .eq("site_id", site_id)
+                .eq("route_key", r["route_key"])
+                .limit(1)
+                .execute()
+                .data[0]["id"]
+            )
+
+            for task in TASKS:
+                task_key = task["task_key"]
+
+                # Map which tasks apply to which routes
+                if r["route_key"] == "job_list" and task_key not in ("extract_job_list",):
+                    continue
+                if r["route_key"] == "job_detail" and task_key not in ("extract_job_detail", "start_application"):
+                    continue
+
+                task_id = get_id_by_unique(sb, "tasks", "task_key", task_key)
+                if not task_id:
+                    raise RuntimeError(f"Failed to fetch task_id for task_key={task_key}")
+
+                sb.table("playbooks").upsert(
+                    {
+                        "site_id": site_id,
+                        "route_id": route_id,
+                        "task_id": task_id,
+                        "variant_key": "desktop_enUS_loggedout",
+                        "version": 1,
+                        "status": "experimental",
+                        "confidence": 0.2,
+                        "ttl_days": 14,
+                        "payload": default_playbook_payload(task_key),
+                    },
+                    on_conflict="site_id,route_id,task_id,variant_key,version",
+                ).execute()
+
+    print("✅ Seed complete.")
+
+if __name__ == "__main__":
+    main()

agentatlas-0.1.0/agentatlas/supabase_client.py

@@ -0,0 +1,9 @@
+import os
+from supabase import create_client, Client
+
+def get_supabase() -> Client:
+    url = os.getenv("SUPABASE_URL", "").strip()
+    key = os.getenv("SUPABASE_SERVICE_ROLE_KEY", "").strip()
+    if not url or not key:
+        raise RuntimeError("Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY in environment.")
+    return create_client(url, key)

agentatlas-0.1.0/agentatlas.egg-info/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.4
+Name: agentatlas
+Version: 0.1.0
+Summary: Shared browser interaction schema registry for AI agents. Reduces LLM token usage by 80-100% on known sites.
+License: MIT
+Project-URL: Homepage, https://github.com/yourusername/agentatlas
+Project-URL: Issues, https://github.com/yourusername/agentatlas/issues
+Keywords: ai,agents,browser-automation,llm,playwright
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: supabase>=2.6.0
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: playwright>=1.44.0
+Requires-Dist: openai>=1.109.1
+Requires-Dist: playwright-stealth>=1.0.6
+
+# AgentAtlas
+
+**Shared browser interaction schema registry for AI agents.**
+
+Reduces LLM token usage by 80-100% on known sites by storing and sharing site interaction schemas across all users.
+
+## How it works
+```
+First user  → LLM learns the site → saved to shared registry
+Every user after → 0 tokens, instant response
+```
+
+## Benchmark results (real data)
+
+| | Without AgentAtlas | With AgentAtlas |
+|---|---|---|
+| Tokens | 2,597 | 0-445 |
+| Cost | $0.018 | $0.000-$0.002 |
+| Time | 19s | 0.2-12s |
+| Real URLs | ❌ | ✅ |
+
+**82.9% token reduction** when LLM still needed. **100% reduction** for repeat workflows.
+
+## Install
+```bash
+pip install agentatlas
+playwright install chromium
+```
+
+## Usage
+```python
+from agentatlas.atlas import Atlas
+
+atlas = Atlas()
+
+# Get schema for any site
+# Found in registry → 0 tokens
+# New site → learns once, saves for everyone
+schema = await atlas.get_schema(
+    site="greenhouse.io",
+    url="https://boards.greenhouse.io/anthropic"
+)
+
+# Pass compact schema to YOUR LLM
+# 150-500 tokens instead of 50,000
+print(schema.elements)
+print(schema.tokens_used)  # 0 if registry hit
+print(schema.source)       # "registry" or "llm_learned"
+```
+
+## Environment variables
+```bash
+SUPABASE_URL=your_supabase_url
+SUPABASE_SERVICE_ROLE_KEY=your_key
+OPENAI_API_KEY=your_key
+```
+
+## The flywheel
+```
+More developers use AgentAtlas
+        ↓
+More new sites get learned automatically
+        ↓
+Registry grows → higher hit rate
+        ↓
+Less tokens burned across the whole network
+        ↓
+Cheaper + faster → more developers adopt
+```
+
+## License
+
+MIT

agentatlas-0.1.0/agentatlas.egg-info/SOURCES.txt

@@ -0,0 +1,14 @@
+README.md
+pyproject.toml
+agentatlas/__init__.py
+agentatlas/atlas.py
+agentatlas/crawler.py
+agentatlas/executor.py
+agentatlas/schemas.py
+agentatlas/seed_registry.py
+agentatlas/supabase_client.py
+agentatlas.egg-info/PKG-INFO
+agentatlas.egg-info/SOURCES.txt
+agentatlas.egg-info/dependency_links.txt
+agentatlas.egg-info/requires.txt
+agentatlas.egg-info/top_level.txt

agentatlas-0.1.0/agentatlas.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

agentatlas-0.1.0/agentatlas.egg-info/requires.txt

@@ -0,0 +1,5 @@
+supabase>=2.6.0
+python-dotenv>=1.0.1
+playwright>=1.44.0
+openai>=1.109.1
+playwright-stealth>=1.0.6

agentatlas-0.1.0/agentatlas.egg-info/top_level.txt

@@ -0,0 +1 @@
+agentatlas

agentatlas-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "agentatlas"
+version = "0.1.0"
+description = "Shared browser interaction schema registry for AI agents. Reduces LLM token usage by 80-100% on known sites."
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.10"
+keywords = ["ai", "agents", "browser-automation", "llm", "playwright"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "supabase>=2.6.0",
+    "python-dotenv>=1.0.1",
+    "playwright>=1.44.0",
+    "openai>=1.109.1",
+    "playwright-stealth>=1.0.6",
+]
+
+[project.urls]
+Homepage = "https://github.com/yourusername/agentatlas"
+Issues = "https://github.com/yourusername/agentatlas/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["agentatlas*"]

agentatlas-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+