botbrowser 0.1.0__tar.gz

botbrowser-0.1.0/.gitignore

@@ -0,0 +1,13 @@
+node_modules/
+dist/
+build/
+*.tsbuildinfo
+.env
+*.egg-info/
+__pycache__/
+.pytest_cache/
+*.pyc
+.venv/
+venv/
+.DS_Store
+coverage/

botbrowser-0.1.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.4
+Name: botbrowser
+Version: 0.1.0
+Summary: Token-efficient web content extraction for LLM agents
+Project-URL: Homepage, https://github.com/AmplifyCo/botbrowser
+Project-URL: Repository, https://github.com/AmplifyCo/botbrowser
+Project-URL: Issues, https://github.com/AmplifyCo/botbrowser/issues
+License-Expression: MIT
+Keywords: ai-agents,content-extraction,llm,token-optimization,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12.0
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: markdownify>=0.13.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: trafilatura>=1.12.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# BotBrowser (Python)
+
+Token-efficient web content extraction for LLM agents.
+
+## Install
+
+```bash
+pip install botbrowser
+```
+
+## Usage
+
+```python
+from botbrowser import extract
+
+result = extract("https://example.com")
+
+print(result.content)                          # clean markdown
+print(result.metadata.token_savings_percent)   # e.g. 94
+print(result.title)                            # page title
+print(result.links)                            # extracted links
+```
+
+## Options
+
+```python
+result = extract(
+    "https://example.com",
+    format="text",          # "markdown" (default) or "text"
+    timeout=10000,          # request timeout in ms (default: 15000)
+    include_links=False,    # extract links (default: True)
+)
+```
+
+## Client Mode
+
+If you're running the BotBrowser REST API server, you can use the client:
+
+```python
+from botbrowser import BotBrowserClient
+
+client = BotBrowserClient("http://localhost:3000")
+result = client.extract("https://example.com")
+```
+
+## License
+
+MIT

botbrowser-0.1.0/README.md

@@ -0,0 +1,48 @@
+# BotBrowser (Python)
+
+Token-efficient web content extraction for LLM agents.
+
+## Install
+
+```bash
+pip install botbrowser
+```
+
+## Usage
+
+```python
+from botbrowser import extract
+
+result = extract("https://example.com")
+
+print(result.content)                          # clean markdown
+print(result.metadata.token_savings_percent)   # e.g. 94
+print(result.title)                            # page title
+print(result.links)                            # extracted links
+```
+
+## Options
+
+```python
+result = extract(
+    "https://example.com",
+    format="text",          # "markdown" (default) or "text"
+    timeout=10000,          # request timeout in ms (default: 15000)
+    include_links=False,    # extract links (default: True)
+)
+```
+
+## Client Mode
+
+If you're running the BotBrowser REST API server, you can use the client:
+
+```python
+from botbrowser import BotBrowserClient
+
+client = BotBrowserClient("http://localhost:3000")
+result = client.extract("https://example.com")
+```
+
+## License
+
+MIT

botbrowser-0.1.0/botbrowser/__init__.py

@@ -0,0 +1,15 @@
+"""BotBrowser — Token-efficient web content extraction for LLM agents."""
+
+from botbrowser.core import extract
+from botbrowser.client import BotBrowserClient
+from botbrowser.models import BotBrowserResult, ExtractOptions, ExtractedLink, ExtractionMetadata
+
+__version__ = "0.1.0"
+__all__ = [
+    "extract",
+    "BotBrowserClient",
+    "BotBrowserResult",
+    "ExtractOptions",
+    "ExtractedLink",
+    "ExtractionMetadata",
+]

botbrowser-0.1.0/botbrowser/cleaner.py

@@ -0,0 +1,99 @@
+"""HTML cleaning pipeline — strips bloat from web pages."""
+
+from __future__ import annotations
+
+from bs4 import BeautifulSoup, Comment, Tag
+
+REMOVE_TAGS = {
+    "script", "style", "noscript", "iframe", "object", "embed",
+    "applet", "svg", "canvas", "video", "audio", "source", "track",
+    "map", "area",
+}
+
+REMOVE_SELECTORS = [
+    "nav",
+    "[role='navigation']",
+    "[role='banner']",
+    "[role='complementary']",
+    "[aria-hidden='true']",
+    ".ad", ".ads", ".advertisement", ".adsbygoogle",
+    ".sidebar", ".side-bar",
+    ".cookie-banner", ".cookie-notice", ".cookie-consent",
+    ".popup", ".modal", ".overlay",
+    ".social-share", ".share-buttons", ".social-links",
+    ".comments", ".comment-section", "#comments",
+    ".newsletter", ".subscribe",
+    ".breadcrumb", ".breadcrumbs",
+    ".pagination",
+    ".related-posts", ".related-articles",
+    ".widget",
+    "[data-ad]",
+    "[data-tracking]",
+]
+
+STRIP_ATTR_PREFIXES = ("data-", "aria-", "on")
+
+SELF_CLOSING = {"img", "br", "hr", "input", "meta", "link"}
+
+
+def clean_html(html: str) -> str:
+    """Remove non-content elements and unnecessary attributes from HTML."""
+    soup = BeautifulSoup(html, "html.parser")
+
+    # Remove HTML comments
+    for comment in soup.find_all(string=lambda t: isinstance(t, Comment)):
+        comment.extract()
+
+    # Remove unwanted tags entirely
+    for tag_name in REMOVE_TAGS:
+        for tag in soup.find_all(tag_name):
+            tag.decompose()
+
+    # Remove non-content elements by selector
+    for selector in REMOVE_SELECTORS:
+        try:
+            for el in soup.select(selector):
+                el.decompose()
+        except Exception:
+            pass
+
+    # Remove hidden elements
+    for el in soup.find_all(True):
+        if not isinstance(el, Tag):
+            continue
+        if el.has_attr("hidden"):
+            el.decompose()
+            continue
+        style = el.get("style", "")
+        if isinstance(style, str) and (
+            "display:none" in style.replace(" ", "")
+            or "visibility:hidden" in style.replace(" ", "")
+        ):
+            el.decompose()
+
+    # Strip unnecessary attributes
+    for el in soup.find_all(True):
+        if not isinstance(el, Tag):
+            continue
+        attrs_to_remove = []
+        for attr_name in list(el.attrs.keys()):
+            lower = attr_name.lower()
+            if lower in ("style", "class", "id", "role", "tabindex", "draggable", "contenteditable"):
+                attrs_to_remove.append(attr_name)
+            elif any(lower.startswith(p) for p in STRIP_ATTR_PREFIXES):
+                attrs_to_remove.append(attr_name)
+        for attr_name in attrs_to_remove:
+            del el[attr_name]
+
+    # Remove empty elements
+    for el in soup.find_all(True):
+        if not isinstance(el, Tag):
+            continue
+        if (
+            el.name not in SELF_CLOSING
+            and not el.get_text(strip=True)
+            and not el.find("img")
+        ):
+            el.decompose()
+
+    return str(soup)

botbrowser-0.1.0/botbrowser/client.py

@@ -0,0 +1,79 @@
+"""HTTP client for BotBrowser REST API server."""
+
+from __future__ import annotations
+
+import httpx
+
+from botbrowser.models import BotBrowserResult
+
+
+class BotBrowserClient:
+    """
+    Client for the BotBrowser REST API server.
+
+    Usage:
+        client = BotBrowserClient("http://localhost:3000")
+        result = client.extract("https://example.com")
+        print(result.content)
+    """
+
+    def __init__(self, server_url: str = "http://localhost:3000") -> None:
+        self.server_url = server_url.rstrip("/")
+        self._client = httpx.Client(timeout=30)
+
+    def extract(
+        self,
+        url: str,
+        *,
+        format: str = "markdown",
+        timeout: int = 15000,
+        include_links: bool = True,
+    ) -> BotBrowserResult:
+        """Extract content via the BotBrowser REST API server."""
+        response = self._client.post(
+            f"{self.server_url}/extract",
+            json={
+                "url": url,
+                "format": format,
+                "timeout": timeout,
+                "includeLinks": include_links,
+            },
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        # Map camelCase API response to snake_case Python model
+        return BotBrowserResult(
+            url=data["url"],
+            title=data["title"],
+            description=data["description"],
+            content=data["content"],
+            text_content=data.get("textContent", data.get("text_content", "")),
+            links=[
+                {"text": link["text"], "href": link["href"]}
+                for link in data.get("links", [])
+            ],
+            metadata={
+                "raw_token_estimate": data["metadata"].get("rawTokenEstimate", data["metadata"].get("raw_token_estimate", 0)),
+                "clean_token_estimate": data["metadata"].get("cleanTokenEstimate", data["metadata"].get("clean_token_estimate", 0)),
+                "token_savings_percent": data["metadata"].get("tokenSavingsPercent", data["metadata"].get("token_savings_percent", 0)),
+                "word_count": data["metadata"].get("wordCount", data["metadata"].get("word_count", 0)),
+                "fetched_at": data["metadata"].get("fetchedAt", data["metadata"].get("fetched_at", "")),
+            },
+        )
+
+    def health(self) -> dict:
+        """Check server health."""
+        response = self._client.get(f"{self.server_url}/health")
+        response.raise_for_status()
+        return response.json()
+
+    def close(self) -> None:
+        """Close the HTTP client."""
+        self._client.close()
+
+    def __enter__(self) -> BotBrowserClient:
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()

botbrowser-0.1.0/botbrowser/converter.py

@@ -0,0 +1,49 @@
+"""HTML to Markdown/text conversion."""
+
+from __future__ import annotations
+
+import re
+
+from markdownify import markdownify
+
+
+def html_to_markdown(html: str) -> str:
+    """Convert HTML to clean markdown."""
+    # Pre-process: remove images with data URIs (bloated base64) but keep normal images
+    html = re.sub(r'<img[^>]+src="data:[^"]*"[^>]*/?>', "", html)
+
+    markdown = markdownify(
+        html,
+        heading_style="ATX",
+        bullets="-",
+        convert=["a", "p", "h1", "h2", "h3", "h4", "h5", "h6",
+                 "ul", "ol", "li", "table", "thead", "tbody", "tr", "th", "td",
+                 "blockquote", "pre", "code", "em", "strong", "br", "hr", "img"],
+    )
+
+    # Clean up excessive whitespace
+    markdown = re.sub(r"\n{3,}", "\n\n", markdown)  # Max 2 consecutive newlines
+    markdown = re.sub(r"[ \t]+$", "", markdown, flags=re.MULTILINE)  # Trailing whitespace
+    markdown = markdown.strip()
+
+    return markdown
+
+
+def html_to_text(html: str) -> str:
+    """Convert HTML to plain text."""
+    markdown = html_to_markdown(html)
+
+    text = markdown
+    text = re.sub(r"#{1,6}\s+", "", text)  # Remove heading markers
+    text = re.sub(r"\*\*(.+?)\*\*", r"\1", text)  # Remove bold
+    text = re.sub(r"\*(.+?)\*", r"\1", text)  # Remove italic
+    text = re.sub(r"\[(.+?)\]\(.+?\)", r"\1", text)  # Remove links, keep text
+    text = re.sub(r"!\[.*?\]\(.+?\)", "", text)  # Remove images
+    text = re.sub(r"`{1,3}[^`]*`{1,3}", lambda m: m.group().strip("`"), text)  # Remove code markers
+    text = re.sub(r"^[-*+]\s+", "", text, flags=re.MULTILINE)  # Remove list markers
+    text = re.sub(r"^\d+\.\s+", "", text, flags=re.MULTILINE)  # Remove numbered list markers
+    text = re.sub(r"^>\s+", "", text, flags=re.MULTILINE)  # Remove blockquote markers
+    text = text.replace("---", "")  # Remove horizontal rules
+    text = re.sub(r"\n{3,}", "\n\n", text)
+
+    return text.strip()

botbrowser-0.1.0/botbrowser/core.py

@@ -0,0 +1,186 @@
+"""Core extraction engine — native Python implementation."""
+
+from __future__ import annotations
+
+import math
+from datetime import datetime, timezone
+from urllib.parse import urljoin, urlparse
+
+from bs4 import BeautifulSoup
+
+from botbrowser.cleaner import clean_html
+from botbrowser.converter import html_to_markdown, html_to_text
+from botbrowser.fetcher import fetch_page
+from botbrowser.models import (
+    BotBrowserResult,
+    ExtractedLink,
+    ExtractionMetadata,
+    ExtractOptions,
+)
+
+import trafilatura
+
+
+def _estimate_tokens(text: str) -> int:
+    """Rough token estimation: ~4 chars per token for English text."""
+    return math.ceil(len(text) / 4)
+
+
+def _extract_description(html: str) -> str:
+    """Extract meta description from HTML."""
+    soup = BeautifulSoup(html, "html.parser")
+
+    meta = soup.find("meta", attrs={"name": "description"})
+    if meta and meta.get("content"):
+        return str(meta["content"])
+
+    og = soup.find("meta", attrs={"property": "og:description"})
+    if og and og.get("content"):
+        return str(og["content"])
+
+    return ""
+
+
+def _extract_title(html: str) -> str:
+    """Extract page title from HTML."""
+    soup = BeautifulSoup(html, "html.parser")
+    title_tag = soup.find("title")
+    if title_tag and title_tag.string:
+        return title_tag.string.strip()
+
+    og_title = soup.find("meta", attrs={"property": "og:title"})
+    if og_title and og_title.get("content"):
+        return str(og_title["content"])
+
+    return ""
+
+
+def _extract_links(html: str, base_url: str) -> list[ExtractedLink]:
+    """Extract unique links from HTML content."""
+    soup = BeautifulSoup(html, "html.parser")
+    links: list[ExtractedLink] = []
+    seen: set[str] = set()
+
+    for a in soup.find_all("a", href=True):
+        href = a["href"]
+
+        # Resolve relative URLs
+        try:
+            absolute_url = urljoin(base_url, href)
+        except Exception:
+            continue
+
+        # Skip non-HTTP, anchors, mailto, tel
+        parsed = urlparse(absolute_url)
+        if parsed.scheme not in ("http", "https"):
+            continue
+        if parsed.fragment and parsed.path == urlparse(base_url).path:
+            continue
+
+        if absolute_url in seen:
+            continue
+        seen.add(absolute_url)
+
+        text = a.get_text(strip=True)
+        if text:
+            links.append(ExtractedLink(text=text, href=absolute_url))
+
+    return links
+
+
+def extract(
+    url_or_options: str | ExtractOptions | None = None,
+    *,
+    url: str | None = None,
+    format: str = "markdown",
+    timeout: int = 15000,
+    include_links: bool = True,
+    headers: dict[str, str] | None = None,
+) -> BotBrowserResult:
+    """
+    Extract clean, token-efficient content from a web page.
+
+    Usage:
+        result = extract("https://example.com")
+        result = extract("https://example.com", format="text")
+        result = extract(ExtractOptions(url="https://example.com"))
+    """
+    # Normalize arguments
+    if isinstance(url_or_options, ExtractOptions):
+        opts = url_or_options
+    elif isinstance(url_or_options, str):
+        opts = ExtractOptions(
+            url=url_or_options,
+            format=format,  # type: ignore[arg-type]
+            timeout=timeout,
+            include_links=include_links,
+            headers=headers,
+        )
+    elif url is not None:
+        opts = ExtractOptions(
+            url=url,
+            format=format,  # type: ignore[arg-type]
+            timeout=timeout,
+            include_links=include_links,
+            headers=headers,
+        )
+    else:
+        raise ValueError("url is required")
+
+    # Step 1: Fetch the page
+    fetched = fetch_page(opts.url, timeout=opts.timeout, headers=opts.headers)
+    raw_token_estimate = _estimate_tokens(fetched.html)
+
+    # Step 2: Extract metadata from raw HTML
+    title = _extract_title(fetched.html)
+    description = _extract_description(fetched.html)
+
+    # Step 3: Extract main content using trafilatura
+    main_content_html = trafilatura.extract(
+        fetched.html,
+        output_format="html",
+        include_links=True,
+        include_tables=True,
+        include_formatting=True,
+    )
+
+    # Step 4: Clean HTML
+    if main_content_html:
+        cleaned_html = clean_html(main_content_html)
+    else:
+        # Fallback: clean the full page HTML
+        cleaned_html = clean_html(fetched.html)
+
+    # Step 5: Convert to desired format
+    if opts.format == "markdown":
+        content = html_to_markdown(cleaned_html)
+    else:
+        content = html_to_text(cleaned_html)
+
+    text_content = html_to_text(cleaned_html)
+
+    # Step 6: Extract links from raw HTML (not cleaned — cleaning strips nav links)
+    links = _extract_links(fetched.html, fetched.final_url) if opts.include_links else []
+
+    clean_token_estimate = _estimate_tokens(content)
+    savings = (
+        round((1 - clean_token_estimate / raw_token_estimate) * 100)
+        if raw_token_estimate > 0
+        else 0
+    )
+
+    return BotBrowserResult(
+        url=fetched.final_url,
+        title=title,
+        description=description,
+        content=content,
+        text_content=text_content,
+        links=links,
+        metadata=ExtractionMetadata(
+            raw_token_estimate=raw_token_estimate,
+            clean_token_estimate=clean_token_estimate,
+            token_savings_percent=savings,
+            word_count=len(text_content.split()),
+            fetched_at=datetime.now(timezone.utc).isoformat(),
+        ),
+    )

botbrowser-0.1.0/botbrowser/fetcher.py

@@ -0,0 +1,63 @@
+"""HTTP fetching with smart defaults."""
+
+from __future__ import annotations
+
+import random
+from dataclasses import dataclass
+
+import httpx
+
+USER_AGENTS = [
+    "Mozilla/5.0 (compatible; BotBrowser/0.1; +https://github.com/AmplifyCo/botbrowser)",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+]
+
+
+@dataclass
+class FetchResult:
+    html: str
+    final_url: str
+    status_code: int
+    content_type: str
+
+
+def fetch_page(
+    url: str,
+    *,
+    timeout: int = 15000,
+    headers: dict[str, str] | None = None,
+) -> FetchResult:
+    """Fetch a web page with smart defaults."""
+    user_agent = random.choice(USER_AGENTS)
+
+    default_headers = {
+        "User-Agent": user_agent,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+    if headers:
+        default_headers.update(headers)
+
+    timeout_seconds = timeout / 1000
+
+    response = httpx.get(
+        url,
+        headers=default_headers,
+        follow_redirects=True,
+        timeout=timeout_seconds,
+    )
+    response.raise_for_status()
+
+    content_type = response.headers.get("content-type", "")
+    if "text/html" not in content_type and "application/xhtml" not in content_type:
+        raise ValueError(
+            f"Unsupported content type: {content_type}. Only HTML pages are supported."
+        )
+
+    return FetchResult(
+        html=response.text,
+        final_url=str(response.url),
+        status_code=response.status_code,
+        content_type=content_type,
+    )

botbrowser-0.1.0/botbrowser/models.py

@@ -0,0 +1,45 @@
+"""Data models for BotBrowser."""
+
+from __future__ import annotations
+
+from typing import Literal
+from pydantic import BaseModel
+
+
+class ExtractOptions(BaseModel):
+    """Options for content extraction."""
+
+    url: str
+    format: Literal["markdown", "text"] = "markdown"
+    timeout: int = 15000
+    include_links: bool = True
+    headers: dict[str, str] | None = None
+
+
+class ExtractedLink(BaseModel):
+    """A link extracted from page content."""
+
+    text: str
+    href: str
+
+
+class ExtractionMetadata(BaseModel):
+    """Metadata about the extraction including token savings."""
+
+    raw_token_estimate: int
+    clean_token_estimate: int
+    token_savings_percent: int
+    word_count: int
+    fetched_at: str
+
+
+class BotBrowserResult(BaseModel):
+    """Result of content extraction from a web page."""
+
+    url: str
+    title: str
+    description: str
+    content: str
+    text_content: str
+    links: list[ExtractedLink]
+    metadata: ExtractionMetadata

botbrowser-0.1.0/pyproject.toml

@@ -0,0 +1,47 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "botbrowser"
+version = "0.1.0"
+description = "Token-efficient web content extraction for LLM agents"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+keywords = ["web-scraping", "content-extraction", "llm", "ai-agents", "token-optimization"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "trafilatura>=1.12.0",
+    "httpx>=0.27.0",
+    "markdownify>=0.13.0",
+    "pydantic>=2.0.0",
+    "beautifulsoup4>=4.12.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/AmplifyCo/botbrowser"
+Repository = "https://github.com/AmplifyCo/botbrowser"
+Issues = "https://github.com/AmplifyCo/botbrowser/issues"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["botbrowser"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

botbrowser-0.1.0/tests/test_core.py

@@ -0,0 +1,249 @@
+"""Tests for BotBrowser core extraction."""
+
+from botbrowser.cleaner import clean_html
+from botbrowser.converter import html_to_markdown, html_to_text
+from botbrowser.models import BotBrowserResult, ExtractOptions, ExtractedLink, ExtractionMetadata
+from botbrowser.core import _extract_title, _extract_description, _extract_links, _estimate_tokens
+
+
+SAMPLE_HTML = """
+<html>
+<head>
+    <title>Test Page</title>
+    <meta name="description" content="A test page for BotBrowser">
+    <script>console.log('tracking');</script>
+    <style>body { color: red; }</style>
+</head>
+<body>
+    <nav><a href="/">Home</a> <a href="/about">About</a></nav>
+    <main>
+        <h1>Hello World</h1>
+        <p>This is the <strong>main content</strong> of the page.</p>
+        <p>It has <a href="https://example.com">a link</a> and some text.</p>
+        <ul>
+            <li>Item one</li>
+            <li>Item two</li>
+            <li>Item three</li>
+        </ul>
+    </main>
+    <footer>Copyright 2026</footer>
+    <div class="cookie-banner">We use cookies</div>
+    <div style="display:none">Hidden content</div>
+</body>
+</html>
+"""
+
+
+# --- Cleaner tests ---
+
+def test_clean_html_removes_scripts():
+    result = clean_html(SAMPLE_HTML)
+    assert "<script>" not in result
+    assert "tracking" not in result
+
+
+def test_clean_html_removes_styles():
+    result = clean_html(SAMPLE_HTML)
+    assert "<style>" not in result
+
+
+def test_clean_html_removes_nav():
+    result = clean_html(SAMPLE_HTML)
+    assert "<nav>" not in result
+
+
+def test_clean_html_removes_cookie_banner():
+    result = clean_html(SAMPLE_HTML)
+    assert "cookie" not in result.lower()
+
+
+def test_clean_html_removes_hidden():
+    result = clean_html(SAMPLE_HTML)
+    assert "Hidden content" not in result
+
+
+def test_clean_html_preserves_main_content():
+    result = clean_html(SAMPLE_HTML)
+    assert "Hello World" in result
+    assert "main content" in result
+
+
+def test_clean_html_strips_attributes():
+    html = '<div class="foo" id="bar" data-track="yes" style="color:red"><p>Text</p></div>'
+    result = clean_html(html)
+    assert 'class=' not in result
+    assert 'id=' not in result
+    assert 'data-track' not in result
+    assert 'style=' not in result
+
+
+def test_clean_html_removes_ads():
+    html = '<div class="ad">Buy stuff</div><p>Real content</p>'
+    result = clean_html(html)
+    assert "Buy stuff" not in result
+    assert "Real content" in result
+
+
+def test_clean_html_handles_empty_input():
+    result = clean_html("")
+    assert isinstance(result, str)
+
+
+# --- Converter tests ---
+
+def test_html_to_markdown():
+    html = "<h1>Title</h1><p>Hello <strong>world</strong></p>"
+    md = html_to_markdown(html)
+    assert "# Title" in md
+    assert "**world**" in md
+
+
+def test_html_to_markdown_links():
+    html = '<p><a href="https://example.com">click</a></p>'
+    md = html_to_markdown(html)
+    assert "[click](https://example.com)" in md
+
+
+def test_html_to_markdown_lists():
+    html = "<ul><li>One</li><li>Two</li></ul>"
+    md = html_to_markdown(html)
+    assert "- One" in md
+    assert "- Two" in md
+
+
+def test_html_to_markdown_strips_data_uri_images():
+    html = '<img src="data:image/png;base64,abc123" alt="bloat"><p>Text</p>'
+    md = html_to_markdown(html)
+    assert "data:" not in md
+    assert "Text" in md
+
+
+def test_html_to_markdown_keeps_normal_images():
+    html = '<img src="https://example.com/img.png" alt="photo"><p>Text</p>'
+    md = html_to_markdown(html)
+    assert "![photo](https://example.com/img.png)" in md
+
+
+def test_html_to_markdown_collapses_whitespace():
+    html = "<p>Hello</p>\n\n\n\n\n<p>World</p>"
+    md = html_to_markdown(html)
+    assert "\n\n\n" not in md
+
+
+def test_html_to_text():
+    html = "<h1>Title</h1><p>Hello <strong>world</strong></p>"
+    text = html_to_text(html)
+    assert "Title" in text
+    assert "world" in text
+    assert "#" not in text
+    assert "**" not in text
+
+
+def test_html_to_text_strips_links():
+    html = '<p><a href="https://example.com">click here</a></p>'
+    text = html_to_text(html)
+    assert "click here" in text
+    assert "https://example.com" not in text
+
+
+# --- Extractor helper tests ---
+
+def test_extract_title():
+    html = "<html><head><title>My Page</title></head><body></body></html>"
+    assert _extract_title(html) == "My Page"
+
+
+def test_extract_title_og_fallback():
+    html = '<html><head><meta property="og:title" content="OG Title"></head><body></body></html>'
+    assert _extract_title(html) == "OG Title"
+
+
+def test_extract_title_empty():
+    assert _extract_title("<html><head></head><body></body></html>") == ""
+
+
+def test_extract_description():
+    html = '<html><head><meta name="description" content="Test desc"></head><body></body></html>'
+    assert _extract_description(html) == "Test desc"
+
+
+def test_extract_description_og_fallback():
+    html = '<html><head><meta property="og:description" content="OG desc"></head><body></body></html>'
+    assert _extract_description(html) == "OG desc"
+
+
+def test_extract_description_empty():
+    assert _extract_description("<html><head></head><body></body></html>") == ""
+
+
+def test_extract_links():
+    html = """
+    <a href="https://example.com/a">Link A</a>
+    <a href="https://example.com/b">Link B</a>
+    <a href="https://example.com/a">Link A again</a>
+    """
+    links = _extract_links(html, "https://example.com")
+    assert len(links) == 2
+    assert links[0].text == "Link A"
+    assert links[0].href == "https://example.com/a"
+
+
+def test_extract_links_resolves_relative():
+    html = '<a href="/about">About</a>'
+    links = _extract_links(html, "https://example.com")
+    assert links[0].href == "https://example.com/about"
+
+
+def test_extract_links_skips_non_http():
+    html = """
+    <a href="javascript:void(0)">JS</a>
+    <a href="mailto:test@example.com">Email</a>
+    <a href="https://example.com/real">Real</a>
+    """
+    links = _extract_links(html, "https://example.com")
+    assert len(links) == 1
+    assert links[0].text == "Real"
+
+
+def test_extract_links_skips_same_page_anchors():
+    html = '<a href="#section">Jump</a>'
+    links = _extract_links(html, "https://example.com")
+    assert len(links) == 0
+
+
+# --- Token estimation ---
+
+def test_estimate_tokens():
+    assert _estimate_tokens("") == 0
+    assert _estimate_tokens("abcd") == 1
+    assert _estimate_tokens("abcdefgh") == 2
+
+
+# --- Models ---
+
+def test_extract_options_defaults():
+    opts = ExtractOptions(url="https://example.com")
+    assert opts.format == "markdown"
+    assert opts.timeout == 15000
+    assert opts.include_links is True
+    assert opts.headers is None
+
+
+def test_botbrowser_result_construction():
+    result = BotBrowserResult(
+        url="https://example.com",
+        title="Test",
+        description="Desc",
+        content="# Test",
+        text_content="Test",
+        links=[ExtractedLink(text="link", href="https://example.com")],
+        metadata=ExtractionMetadata(
+            raw_token_estimate=1000,
+            clean_token_estimate=100,
+            token_savings_percent=90,
+            word_count=50,
+            fetched_at="2026-01-01T00:00:00Z",
+        ),
+    )
+    assert result.url == "https://example.com"
+    assert result.metadata.token_savings_percent == 90