ember-browser 0.1.0__py3-none-any.whl

emb/__init__.py

@@ -0,0 +1,38 @@
+"""ember — open source, lightweight headless browser for AI agents."""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+# Lazily re-export the most-used public functions so `from emb import scrape_url`
+# works without loading heavy dependencies at `import emb` time.
+#
+# Names that clash with a same-named submodule (search, crawl) can't be re-exported
+# this way — Python returns the submodule before __getattr__ fires. Use the submodule
+# form for those: `from emb.search import search`, `from emb.crawl import crawl`.
+
+__all__ = [
+    "__version__",
+    "scrape_url",
+    "scrape_url_async",
+    "scrape_markdown",
+    "scrape_markdown_async",
+    "map_url",
+]
+
+_LAZY: dict[str, str] = {
+    "scrape_url":            "emb.scrape",
+    "scrape_url_async":      "emb.scrape",
+    "scrape_markdown":       "emb.scrape",
+    "scrape_markdown_async": "emb.scrape",
+    "map_url":               "emb.map",
+}
+
+
+def __getattr__(name: str):
+    module_path = _LAZY.get(name)
+    if module_path is None:
+        raise AttributeError(f"module 'emb' has no attribute {name!r}")
+    import importlib
+    module = importlib.import_module(module_path)
+    return getattr(module, name)

emb/_browser.py

@@ -0,0 +1,157 @@
+"""Auto-download and cache Lightpanda browser binary.
+
+The browser is downloaded on first use — no manual install required.
+Cached at ~/.cache/ember/lightpanda for subsequent use.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import platform
+import secrets
+import stat
+import subprocess
+import sys
+from pathlib import Path
+
+import httpx
+
+CACHE_DIR = Path.home() / ".cache" / "ember"
+BINARY_NAME = "lightpanda"
+BINARY_PATH = CACHE_DIR / BINARY_NAME
+
+LIGHTPANDA_VERSION = "0.3.3"
+
+# SHA-256 digests for each platform binary at LIGHTPANDA_VERSION.
+# Update both this dict and LIGHTPANDA_VERSION together on every release bump.
+_BINARY_HASHES: dict[tuple[str, str], str] = {
+    ("Linux",  "x86_64"):  "b6ab613846f5291cc6bafd7f44ffb9718df51bf00eb83954e1fc5d7f52c7b886",
+    ("Linux",  "aarch64"): "db35c06ee074a79c2e039965c404e578748c1d22cb296e853461970ea0c2945f",
+    ("Darwin", "x86_64"):  "631cec32766d2f98f1005e3af6af74794fb55cc28d193d97176cfa21f1d26d0c",
+    ("Darwin", "aarch64"): "1ef236a72e63975cf8acc7430e52dd31af5fff27a0b62ba81876eb8dde3e18e2",
+}
+
+_DOWNLOAD_URLS = {
+    ("Linux", "x86_64"): (
+        f"https://github.com/lightpanda-io/browser/releases/download/{LIGHTPANDA_VERSION}/"
+        "lightpanda-x86_64-linux"
+    ),
+    ("Linux", "aarch64"): (
+        f"https://github.com/lightpanda-io/browser/releases/download/{LIGHTPANDA_VERSION}/"
+        "lightpanda-aarch64-linux"
+    ),
+    ("Darwin", "x86_64"): (
+        f"https://github.com/lightpanda-io/browser/releases/download/{LIGHTPANDA_VERSION}/"
+        "lightpanda-x86_64-macos"
+    ),
+    ("Darwin", "aarch64"): (
+        f"https://github.com/lightpanda-io/browser/releases/download/{LIGHTPANDA_VERSION}/"
+        "lightpanda-aarch64-macos"
+    ),
+}
+
+
+def is_available() -> bool:
+    if BINARY_PATH.exists():
+        return True
+    try:
+        subprocess.run(["lightpanda", "version"], capture_output=True, timeout=5)
+        return True
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        return False
+
+
+def _platform_url() -> str | None:
+    system = platform.system()
+    machine = platform.machine()
+    return _DOWNLOAD_URLS.get((system, machine))
+
+
+def ensure() -> str:
+    # Check env override
+    env_path = os.environ.get("EMBER_LIGHTPANDA_PATH")
+    if env_path:
+        p = Path(env_path)
+        if p.exists() and p.is_file():
+            return str(p)
+        # Bare name (no path separator) — allow PATH resolution
+        if os.sep not in env_path and "/" not in env_path:
+            try:
+                r = subprocess.run([env_path, "version"], capture_output=True, text=True, timeout=5)
+                if r.returncode == 0:
+                    return env_path
+            except (FileNotFoundError, subprocess.TimeoutExpired):
+                pass
+        raise RuntimeError(
+            f"EMBER_LIGHTPANDA_PATH={env_path!r} — binary not found or not executable. "
+            f"Check the path and try again."
+        )
+
+    # Check PATH
+    try:
+        r = subprocess.run(["lightpanda", "version"], capture_output=True, text=True, timeout=5)
+        if r.returncode == 0:
+            return "lightpanda"
+    except (FileNotFoundError, subprocess.TimeoutExpired):
+        pass
+
+    # Check cached binary
+    if BINARY_PATH.exists():
+        try:
+            r = subprocess.run([str(BINARY_PATH), "version"], capture_output=True, text=True, timeout=5)
+            if r.returncode == 0:
+                return str(BINARY_PATH)
+        except (FileNotFoundError, subprocess.TimeoutExpired):
+            pass
+        BINARY_PATH.unlink(missing_ok=True)
+
+    # Download
+    url = _platform_url()
+    if not url:
+        if platform.system() == "Windows":
+            raise RuntimeError(
+                f"Lightpanda does not support Windows natively ({platform.machine()}).\n"
+                "Agents using ember should run on Linux. For local development, use WSL2:\n"
+                "  wsl --install\n"
+                "  then run ember inside WSL.\n"
+                "Browser-free features (scrape, search, crawl, map) work on Windows as-is."
+            )
+        raise RuntimeError(
+            f"Lightpanda not available for {platform.system()} {platform.machine()}. "
+            f"See: https://lightpanda.io/docs/open-source/installation"
+        )
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    tmp = BINARY_PATH.with_suffix(".tmp")
+    try:
+        print("  Downloading Lightpanda...", file=sys.stderr)
+        with httpx.stream("GET", url, follow_redirects=True, timeout=120) as resp:
+            resp.raise_for_status()
+            with open(tmp, "wb") as f:
+                for chunk in resp.iter_bytes(65536):
+                    f.write(chunk)
+
+        expected = _BINARY_HASHES.get((platform.system(), platform.machine()))
+        if expected is None:
+            tmp.unlink(missing_ok=True)
+            raise RuntimeError(
+                f"No SHA-256 hash registered for {platform.system()} {platform.machine()}. "
+                "Add the expected hash to _BINARY_HASHES before enabling downloads for this platform."
+            )
+        actual = hashlib.sha256(tmp.read_bytes()).hexdigest()
+        if not secrets.compare_digest(actual, expected):
+            tmp.unlink(missing_ok=True)
+            raise RuntimeError(
+                f"Lightpanda binary digest mismatch — download may be corrupted or tampered.\n"
+                f"  expected: {expected}\n"
+                f"  got:      {actual}"
+            )
+
+        tmp.chmod(tmp.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+        tmp.rename(BINARY_PATH)
+        print(f"  ✓ Lightpanda cached at {BINARY_PATH}", file=sys.stderr)
+    except Exception as e:
+        tmp.unlink(missing_ok=True)
+        raise RuntimeError(f"Failed to download Lightpanda: {e}") from e
+
+    return str(BINARY_PATH)

emb/_url_validator.py

@@ -0,0 +1,59 @@
+"""URL validation to block SSRF and unsafe schemes."""
+
+from __future__ import annotations
+
+import ipaddress
+import socket
+from urllib.parse import urlparse
+
+_BLOCKED_NETWORKS = [
+    ipaddress.ip_network("10.0.0.0/8"),
+    ipaddress.ip_network("172.16.0.0/12"),
+    ipaddress.ip_network("192.168.0.0/16"),
+    ipaddress.ip_network("127.0.0.0/8"),
+    ipaddress.ip_network("169.254.0.0/16"),
+    ipaddress.ip_network("0.0.0.0/8"),
+    ipaddress.ip_network("::1/128"),
+    ipaddress.ip_network("fc00::/7"),
+    ipaddress.ip_network("fe80::/10"),
+]
+
+
+def _is_blocked(addr_str: str) -> bool:
+    try:
+        addr = ipaddress.ip_address(addr_str)
+    except ValueError:
+        return False
+    # IPv6-mapped IPv4 (e.g. ::ffff:10.0.0.1) — unwrap to check against IPv4 ranges
+    if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
+        addr = addr.ipv4_mapped
+    for network in _BLOCKED_NETWORKS:
+        try:
+            if addr in network:
+                return True
+        except TypeError:
+            pass  # address family mismatch — not blocked by this entry
+    return False
+
+
+# DNS rebinding can bypass this check — pair with network-level egress controls
+# in high-security deployments.
+def validate_url(url: str) -> None:
+    parsed = urlparse(url)
+
+    if parsed.scheme not in ("http", "https"):
+        raise ValueError(f"URL scheme must be http or https, got {parsed.scheme!r}")
+
+    hostname = parsed.hostname
+    if not hostname:
+        raise ValueError("URL has no hostname")
+
+    try:
+        infos = socket.getaddrinfo(hostname, None)
+    except socket.gaierror as exc:
+        raise ValueError(f"Cannot resolve hostname {hostname!r}: {exc}") from exc
+
+    for info in infos:
+        addr_str = info[4][0]
+        if _is_blocked(addr_str):
+            raise ValueError(f"URL resolves to a blocked address ({addr_str})")

emb/agent.py

@@ -0,0 +1,70 @@
+"""LLM-powered structured extraction from scraped pages."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any
+
+import httpx
+
+from emb.scrape import scrape_url
+
+_log = logging.getLogger(__name__)
+
+_DEFAULT_API_KEY = os.environ.get("EMBER_LLM_API_KEY", "")
+_DEFAULT_BASE_URL = os.environ.get("EMBER_LLM_BASE_URL", "https://api.openai.com/v1")
+_DEFAULT_MODEL = os.environ.get("EMBER_LLM_MODEL", "gpt-4o-mini")
+_MAX_CONTENT_CHARS = 15_000
+
+
+# Requires EMBER_LLM_API_KEY. Falls back to returning raw markdown when no key is set.
+def extract(
+    url: str,
+    *,
+    prompt: str = "",
+    model: str = _DEFAULT_MODEL,
+    api_key: str = _DEFAULT_API_KEY,
+    base_url: str = _DEFAULT_BASE_URL,
+    timeout: int = 60,
+    use_browser: bool | None = None,
+) -> dict[str, Any]:
+    scraped = scrape_url(url, use_browser=use_browser, timeout=timeout)
+    if not scraped.success:
+        return {"error": scraped.error or "Failed to scrape URL"}
+
+    if not api_key:
+        return {"markdown": scraped.markdown, "title": scraped.title}
+
+    user_prompt = f"Page: {url}\nTitle: {scraped.title}\n\n{scraped.markdown[:_MAX_CONTENT_CHARS]}\n\n"
+    if prompt:
+        user_prompt += f"Task: {prompt}"
+    else:
+        user_prompt += "Extract the main structured information."
+
+    try:
+        resp = httpx.post(
+            f"{base_url.rstrip('/')}/chat/completions",
+            headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+            json={
+                "model": model,
+                "messages": [
+                    {"role": "system", "content": "Extract the requested information from the page content."},
+                    {"role": "user", "content": user_prompt},
+                ],
+                "temperature": 0.1,
+                "max_tokens": 4096,
+            },
+            timeout=timeout,
+        )
+        resp.raise_for_status()
+        data = resp.json()
+        content = data["choices"][0]["message"]["content"]
+        try:
+            return json.loads(content)
+        except (json.JSONDecodeError, TypeError):
+            return {"content": content, "sources": [url]}
+    except Exception as e:
+        _log.debug("LLM request error: %s", e)
+        return {"error": "LLM request failed"}

emb/api.py

@@ -0,0 +1,193 @@
+"""FastAPI server. Exposes every ember feature as an HTTP endpoint."""
+
+from __future__ import annotations
+
+import os
+import secrets
+import sys
+from collections.abc import Awaitable, Callable
+from typing import Any, Literal
+
+from fastapi import FastAPI, HTTPException, Request, Response
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, Field
+
+from emb import __version__
+from emb._url_validator import validate_url
+from emb.agent import extract as agent_extract
+from emb.crawl import crawl as do_crawl
+from emb.interact import interact as do_interact
+from emb.map import map_url
+from emb.scrape import scrape_url
+from emb.search import search
+
+app = FastAPI(title="ember", version=__version__)
+
+_API_KEY = os.environ.get("EMBER_API_KEY", "")
+
+if not _API_KEY:
+    print(
+        "WARNING: EMBER_API_KEY is not set. The API server is open to any caller. "
+        "Set EMBER_API_KEY to require authentication.",
+        file=sys.stderr,
+    )
+
+
+@app.middleware("http")
+async def _auth_middleware(
+    request: Request,
+    call_next: Callable[[Request], Awaitable[Response]],
+) -> Response:
+    if request.url.path in ("/", "/health"):
+        return await call_next(request)
+    if _API_KEY:
+        key = request.headers.get("X-API-Key", "")
+        if not secrets.compare_digest(key, _API_KEY):
+            return JSONResponse({"detail": "Unauthorized"}, status_code=401)
+    return await call_next(request)
+
+
+def _safe_url(url: str) -> str:
+    try:
+        validate_url(url)
+    except ValueError as exc:
+        raise HTTPException(status_code=400, detail=str(exc))
+    return url
+
+
+class ScrapeRequest(BaseModel):
+    url: str = Field(..., max_length=2048, description="URL to scrape")
+    use_browser: bool | None = Field(None, description="Force browser rendering")
+    timeout: int = Field(30, ge=1, le=120)
+
+
+class SearchRequest(BaseModel):
+    query: str = Field(..., max_length=500, description="Search query")
+    limit: int = Field(5, ge=1, le=50)
+
+
+class CrawlRequest(BaseModel):
+    url: str = Field(..., max_length=2048, description="URL to start from")
+    max_pages: int = Field(50, ge=1, le=500)
+    max_depth: int = Field(3, ge=1, le=10)
+    timeout: int = Field(30, ge=1, le=300)
+    delay: float = Field(0.0, ge=0.0, le=10.0, description="Seconds between requests")
+
+
+class MapRequest(BaseModel):
+    url: str = Field(..., max_length=2048, description="Website URL")
+    max_links: int = Field(500, ge=1, le=5000)
+
+
+class InteractRequest(BaseModel):
+    url: str = Field(..., max_length=2048, description="URL to open")
+    prompt: str = Field("", description="Natural language action")
+    provider: Literal[
+        "openai", "anthropic", "gemini", "mistral",
+        "huggingface", "vercel", "ollama", "llama_cpp"
+    ] = Field("openai", description="LLM provider")
+    model: str = Field("", max_length=128, description="Model name override")
+    timeout: int = Field(60, ge=1, le=300)
+
+
+class ExtractRequest(BaseModel):
+    url: str = Field(..., max_length=2048, description="URL to extract from")
+    prompt: str = Field("", description="What to extract")
+    model: str = Field("gpt-4o-mini", max_length=128, description="LLM model")
+
+
+@app.get("/")
+def root() -> dict[str, Any]:
+    return {
+        "name": "ember",
+        "version": __version__,
+        "endpoints": {
+            "POST /scrape": "Extract markdown from a URL",
+            "POST /crawl": "Crawl a website",
+            "POST /search": "Search the web",
+            "POST /map": "Discover URLs on a site",
+            "POST /interact": "Browser interaction",
+            "POST /extract": "LLM-powered structured extraction",
+            "POST /agent": "Alias for /extract",
+            "GET /health": "Health check",
+        },
+    }
+
+
+@app.post("/scrape")
+def api_scrape(req: ScrapeRequest) -> dict[str, Any]:
+    _safe_url(req.url)
+    result = scrape_url(req.url, use_browser=req.use_browser, timeout=req.timeout)
+    if not result.success:
+        raise HTTPException(status_code=502, detail=result.error)
+    return {"url": result.url, "title": result.title, "markdown": result.markdown}
+
+
+@app.post("/crawl")
+def api_crawl(req: CrawlRequest) -> dict[str, Any]:
+    _safe_url(req.url)
+    result = do_crawl(
+        req.url,
+        max_pages=req.max_pages,
+        max_depth=req.max_depth,
+        timeout=req.timeout,
+        delay=req.delay,
+    )
+    if not result.success:
+        raise HTTPException(status_code=502, detail=result.error)
+    return {
+        "url": result.url,
+        "total": result.total,
+        "pages": [{"url": p.url, "title": p.title, "markdown": p.markdown, "depth": p.depth}
+                  for p in result.pages],
+    }
+
+
+@app.post("/search")
+def api_search(req: SearchRequest) -> dict[str, Any]:
+    try:
+        results = search(req.query, limit=req.limit)
+    except RuntimeError as e:
+        raise HTTPException(status_code=502, detail=str(e))
+    return {
+        "query": req.query,
+        "results": [{"url": r.url, "title": r.title, "description": r.description} for r in results],
+    }
+
+
+@app.post("/map")
+def api_map(req: MapRequest) -> dict[str, Any]:
+    _safe_url(req.url)
+    result = map_url(req.url, max_links=req.max_links)
+    if result.error:
+        raise HTTPException(status_code=502, detail=result.error)
+    return {"url": result.url, "total": result.total, "links": result.links}
+
+
+@app.post("/interact")
+def api_interact(req: InteractRequest) -> dict[str, Any]:
+    _safe_url(req.url)
+    result = do_interact(req.url, prompt=req.prompt, provider=req.provider, model=req.model, timeout=req.timeout)
+    if not result.success:
+        raise HTTPException(status_code=502, detail=result.error)
+    return {"url": result.url, "content": result.content}
+
+
+@app.post("/extract")
+@app.post("/agent")
+def api_extract(req: ExtractRequest) -> dict[str, Any]:
+    _safe_url(req.url)
+    result = agent_extract(req.url, prompt=req.prompt, model=req.model)
+    if "error" in result:
+        raise HTTPException(status_code=502, detail=result["error"])
+    return result
+
+
+@app.get("/health")
+def health() -> dict[str, Any]:
+    return {"status": "ok", "version": __version__}
+
+
+def start_server(host: str = "127.0.0.1", port: int = 51251) -> None:
+    import uvicorn
+    uvicorn.run(app, host=host, port=port)