crw 0.2.1__tar.gz

crw-0.2.1/.gitignore

@@ -0,0 +1,7 @@
+/target
+.claude
+PROGRESS.md
+.gstack/
+python/__pycache__/
+python/.venv/
+python/uv.lock

crw-0.2.1/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: crw
+Version: 0.2.1
+Summary: Python SDK for CRW web scraper — scrape, crawl, and map any website from Python
+Project-URL: Homepage, https://github.com/us/crw
+Project-URL: Documentation, https://us.github.io/crw
+Project-URL: Repository, https://github.com/us/crw
+License-Expression: MIT
+Keywords: ai-agent,crawler,firecrawl,mcp,scraper,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.9
+Requires-Dist: platformdirs>=3.0
+Description-Content-Type: text/markdown
+
+# crw
+
+Python SDK for [CRW](https://github.com/us/crw) — the open-source web scraper built for AI agents.
+
+## Install
+
+```bash
+pip install crw
+```
+
+## Usage
+
+```python
+from crw import CrwClient
+
+# Zero-config (downloads crw-mcp binary automatically):
+client = CrwClient()
+result = client.scrape("https://example.com")
+print(result["markdown"])
+
+# Or connect to a remote server:
+client = CrwClient(api_url="https://fastcrw.com/api", api_key="fc-...")
+```
+
+## MCP Server
+
+After installing, you can also use `crw-mcp` as an MCP server:
+
+```bash
+crw-mcp  # starts stdio MCP server
+```

crw-0.2.1/README.md

@@ -0,0 +1,31 @@
+# crw
+
+Python SDK for [CRW](https://github.com/us/crw) — the open-source web scraper built for AI agents.
+
+## Install
+
+```bash
+pip install crw
+```
+
+## Usage
+
+```python
+from crw import CrwClient
+
+# Zero-config (downloads crw-mcp binary automatically):
+client = CrwClient()
+result = client.scrape("https://example.com")
+print(result["markdown"])
+
+# Or connect to a remote server:
+client = CrwClient(api_url="https://fastcrw.com/api", api_key="fc-...")
+```
+
+## MCP Server
+
+After installing, you can also use `crw-mcp` as an MCP server:
+
+```bash
+crw-mcp  # starts stdio MCP server
+```

crw-0.2.1/pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "crw"
+version = "0.2.1"
+description = "Python SDK for CRW web scraper — scrape, crawl, and map any website from Python"
+readme = "README.md"
+requires-python = ">=3.9"
+license = "MIT"
+keywords = ["web-scraping", "mcp", "ai-agent", "crawler", "firecrawl", "scraper"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = ["platformdirs>=3.0"]
+
+[project.urls]
+Homepage = "https://github.com/us/crw"
+Documentation = "https://us.github.io/crw"
+Repository = "https://github.com/us/crw"
+
+[project.scripts]
+crw-mcp = "crw.__main__:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/crw"]

crw-0.2.1/src/crw/__init__.py

@@ -0,0 +1,6 @@
+"""CRW Python SDK — scrape, crawl, and map any website."""
+
+from crw.client import CrwClient
+from crw.exceptions import CrwError, CrwBinaryNotFoundError, CrwTimeoutError
+
+__all__ = ["CrwClient", "CrwError", "CrwBinaryNotFoundError", "CrwTimeoutError"]

crw-0.2.1/src/crw/__main__.py

@@ -0,0 +1,15 @@
+"""CLI entry point — exec crw-mcp binary."""
+
+import os
+import sys
+
+from crw._binary import ensure_binary
+
+
+def main() -> None:
+    binary = ensure_binary()
+    os.execvp(str(binary), [str(binary)] + sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()

crw-0.2.1/src/crw/_binary.py

@@ -0,0 +1,115 @@
+"""Download, cache, and locate the crw-mcp binary."""
+
+from __future__ import annotations
+
+import os
+import stat
+import tarfile
+import zipfile
+from io import BytesIO
+from pathlib import Path
+from urllib.request import Request, urlopen
+
+from platformdirs import user_cache_dir
+
+from crw._platform import BINARY_NAME, get_asset_name
+from crw.exceptions import CrwBinaryNotFoundError
+
+try:
+    from importlib.metadata import version as _pkg_version
+    BINARY_VERSION = _pkg_version("crw")
+except Exception:
+    BINARY_VERSION = "0.2.1"  # fallback for development
+GITHUB_REPO = "us/crw"
+DOWNLOAD_URL = f"https://github.com/{GITHUB_REPO}/releases/download/v{BINARY_VERSION}"
+
+
+def _cache_dir() -> Path:
+    return Path(user_cache_dir("crw")) / f"v{BINARY_VERSION}"
+
+
+def _cached_binary() -> Path | None:
+    """Return path to cached binary if it exists and is executable."""
+    path = _cache_dir() / BINARY_NAME
+    if path.is_file() and os.access(path, os.X_OK):
+        return path
+    return None
+
+
+def _download_binary() -> Path:
+    """Download the binary from GitHub Releases and cache it."""
+    asset = get_asset_name()
+    if asset is None:
+        raise CrwBinaryNotFoundError(
+            f"No prebuilt binary for this platform. "
+            f"Install from source: cargo install crw-mcp"
+        )
+
+    url = f"{DOWNLOAD_URL}/{asset}"
+    cache = _cache_dir()
+    cache.mkdir(parents=True, exist_ok=True)
+
+    req = Request(url, headers={"User-Agent": f"crw-python/{BINARY_VERSION}"})
+    with urlopen(req, timeout=120) as resp:
+        data = resp.read()
+
+    # Extract binary from archive
+    if asset.endswith(".tar.gz"):
+        with tarfile.open(fileobj=BytesIO(data), mode="r:gz") as tar:
+            for member in tar.getmembers():
+                if member.name.endswith("crw-mcp"):
+                    member.name = BINARY_NAME
+                    tar.extract(member, path=cache)
+                    break
+            else:
+                raise CrwBinaryNotFoundError(f"crw-mcp not found in {asset}")
+    elif asset.endswith(".zip"):
+        with zipfile.ZipFile(BytesIO(data)) as zf:
+            for name in zf.namelist():
+                if name.endswith("crw-mcp.exe") or name.endswith("crw-mcp"):
+                    target = cache / BINARY_NAME
+                    target.write_bytes(zf.read(name))
+                    break
+            else:
+                raise CrwBinaryNotFoundError(f"crw-mcp not found in {asset}")
+
+    binary = cache / BINARY_NAME
+    binary.chmod(binary.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+    return binary
+
+
+def _is_native_binary(path: Path) -> bool:
+    """Check if a file is a native binary (not a Python/shell script)."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(4)
+        # ELF, Mach-O, or PE magic bytes
+        return header[:4] in (b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe", b"MZ\x90\x00", b"MZ\x00\x00")
+    except OSError:
+        return False
+
+
+def ensure_binary() -> Path:
+    """Return path to crw-mcp binary, downloading if necessary."""
+    # 1. Check CRW_BINARY env override
+    env_path = os.environ.get("CRW_BINARY")
+    if env_path:
+        p = Path(env_path)
+        if p.is_file():
+            return p
+        raise CrwBinaryNotFoundError(f"CRW_BINARY={env_path} does not exist")
+
+    # 2. Check if crw-mcp native binary is on PATH (e.g. cargo install)
+    # Search all PATH entries, skipping our Python wrapper
+    for directory in os.environ.get("PATH", "").split(os.pathsep):
+        candidate = Path(directory) / BINARY_NAME
+        if candidate.is_file() and _is_native_binary(candidate):
+            return candidate
+
+    # 3. Check cache
+    cached = _cached_binary()
+    if cached:
+        return cached
+
+    # 4. Download
+    return _download_binary()

crw-0.2.1/src/crw/_platform.py

@@ -0,0 +1,22 @@
+"""Platform detection and GitHub Release asset mapping."""
+
+import platform
+import sys
+
+# Maps (system, machine) to GitHub Release asset filename
+PLATFORM_MAP: dict[tuple[str, str], str] = {
+    ("Darwin", "arm64"): "crw-mcp-darwin-arm64.tar.gz",
+    ("Darwin", "x86_64"): "crw-mcp-darwin-x64.tar.gz",
+    ("Linux", "x86_64"): "crw-mcp-linux-x64.tar.gz",
+    ("Linux", "aarch64"): "crw-mcp-linux-arm64.tar.gz",
+    ("Windows", "AMD64"): "crw-mcp-win32-x64.zip",
+    ("Windows", "ARM64"): "crw-mcp-win32-arm64.zip",
+}
+
+BINARY_NAME = "crw-mcp.exe" if sys.platform == "win32" else "crw-mcp"
+
+
+def get_asset_name() -> str | None:
+    """Return the GitHub Release asset filename for the current platform."""
+    key = (platform.system(), platform.machine())
+    return PLATFORM_MAP.get(key)

crw-0.2.1/src/crw/client.py

@@ -0,0 +1,247 @@
+"""CRW client — subprocess (embedded) or HTTP mode."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import time
+from typing import Any
+
+from crw._binary import ensure_binary
+from crw.exceptions import CrwApiError, CrwError, CrwTimeoutError
+
+_REQUEST_ID = 0
+
+
+def _next_id() -> int:
+    global _REQUEST_ID
+    _REQUEST_ID += 1
+    return _REQUEST_ID
+
+
+class CrwClient:
+    """CRW web scraper client.
+
+    Args:
+        api_url: CRW server URL for HTTP mode. If None, uses subprocess mode
+                 (spawns crw-mcp binary, no server required).
+        api_key: API key for authentication (HTTP mode or fastcrw.com).
+
+    Examples:
+        # Subprocess mode (zero config, no server):
+        client = CrwClient()
+        result = client.scrape("https://example.com")
+
+        # HTTP mode (remote server):
+        client = CrwClient(api_url="https://fastcrw.com/api", api_key="fc-...")
+        result = client.scrape("https://example.com")
+    """
+
+    def __init__(self, api_url: str | None = None, api_key: str | None = None):
+        self._api_url = api_url
+        self._api_key = api_key
+        self._process: subprocess.Popen | None = None
+
+    def scrape(
+        self,
+        url: str,
+        formats: list[str] | None = None,
+        only_main_content: bool = True,
+        include_tags: list[str] | None = None,
+        exclude_tags: list[str] | None = None,
+        **kwargs: Any,
+    ) -> dict:
+        """Scrape a single URL and return its content.
+
+        Returns:
+            Dict with keys like 'markdown', 'html', 'metadata', etc.
+        """
+        args: dict[str, Any] = {"url": url, "onlyMainContent": only_main_content}
+        if formats:
+            args["formats"] = formats
+        if include_tags:
+            args["includeTags"] = include_tags
+        if exclude_tags:
+            args["excludeTags"] = exclude_tags
+        args.update(kwargs)
+
+        if self._api_url:
+            return self._http_post("/v1/scrape", args)
+        return self._tool_call("crw_scrape", args)
+
+    def crawl(
+        self,
+        url: str,
+        max_depth: int = 2,
+        max_pages: int = 10,
+        poll_interval: float = 2.0,
+        timeout: float = 300.0,
+        **kwargs: Any,
+    ) -> list[dict]:
+        """Crawl a website and return all page results.
+
+        Starts an async crawl, polls for completion, and returns all results.
+        """
+        args: dict[str, Any] = {"url": url, "maxDepth": max_depth, "maxPages": max_pages}
+        args.update(kwargs)
+
+        if self._api_url:
+            return self._http_crawl(args, poll_interval, timeout)
+
+        # Subprocess mode: start crawl, poll status
+        result = self._tool_call("crw_crawl", args)
+        job_id = result.get("id")
+        if not job_id:
+            raise CrwError(f"Crawl did not return job ID: {result}")
+
+        return self._poll_crawl(job_id, poll_interval, timeout)
+
+    def map(
+        self,
+        url: str,
+        max_depth: int = 2,
+        use_sitemap: bool = True,
+        **kwargs: Any,
+    ) -> list[str]:
+        """Discover URLs on a website.
+
+        Returns:
+            List of discovered URLs.
+        """
+        args: dict[str, Any] = {"url": url, "maxDepth": max_depth, "useSitemap": use_sitemap}
+        args.update(kwargs)
+
+        if self._api_url:
+            data = self._http_post("/v1/map", args)
+            return data.get("links", [])
+
+        result = self._tool_call("crw_map", args)
+        return result.get("links", [])
+
+    def close(self) -> None:
+        """Shut down the subprocess if running."""
+        if self._process and self._process.poll() is None:
+            self._process.stdin.close()
+            try:
+                self._process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                self._process.terminate()
+                self._process.wait(timeout=5)
+            self._process = None
+
+    def __enter__(self) -> CrwClient:
+        return self
+
+    def __exit__(self, *_: Any) -> None:
+        self.close()
+
+    def __del__(self) -> None:
+        self.close()
+
+    # --- Subprocess (embedded) mode ---
+
+    def _ensure_process(self) -> subprocess.Popen:
+        if self._process is None or self._process.poll() is not None:
+            binary = ensure_binary()
+            self._process = subprocess.Popen(
+                [str(binary)],
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.DEVNULL,
+                text=True,
+            )
+        return self._process
+
+    def _jsonrpc(self, method: str, params: dict | None = None) -> Any:
+        proc = self._ensure_process()
+        req = {
+            "jsonrpc": "2.0",
+            "id": _next_id(),
+            "method": method,
+            "params": params or {},
+        }
+        proc.stdin.write(json.dumps(req) + "\n")
+        proc.stdin.flush()
+
+        line = proc.stdout.readline()
+        if not line:
+            raise CrwError("crw-mcp process closed unexpectedly")
+
+        resp = json.loads(line)
+        if "error" in resp:
+            raise CrwApiError(resp["error"].get("message", str(resp["error"])))
+        return resp.get("result")
+
+    def _tool_call(self, tool_name: str, arguments: dict) -> dict:
+        result = self._jsonrpc("tools/call", {"name": tool_name, "arguments": arguments})
+        if not result or not result.get("content"):
+            raise CrwError(f"Empty response from {tool_name}")
+
+        content = result["content"][0]
+        if result.get("isError"):
+            raise CrwApiError(content.get("text", "Unknown error"))
+
+        return json.loads(content["text"])
+
+    def _poll_crawl(self, job_id: str, poll_interval: float, timeout: float) -> list[dict]:
+        start = time.monotonic()
+        while True:
+            if time.monotonic() - start > timeout:
+                raise CrwTimeoutError(f"Crawl {job_id} timed out after {timeout}s")
+
+            result = self._tool_call("crw_check_crawl_status", {"id": job_id})
+            status = result.get("status")
+
+            if status == "completed":
+                return result.get("data", [])
+            if status == "failed":
+                raise CrwError(f"Crawl failed: {result.get('error', 'unknown')}")
+
+            time.sleep(poll_interval)
+
+    # --- HTTP mode ---
+
+    def _http_request(self, method: str, path: str, body: dict | None = None) -> dict:
+        import urllib.request
+
+        url = f"{self._api_url.rstrip('/')}{path}"
+        headers = {"Content-Type": "application/json"}
+        if self._api_key:
+            headers["Authorization"] = f"Bearer {self._api_key}"
+
+        data = json.dumps(body).encode() if body else None
+        req = urllib.request.Request(url, data=data, headers=headers, method=method)
+
+        with urllib.request.urlopen(req, timeout=120) as resp:
+            result = json.loads(resp.read())
+
+        if not result.get("success"):
+            raise CrwApiError(result.get("error", "API error"))
+        return result.get("data", result)
+
+    def _http_post(self, path: str, body: dict) -> dict:
+        return self._http_request("POST", path, body)
+
+    def _http_get(self, path: str) -> dict:
+        return self._http_request("GET", path)
+
+    def _http_crawl(self, args: dict, poll_interval: float, timeout: float) -> list[dict]:
+        result = self._http_post("/v1/crawl", args)
+        job_id = result.get("id")
+        if not job_id:
+            raise CrwError(f"Crawl did not return job ID: {result}")
+
+        start = time.monotonic()
+        while True:
+            if time.monotonic() - start > timeout:
+                raise CrwTimeoutError(f"Crawl {job_id} timed out after {timeout}s")
+
+            status_result = self._http_get(f"/v1/crawl/{job_id}")
+            status = status_result.get("status")
+
+            if status == "completed":
+                return status_result.get("data", [])
+            if status == "failed":
+                raise CrwError(f"Crawl failed: {status_result.get('error', 'unknown')}")
+
+            time.sleep(poll_interval)

crw-0.2.1/src/crw/exceptions.py

@@ -0,0 +1,21 @@
+"""CRW SDK exceptions."""
+
+
+class CrwError(Exception):
+    """Base exception for CRW SDK."""
+
+
+class CrwBinaryNotFoundError(CrwError):
+    """Binary could not be found or downloaded."""
+
+
+class CrwTimeoutError(CrwError):
+    """Operation timed out."""
+
+
+class CrwApiError(CrwError):
+    """API returned an error."""
+
+    def __init__(self, message: str, status_code: int | None = None):
+        super().__init__(message)
+        self.status_code = status_code