webclaw 0.1.0__py3-none-any.whl

webclaw/__init__.py

@@ -0,0 +1,72 @@
+"""Webclaw Python SDK -- web extraction API client."""
+
+from .async_client import AsyncCrawlJobHandle, AsyncWebclaw
+from .client import CrawlJobHandle, Webclaw
+from .errors import (
+    AuthenticationError,
+    NotFoundError,
+    RateLimitError,
+    TimeoutError,
+    WebclawError,
+)
+from .types import (
+    AgentScrapeResponse,
+    BatchResponse,
+    BatchResult,
+    BrandResponse,
+    CacheInfo,
+    CrawlJob,
+    CrawlPage,
+    CrawlStatus,
+    DiffResponse,
+    ExtractResponse,
+    MapResponse,
+    ResearchFinding,
+    ResearchSource,
+    ResearchStartResponse,
+    ResearchStatusResponse,
+    ScrapeResponse,
+    SearchResponse,
+    SearchResult,
+    SummarizeResponse,
+    WatchCheckResponse,
+    WatchEntry,
+    WatchListResponse,
+)
+
+__all__ = [
+    # clients
+    "Webclaw",
+    "AsyncWebclaw",
+    "CrawlJobHandle",
+    "AsyncCrawlJobHandle",
+    # types
+    "AgentScrapeResponse",
+    "BatchResponse",
+    "BatchResult",
+    "BrandResponse",
+    "CacheInfo",
+    "CrawlJob",
+    "CrawlPage",
+    "CrawlStatus",
+    "DiffResponse",
+    "ExtractResponse",
+    "MapResponse",
+    "ResearchFinding",
+    "ResearchSource",
+    "ResearchStartResponse",
+    "ResearchStatusResponse",
+    "ScrapeResponse",
+    "SearchResponse",
+    "SearchResult",
+    "SummarizeResponse",
+    "WatchCheckResponse",
+    "WatchEntry",
+    "WatchListResponse",
+    # errors
+    "WebclawError",
+    "AuthenticationError",
+    "RateLimitError",
+    "NotFoundError",
+    "TimeoutError",
+]

webclaw/_endpoints.py

@@ -0,0 +1,264 @@
+"""Shared request body builders and response parsers.
+
+Both sync and async clients delegate to these functions so endpoint
+logic lives in exactly one place.  Only the transport layer (sync
+httpx vs async httpx) differs between the two clients.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from .types import (
+    BatchResponse,
+    BatchResult,
+    BrandResponse,
+    CacheInfo,
+    CrawlJob,
+    CrawlPage,
+    CrawlStatus,
+    ExtractResponse,
+    MapResponse,
+    ResearchStatusResponse,
+    ScrapeResponse,
+    SummarizeResponse,
+    WatchCheckResponse,
+    WatchEntry,
+    WatchListResponse,
+)
+
+DEFAULT_BASE_URL = "https://api.webclaw.io"
+DEFAULT_TIMEOUT = 30.0
+
+# Terminal states shared by crawl and research polling.
+TERMINAL_STATES = frozenset({"completed", "failed"})
+
+
+# ---------------------------------------------------------------------------
+# Request body builders
+# ---------------------------------------------------------------------------
+
+def build_scrape_body(
+    url: str,
+    *,
+    formats: Sequence[str] | None = None,
+    include_selectors: list[str] | None = None,
+    exclude_selectors: list[str] | None = None,
+    only_main_content: bool = False,
+    no_cache: bool = False,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"url": url}
+    if formats is not None:
+        body["formats"] = list(formats)
+    if include_selectors:
+        body["include_selectors"] = include_selectors
+    if exclude_selectors:
+        body["exclude_selectors"] = exclude_selectors
+    if only_main_content:
+        body["only_main_content"] = True
+    if no_cache:
+        body["no_cache"] = True
+    return body
+
+
+def build_crawl_body(
+    url: str,
+    *,
+    max_depth: int = 2,
+    max_pages: int = 50,
+    use_sitemap: bool = False,
+) -> dict[str, Any]:
+    return {
+        "url": url,
+        "max_depth": max_depth,
+        "max_pages": max_pages,
+        "use_sitemap": use_sitemap,
+    }
+
+
+def build_batch_body(
+    urls: list[str],
+    *,
+    formats: Sequence[str] | None = None,
+    concurrency: int = 5,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"urls": urls, "concurrency": concurrency}
+    if formats is not None:
+        body["formats"] = list(formats)
+    return body
+
+
+def build_extract_body(
+    url: str,
+    *,
+    schema: dict[str, Any] | None = None,
+    prompt: str | None = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"url": url}
+    if schema is not None:
+        body["schema"] = schema
+    if prompt is not None:
+        body["prompt"] = prompt
+    return body
+
+
+def build_summarize_body(
+    url: str,
+    *,
+    max_sentences: int | None = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"url": url}
+    if max_sentences is not None:
+        body["max_sentences"] = max_sentences
+    return body
+
+
+def build_search_body(
+    query: str,
+    *,
+    num_results: int | None = None,
+    topic: str | None = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"query": query}
+    if num_results is not None:
+        body["num_results"] = num_results
+    if topic is not None:
+        body["topic"] = topic
+    return body
+
+
+def build_research_body(
+    query: str,
+    *,
+    deep: bool = False,
+    max_sources: int | None = None,
+    max_iterations: int | None = None,
+    topic: str | None = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"query": query, "deep": deep}
+    if max_sources is not None:
+        body["max_sources"] = max_sources
+    if max_iterations is not None:
+        body["max_iterations"] = max_iterations
+    if topic is not None:
+        body["topic"] = topic
+    return body
+
+
+def build_watch_create_body(
+    url: str,
+    *,
+    name: str | None = None,
+    interval_minutes: int = 1440,
+    webhook_url: str | None = None,
+) -> dict[str, Any]:
+    body: dict[str, Any] = {"url": url, "interval_minutes": interval_minutes}
+    if name is not None:
+        body["name"] = name
+    if webhook_url is not None:
+        body["webhook_url"] = webhook_url
+    return body
+
+
+# ---------------------------------------------------------------------------
+# Response parsers
+# ---------------------------------------------------------------------------
+
+def parse_scrape(data: dict[str, Any]) -> ScrapeResponse:
+    cache = None
+    if data.get("cache"):
+        cache = CacheInfo(status=data["cache"]["status"])
+    return ScrapeResponse(
+        url=data["url"],
+        metadata=data.get("metadata", {}),
+        markdown=data.get("markdown"),
+        text=data.get("text"),
+        llm=data.get("llm"),
+        json_data=data.get("json"),
+        cache=cache,
+        warning=data.get("warning"),
+    )
+
+
+def parse_crawl_job(data: dict[str, Any]) -> CrawlJob:
+    return CrawlJob(id=data["id"], status=data["status"])
+
+
+def parse_crawl_status(data: dict[str, Any]) -> CrawlStatus:
+    pages = [
+        CrawlPage(
+            url=p["url"],
+            markdown=p.get("markdown"),
+            metadata=p.get("metadata", {}),
+            error=p.get("error"),
+        )
+        for p in data.get("pages", [])
+    ]
+    return CrawlStatus(
+        id=data["id"],
+        status=data["status"],
+        pages=pages,
+        total=data.get("total", 0),
+        completed=data.get("completed", 0),
+        errors=data.get("errors", 0),
+    )
+
+
+def parse_map(data: dict[str, Any]) -> MapResponse:
+    return MapResponse(urls=data.get("urls", []), count=data.get("count", 0))
+
+
+def parse_batch(data: dict[str, Any]) -> BatchResponse:
+    results = [
+        BatchResult(
+            url=r["url"],
+            markdown=r.get("markdown"),
+            metadata=r.get("metadata", {}),
+            error=r.get("error"),
+        )
+        for r in data.get("results", [])
+    ]
+    return BatchResponse(results=results)
+
+
+def parse_extract(data: dict[str, Any]) -> ExtractResponse:
+    return ExtractResponse(data=data.get("data"))
+
+
+def parse_summarize(data: dict[str, Any]) -> SummarizeResponse:
+    return SummarizeResponse(summary=data.get("summary", ""))
+
+
+def parse_brand(data: dict[str, Any]) -> BrandResponse:
+    return BrandResponse(data=data)
+
+
+def parse_research(data: dict[str, Any]) -> ResearchStatusResponse:
+    return ResearchStatusResponse(
+        id=data.get("id", ""),
+        status=data.get("status", ""),
+        query=data.get("query", ""),
+        report=data.get("report", ""),
+        sources=data.get("sources", []),
+        findings=data.get("findings", []),
+        iterations=data.get("iterations", 0),
+        elapsed_ms=data.get("elapsed_ms", 0),
+    )
+
+
+def parse_watch_entry(data: dict[str, Any]) -> WatchEntry:
+    return WatchEntry.from_dict(data)
+
+
+def parse_watch_list(data: dict[str, Any]) -> WatchListResponse:
+    watches = [WatchEntry.from_dict(w) for w in data.get("watches", [])]
+    return WatchListResponse(watches=watches, total=data.get("total", len(watches)))
+
+
+def parse_watch_check(data: dict[str, Any]) -> WatchCheckResponse:
+    return WatchCheckResponse(
+        id=data.get("id", ""),
+        has_changed=data.get("has_changed", False),
+        diff=data.get("diff"),
+        checked_at=data.get("checked_at", ""),
+    )

webclaw/async_client.py

@@ -0,0 +1,210 @@
+"""Asynchronous Webclaw client."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any, Sequence
+
+import httpx
+
+from . import _endpoints as ep
+from .client import _raise_for_status
+from .errors import TimeoutError, WebclawError
+from .types import (
+    BatchResponse, BrandResponse, CrawlStatus, ExtractResponse, MapResponse,
+    ResearchStatusResponse, ScrapeResponse, SummarizeResponse,
+    WatchCheckResponse, WatchEntry, WatchListResponse,
+)
+
+
+class AsyncWebclaw:
+    """Async client for the Webclaw web extraction API."""
+
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        base_url: str = ep.DEFAULT_BASE_URL,
+        timeout: float = ep.DEFAULT_TIMEOUT,
+    ) -> None:
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self._client = httpx.AsyncClient(
+            base_url=self.base_url,
+            headers={"Authorization": f"Bearer {api_key}"},
+            timeout=timeout,
+        )
+
+    # -- lifecycle ------------------------------------------------------------
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    async def __aenter__(self) -> AsyncWebclaw:
+        return self
+
+    async def __aexit__(self, *_: Any) -> None:
+        await self.close()
+
+    # -- internal -------------------------------------------------------------
+
+    async def _request(self, method: str, path: str, **kwargs: Any) -> Any:
+        response = await self._client.request(method, path, **kwargs)
+        _raise_for_status(response)
+        return response.json()
+
+    # -- endpoints ------------------------------------------------------------
+
+    async def scrape(
+        self,
+        url: str,
+        *,
+        formats: Sequence[str] | None = None,
+        include_selectors: list[str] | None = None,
+        exclude_selectors: list[str] | None = None,
+        only_main_content: bool = False,
+        no_cache: bool = False,
+    ) -> ScrapeResponse:
+        """Scrape a URL and extract content."""
+        body = ep.build_scrape_body(
+            url, formats=formats, include_selectors=include_selectors,
+            exclude_selectors=exclude_selectors, only_main_content=only_main_content,
+            no_cache=no_cache,
+        )
+        return ep.parse_scrape(await self._request("POST", "/v1/scrape", json=body))
+
+    async def crawl(
+        self, url: str, *, max_depth: int = 2, max_pages: int = 50, use_sitemap: bool = False,
+    ) -> AsyncCrawlJobHandle:
+        """Start a crawl job and return a handle for polling."""
+        body = ep.build_crawl_body(url, max_depth=max_depth, max_pages=max_pages, use_sitemap=use_sitemap)
+        job = ep.parse_crawl_job(await self._request("POST", "/v1/crawl", json=body))
+        return AsyncCrawlJobHandle(client=self, job_id=job.id, status=job.status)
+
+    async def get_crawl_status(self, job_id: str) -> CrawlStatus:
+        """Get current status of a crawl job."""
+        return ep.parse_crawl_status(await self._request("GET", f"/v1/crawl/{job_id}"))
+
+    async def map(self, url: str) -> MapResponse:
+        """Discover URLs from a site's sitemap."""
+        return ep.parse_map(await self._request("POST", "/v1/map", json={"url": url}))
+
+    async def batch(
+        self, urls: list[str], *, formats: Sequence[str] | None = None, concurrency: int = 5,
+    ) -> BatchResponse:
+        """Scrape multiple URLs in parallel."""
+        body = ep.build_batch_body(urls, formats=formats, concurrency=concurrency)
+        return ep.parse_batch(await self._request("POST", "/v1/batch", json=body))
+
+    async def extract(self, url: str, *, schema: dict[str, Any] | None = None, prompt: str | None = None) -> ExtractResponse:
+        """LLM-powered structured data extraction."""
+        body = ep.build_extract_body(url, schema=schema, prompt=prompt)
+        return ep.parse_extract(await self._request("POST", "/v1/extract", json=body))
+
+    async def summarize(self, url: str, *, max_sentences: int | None = None) -> SummarizeResponse:
+        """Summarize page content."""
+        return ep.parse_summarize(await self._request("POST", "/v1/summarize", json=ep.build_summarize_body(url, max_sentences=max_sentences)))
+
+    async def brand(self, url: str) -> BrandResponse:
+        """Extract brand identity from a URL."""
+        return ep.parse_brand(await self._request("POST", "/v1/brand", json={"url": url}))
+
+    async def search(self, query: str, *, num_results: int | None = None, topic: str | None = None) -> dict:
+        """Run a web search query via the Serper-backed search endpoint."""
+        return await self._request("POST", "/v1/search", json=ep.build_search_body(query, num_results=num_results, topic=topic))
+
+    async def diff(self, url: str, **kwargs: Any) -> dict:
+        """Detect content changes at a URL since the last check."""
+        return await self._request("POST", "/v1/diff", json={"url": url, **kwargs})
+
+    async def agent_scrape(self, url: str, goal: str, **kwargs: Any) -> dict:
+        """AI-guided scraping that navigates a page to achieve a goal."""
+        return await self._request("POST", "/v1/agent-scrape", json={"url": url, "goal": goal, **kwargs})
+
+    async def research(
+        self, query: str, *, deep: bool = False,
+        max_sources: int | None = None, max_iterations: int | None = None, topic: str | None = None,
+    ) -> ResearchStatusResponse:
+        """Start a research job and await until it completes.
+
+        Normal queries time out after 600s, deep research after 1200s.
+        """
+        body = ep.build_research_body(query, deep=deep, max_sources=max_sources, max_iterations=max_iterations, topic=topic)
+        job_id = (await self._request("POST", "/v1/research", json=body))["id"]
+        return await _async_poll_until_done(
+            fetcher=lambda: self._request("GET", f"/v1/research/{job_id}"),
+            parser=ep.parse_research,
+            label=f"Research {job_id}",
+            interval=2.0,
+            timeout=1200.0 if deep else 600.0,
+        )
+
+    async def get_research_status(self, job_id: str) -> ResearchStatusResponse:
+        """Get status/results of a research job without polling."""
+        return ep.parse_research(await self._request("GET", f"/v1/research/{job_id}"))
+
+    # -- watch endpoints ------------------------------------------------------
+
+    async def watch_create(
+        self, url: str, *, name: str | None = None, interval_minutes: int = 1440, webhook_url: str | None = None,
+    ) -> WatchEntry:
+        """Create a new watch monitor for a URL."""
+        body = ep.build_watch_create_body(url, name=name, interval_minutes=interval_minutes, webhook_url=webhook_url)
+        return ep.parse_watch_entry(await self._request("POST", "/v1/watch", json=body))
+
+    async def watch_list(self, *, limit: int = 50, offset: int = 0) -> WatchListResponse:
+        """List all watch monitors."""
+        return ep.parse_watch_list(await self._request("GET", "/v1/watch", params={"limit": limit, "offset": offset}))
+
+    async def watch_get(self, watch_id: str) -> WatchEntry:
+        """Get a single watch monitor by ID."""
+        return ep.parse_watch_entry(await self._request("GET", f"/v1/watch/{watch_id}"))
+
+    async def watch_delete(self, watch_id: str) -> None:
+        """Delete a watch monitor."""
+        await self._request("DELETE", f"/v1/watch/{watch_id}")
+
+    async def watch_check(self, watch_id: str) -> WatchCheckResponse:
+        """Trigger an immediate check for a watch monitor."""
+        return ep.parse_watch_check(await self._request("POST", f"/v1/watch/{watch_id}/check"))
+
+
+class AsyncCrawlJobHandle:
+    """Wraps a running crawl job with async polling helpers."""
+
+    def __init__(self, client: AsyncWebclaw, job_id: str, status: str) -> None:
+        self.client = client
+        self.id = job_id
+        self.status = status
+
+    async def get_status(self) -> CrawlStatus:
+        return await self.client.get_crawl_status(self.id)
+
+    async def wait(self, *, interval: float = 2.0, timeout: float = 300.0) -> CrawlStatus:
+        """Poll until the crawl completes or fails."""
+        return await _async_poll_until_done(
+            fetcher=self.get_status, parser=lambda s: s,
+            label=f"Crawl {self.id}", interval=interval, timeout=timeout,
+            status_attr="status",
+        )
+
+
+# -- helpers ------------------------------------------------------------------
+
+async def _async_poll_until_done(
+    *, fetcher, parser, label: str, interval: float, timeout: float, status_attr: str = "status",
+) -> Any:
+    """Async version of poll-until-done. See client._poll_until_done."""
+    deadline = time.monotonic() + timeout
+    while True:
+        result = await fetcher()
+        status = result.get("status", "") if isinstance(result, dict) else getattr(result, status_attr)
+        if status == "completed":
+            return parser(result)
+        if status == "failed":
+            error = result.get("error", f"{label} failed") if isinstance(result, dict) else f"{label} failed"
+            raise WebclawError(error, status_code=None)
+        if time.monotonic() >= deadline:
+            raise TimeoutError(f"{label} did not complete within {timeout}s")
+        await asyncio.sleep(interval)