coreclaw-client 1.0.0__py3-none-any.whl

coreclaw/__init__.py

@@ -0,0 +1,16 @@
+"""CoreClaw Python SDK."""
+
+from .client import CoreClawAsyncClient, CoreClawClient
+from .exceptions import CoreClawAPIError, CoreClawError, CoreClawHTTPError, CoreClawResponseError
+
+__version__ = "1.0.0"
+
+__all__ = [
+    "CoreClawAPIError",
+    "CoreClawAsyncClient",
+    "CoreClawClient",
+    "CoreClawError",
+    "CoreClawHTTPError",
+    "CoreClawResponseError",
+    "__version__",
+]

coreclaw/__main__.py

@@ -0,0 +1,5 @@
+from .cli import main
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(["--version"]))

coreclaw/cli.py

@@ -0,0 +1,26 @@
+import argparse
+from collections.abc import Sequence
+
+from . import __version__
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="coreclaw")
+    parser.add_argument(
+        "--version",
+        action="store_true",
+        help="print the package version and exit",
+    )
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.version:
+        print(f"coreclaw {__version__}")
+        return 0
+
+    parser.print_help()
+    return 0

coreclaw/client.py

@@ -0,0 +1,109 @@
+from typing import Any
+
+import httpx
+
+from .exceptions import CoreClawAPIError, CoreClawHTTPError, CoreClawResponseError
+from .resources import AsyncRunClient, AsyncScraperClient, RunClient, ScraperClient
+
+
+DEFAULT_BASE_URL = "https://openapi.cafescraper.com"
+
+
+class CoreClawClient:
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str | None = None,
+        timeout: float = 30.0,
+        transport: httpx.BaseTransport | None = None,
+    ) -> None:
+        self._client = httpx.Client(
+            base_url=_normalize_base_url(base_url or DEFAULT_BASE_URL),
+            headers={"api-key": api_key},
+            timeout=timeout,
+            transport=transport,
+        )
+
+    def scraper(self, scraper_slug: str) -> ScraperClient:
+        return ScraperClient(self, scraper_slug)
+
+    def run(self, run_slug: str) -> RunClient:
+        return RunClient(self, run_slug)
+
+    def post(self, path: str, payload: dict[str, Any]) -> Any:
+        response = self._client.post(path, json=payload)
+        return _parse_response(response)
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "CoreClawClient":
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+
+class CoreClawAsyncClient:
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str | None = None,
+        timeout: float = 30.0,
+        transport: httpx.AsyncBaseTransport | httpx.BaseTransport | None = None,
+    ) -> None:
+        self._client = httpx.AsyncClient(
+            base_url=_normalize_base_url(base_url or DEFAULT_BASE_URL),
+            headers={"api-key": api_key},
+            timeout=timeout,
+            transport=transport,
+        )
+
+    def scraper(self, scraper_slug: str) -> AsyncScraperClient:
+        return AsyncScraperClient(self, scraper_slug)
+
+    def run(self, run_slug: str) -> AsyncRunClient:
+        return AsyncRunClient(self, run_slug)
+
+    async def post(self, path: str, payload: dict[str, Any]) -> Any:
+        response = await self._client.post(path, json=payload)
+        return _parse_response(response)
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    async def __aenter__(self) -> "CoreClawAsyncClient":
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        await self.close()
+
+
+def _normalize_base_url(base_url: str) -> str:
+    return base_url.rstrip("/") + "/api/v1/"
+
+
+def _parse_response(response: httpx.Response) -> Any:
+    if response.status_code < 200 or response.status_code >= 300:
+        raise CoreClawHTTPError(response.status_code, response.text)
+
+    try:
+        payload = response.json()
+    except ValueError as exc:
+        raise CoreClawResponseError("response body is not valid JSON") from exc
+
+    if not isinstance(payload, dict):
+        raise CoreClawResponseError("response body must be a JSON object")
+
+    if "code" not in payload or "message" not in payload:
+        raise CoreClawResponseError("response body is missing code or message")
+
+    code = payload["code"]
+    message = str(payload["message"])
+
+    if code != 0:
+        raise CoreClawAPIError(int(code), message, payload.get("data"))
+
+    return payload.get("data")

coreclaw/exceptions.py

@@ -0,0 +1,24 @@
+from typing import Any
+
+
+class CoreClawError(Exception):
+    """Base class for all CoreClaw SDK errors."""
+
+
+class CoreClawHTTPError(CoreClawError):
+    def __init__(self, status_code: int, message: str) -> None:
+        self.status_code = status_code
+        super().__init__(f"CoreClaw HTTP error {status_code}: {message}")
+
+
+class CoreClawResponseError(CoreClawError):
+    def __init__(self, message: str) -> None:
+        super().__init__(f"CoreClaw response error: {message}")
+
+
+class CoreClawAPIError(CoreClawError):
+    def __init__(self, code: int, message: str, data: Any = None) -> None:
+        self.code = code
+        self.message = message
+        self.data = data
+        super().__init__(f"CoreClaw API error {code}: {message}")

coreclaw/pagination.py

@@ -0,0 +1,9 @@
+def limit_offset_to_page(limit: int, offset: int) -> tuple[int, int]:
+    if limit < 1:
+        raise ValueError("limit must be greater than or equal to 1")
+    if offset < 0:
+        raise ValueError("offset must be greater than or equal to 0")
+    if offset % limit != 0:
+        raise ValueError("offset must be divisible by limit")
+
+    return offset // limit + 1, limit

coreclaw/resources.py

@@ -0,0 +1,118 @@
+from typing import Any
+
+from .pagination import limit_offset_to_page
+
+
+class ScraperClient:
+    def __init__(self, client: Any, scraper_slug: str) -> None:
+        self._client = client
+        self._scraper_slug = scraper_slug
+
+    def run(
+        self,
+        *,
+        input: dict[str, Any],
+        version: str | None = None,
+        callback_url: str | None = None,
+        wait_for_finish: bool = False,
+        limit: int = 10,
+        offset: int = 0,
+    ) -> Any:
+        page_index, page_size = limit_offset_to_page(limit, offset)
+        payload = _build_scraper_run_payload(
+            self._scraper_slug,
+            input,
+            version,
+            callback_url,
+            wait_for_finish,
+            page_index,
+            page_size,
+        )
+        return self._client.post("scraper/run", payload)
+
+
+class AsyncScraperClient:
+    def __init__(self, client: Any, scraper_slug: str) -> None:
+        self._client = client
+        self._scraper_slug = scraper_slug
+
+    async def run(
+        self,
+        *,
+        input: dict[str, Any],
+        version: str | None = None,
+        callback_url: str | None = None,
+        wait_for_finish: bool = False,
+        limit: int = 10,
+        offset: int = 0,
+    ) -> Any:
+        page_index, page_size = limit_offset_to_page(limit, offset)
+        payload = _build_scraper_run_payload(
+            self._scraper_slug,
+            input,
+            version,
+            callback_url,
+            wait_for_finish,
+            page_index,
+            page_size,
+        )
+        return await self._client.post("scraper/run", payload)
+
+
+class RunClient:
+    def __init__(self, client: Any, run_slug: str) -> None:
+        self._client = client
+        self._run_slug = run_slug
+
+    def list_results(self, *, limit: int = 10, offset: int = 0) -> Any:
+        page_index, page_size = limit_offset_to_page(limit, offset)
+        return self._client.post(
+            "run/result/list",
+            {
+                "run_slug": self._run_slug,
+                "page_index": page_index,
+                "page_size": page_size,
+            },
+        )
+
+
+class AsyncRunClient:
+    def __init__(self, client: Any, run_slug: str) -> None:
+        self._client = client
+        self._run_slug = run_slug
+
+    async def list_results(self, *, limit: int = 10, offset: int = 0) -> Any:
+        page_index, page_size = limit_offset_to_page(limit, offset)
+        return await self._client.post(
+            "run/result/list",
+            {
+                "run_slug": self._run_slug,
+                "page_index": page_index,
+                "page_size": page_size,
+            },
+        )
+
+
+def _build_scraper_run_payload(
+    scraper_slug: str,
+    input: dict[str, Any],
+    version: str | None,
+    callback_url: str | None,
+    wait_for_finish: bool,
+    page_index: int,
+    page_size: int,
+) -> dict[str, Any]:
+    payload: dict[str, Any] = {
+        "scraper_slug": scraper_slug,
+        "input": input,
+        "is_async": not wait_for_finish,
+        "page_index": page_index,
+        "page_size": page_size,
+    }
+
+    if version is not None:
+        payload["version"] = version
+    if callback_url is not None:
+        payload["callback_url"] = callback_url
+
+    return payload

coreclaw_client-1.0.0.dist-info/METADATA

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: coreclaw-client
+Version: 1.0.0
+Summary: CoreClaw Python SDK client.
+Author: CoreClaw maintainers
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.24
+
+# CoreClaw Python SDK
+
+CoreClaw Python SDK for running scraper scripts and reading run results.
+
+## Install
+
+```bash
+pip install coreclaw-client
+```
+
+## Run a Scraper
+
+```python
+from coreclaw import CoreClawClient
+
+client = CoreClawClient(api_key="YOUR_API_KEY")
+
+run = client.scraper("SCRAPER_SLUG").run(
+    input={
+        "parameters": {
+            "system": {
+                "proxy_region": "US",
+                "cpus": 0.125,
+                "memory": 512,
+                "execute_limit_time_seconds": 1800,
+                "max_total_charge": 0,
+                "max_total_traffic": 0,
+            },
+            "custom": {
+                "keyword": "python",
+            },
+        }
+    },
+)
+
+print(run["run_slug"])
+client.close()
+```
+
+To run a specific version:
+
+```python
+run = client.scraper("SCRAPER_SLUG").run(
+    input={...},
+    version="v1.0.1",
+)
+```
+
+If `version` is not provided, CoreClaw uses the latest available version.
+
+## Get Run Results
+
+```python
+from coreclaw import CoreClawClient
+
+client = CoreClawClient(api_key="YOUR_API_KEY")
+
+results = client.run("RUN_SLUG").list_results(limit=10, offset=0)
+
+print(results["count"])
+for item in results["list"]:
+    print(item)
+
+client.close()
+```
+
+## Async Usage
+
+Run a scraper:
+
+```python
+from coreclaw import CoreClawAsyncClient
+
+client = CoreClawAsyncClient(api_key="YOUR_API_KEY")
+run = await client.scraper("SCRAPER_SLUG").run(input={...})
+await client.close()
+```
+
+Get run results:
+
+```python
+from coreclaw import CoreClawAsyncClient
+
+client = CoreClawAsyncClient(api_key="YOUR_API_KEY")
+results = await client.run("RUN_SLUG").list_results(limit=10, offset=0)
+await client.close()
+```
+
+## More
+
+- Chinese README: [README.zh-CN.md](README.zh-CN.md)
+- Run scraper demo: [examples/run_scraper.py](examples/run_scraper.py)
+- Get results demo: [examples/get_results.py](examples/get_results.py)
+- Async run scraper demo: [examples/async_run_scraper.py](examples/async_run_scraper.py)
+- Async get results demo: [examples/async_get_results.py](examples/async_get_results.py)

coreclaw_client-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+coreclaw/__init__.py,sha256=XdJf3xchUA5qYMMaktm1Hi9x1uBOl3gpKbU2FoUb-A0,387
+coreclaw/__main__.py,sha256=nBzPa5T3-3BOCBetr5YfKJdwvp8R15bXsL0tFKTTolI,93
+coreclaw/cli.py,sha256=qFQzC338cK9N4SX9ijyeoxcQY3ArRsP4Fu8SJMlelxs,568
+coreclaw/client.py,sha256=GabkL5AlZl9FumkmC8bOKMQc84psGYJCI7VCQlBwIQU,3254
+coreclaw/exceptions.py,sha256=RYZOpdZI1fmvbAZobd1UNhF4dhdRrhaRaLVPAVBBFYY,745
+coreclaw/pagination.py,sha256=boGofnnsdKf_Cx3HggxnduRZn3omfsHpXK9PAIZtq9E,375
+coreclaw/resources.py,sha256=dEZIRkVZnSsvrkT6tfI7_I3te2dl0msoHnZVg-BoJBE,3260
+coreclaw_client-1.0.0.dist-info/METADATA,sha256=nluSQaphR2L5fssx7N0nyNPynAuyuJx8bjzQZDHL5dE,2371
+coreclaw_client-1.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+coreclaw_client-1.0.0.dist-info/entry_points.txt,sha256=NxJt0wGfwPyMWFpMN-RMF3Tc0zIwbt1CrBJD0_tTyP0,47
+coreclaw_client-1.0.0.dist-info/top_level.txt,sha256=_AUKKysucAwL_T-39bXO0ml5dLxGf7kY37wd0zvc7Lk,9
+coreclaw_client-1.0.0.dist-info/RECORD,,

coreclaw_client-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

coreclaw_client-1.0.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+coreclaw = coreclaw.cli:main

coreclaw_client-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+coreclaw