scrapyio-sdk 0.1.0__tar.gz

scrapyio_sdk-0.1.0/.gitignore

@@ -0,0 +1,64 @@
+# Binaries
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+bin/
+dist/
+
+# Test binary
+*.test
+
+# Output of the go coverage tool
+*.out
+
+# Dependency directories
+vendor/
+
+# Go workspace file
+go.work
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Project specific
+stashedfiles/
+data/
+tmp/
+logs/
+*.log
+
+# Config (if contains secrets)
+configs/*.local.yaml
+
+# Node.js
+node_modules/
+
+# Next.js
+.next/
+out/
+
+# Build
+**/dist/
+backend/.env.example
+
+# Admin 
+frontend/src/lib/api/admin.ts
+backend/src/services/s3Image.ts
+backend/src/routes/admin.ts
+frontend/src/app/admin/upload-client.tsx
+**/upload-client.tsx

scrapyio_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,142 @@
+Metadata-Version: 2.5
+Name: scrapyio-sdk
+Version: 0.1.0
+Summary: Official Python SDK for the Scrapy.io Platform API and publisher execution APIs
+Project-URL: Homepage, https://scrapy.io
+Project-URL: Documentation, https://docs.scrapy.io
+Project-URL: Repository, https://github.com/scrapyio/public-dd
+Author: Scrapy.io
+License-Expression: MIT
+Keywords: scrapy.io,scrapyio,sdk,web-scraping
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: httpx>=0.27
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: respx>=0.21; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scrapyio-sdk
+
+Official Python SDK for [Scrapy.io](https://scrapy.io).
+
+**Server-side only.** Keep API keys off client devices.
+
+## Install
+
+```bash
+pip install scrapyio-sdk
+```
+
+## Authentication
+
+```python
+import os
+from scrapyio_sdk import ScrapyIO
+
+client = ScrapyIO(api_key=os.environ["SCRAPYIO_API_KEY"])
+```
+
+Always sends `Authorization: Bearer <api_key>`.
+
+## First request
+
+```python
+tools = client.tools.list(q="instagram", limit=20)
+# {"items", "total", "offset", "limit"}
+
+tool = client.tools.get("datadoping", "instagram-profile-scraper")
+# or:
+same = client.tool("datadoping/instagram-profile-scraper").get()
+```
+
+## Run a scraper
+
+### Sync (`/v1/api`)
+
+```python
+result = client.tool("datadoping/instagram-profile-scraper").call({
+    "username": "nasa",
+})
+```
+
+### Async (`/v1/scraper`) + poll
+
+```python
+run = client.tool("datadoping/instagram-profile-scraper").start({
+    "usernames": ["nasa"],
+})
+
+finished = client.run(run["id"]).wait(
+    poll_interval_ms=3000,
+    timeout_ms=15 * 60_000,
+)
+
+page = client.run(run["id"]).list_items(offset=0, limit=100)
+```
+
+## Schedules
+
+```python
+schedule = client.schedules.create(
+    {
+        "publisher": "datadoping",
+        "slug": "instagram-profile-scraper",
+        "runName": "nightly-ig",
+        "timezone": "UTC",
+        "frequency": "one-time",
+        "date": "2026-09-20",
+        "time": "23:50",
+        "inputs": ["nasa"],
+    },
+    idempotency_key="nightly-ig-v1",
+)
+
+client.schedules.update(schedule["id"], {"isActive": False})
+client.schedules.delete(schedule["id"])
+```
+
+## Errors
+
+```python
+from scrapyio_sdk import ScrapyIO, ScrapyIOError
+
+try:
+    client.tools.get("nope", "missing")
+except ScrapyIOError as err:
+    # err.type, err.status, err.message, err.doc_url
+    ...
+```
+
+## Mental model
+
+```text
+client
+├── tools.list / tools.get
+├── tool("publisher/slug").get / .call / .start
+├── runs.list / runs.get
+├── run(id).get / .wait / .list_items
+└── schedules.list / get / create / update / delete
+```
+
+## Development
+
+```bash
+cd sdk/python
+python -m pip install -e ".[dev]"
+pytest
+```
+
+## Notes
+
+- GET requests may retry on transient 5xx/429; execution POSTs are not auto-retried.
+- `list_items()` is JSON-only in v0.1.
+- Companion JS package: [`@scrapyio/sdk`](https://www.npmjs.com/package/@scrapyio/sdk).

scrapyio_sdk-0.1.0/README.md

@@ -0,0 +1,116 @@
+# scrapyio-sdk
+
+Official Python SDK for [Scrapy.io](https://scrapy.io).
+
+**Server-side only.** Keep API keys off client devices.
+
+## Install
+
+```bash
+pip install scrapyio-sdk
+```
+
+## Authentication
+
+```python
+import os
+from scrapyio_sdk import ScrapyIO
+
+client = ScrapyIO(api_key=os.environ["SCRAPYIO_API_KEY"])
+```
+
+Always sends `Authorization: Bearer <api_key>`.
+
+## First request
+
+```python
+tools = client.tools.list(q="instagram", limit=20)
+# {"items", "total", "offset", "limit"}
+
+tool = client.tools.get("datadoping", "instagram-profile-scraper")
+# or:
+same = client.tool("datadoping/instagram-profile-scraper").get()
+```
+
+## Run a scraper
+
+### Sync (`/v1/api`)
+
+```python
+result = client.tool("datadoping/instagram-profile-scraper").call({
+    "username": "nasa",
+})
+```
+
+### Async (`/v1/scraper`) + poll
+
+```python
+run = client.tool("datadoping/instagram-profile-scraper").start({
+    "usernames": ["nasa"],
+})
+
+finished = client.run(run["id"]).wait(
+    poll_interval_ms=3000,
+    timeout_ms=15 * 60_000,
+)
+
+page = client.run(run["id"]).list_items(offset=0, limit=100)
+```
+
+## Schedules
+
+```python
+schedule = client.schedules.create(
+    {
+        "publisher": "datadoping",
+        "slug": "instagram-profile-scraper",
+        "runName": "nightly-ig",
+        "timezone": "UTC",
+        "frequency": "one-time",
+        "date": "2026-09-20",
+        "time": "23:50",
+        "inputs": ["nasa"],
+    },
+    idempotency_key="nightly-ig-v1",
+)
+
+client.schedules.update(schedule["id"], {"isActive": False})
+client.schedules.delete(schedule["id"])
+```
+
+## Errors
+
+```python
+from scrapyio_sdk import ScrapyIO, ScrapyIOError
+
+try:
+    client.tools.get("nope", "missing")
+except ScrapyIOError as err:
+    # err.type, err.status, err.message, err.doc_url
+    ...
+```
+
+## Mental model
+
+```text
+client
+├── tools.list / tools.get
+├── tool("publisher/slug").get / .call / .start
+├── runs.list / runs.get
+├── run(id).get / .wait / .list_items
+└── schedules.list / get / create / update / delete
+```
+
+## Development
+
+```bash
+cd sdk/python
+python -m pip install -e ".[dev]"
+pytest
+```
+
+## Notes
+
+- GET requests may retry on transient 5xx/429; execution POSTs are not auto-retried.
+- `list_items()` is JSON-only in v0.1.
+- Companion JS package: [`@scrapyio/sdk`](https://www.npmjs.com/package/@scrapyio/sdk).

scrapyio_sdk-0.1.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scrapyio-sdk"
+version = "0.1.0"
+description = "Official Python SDK for the Scrapy.io Platform API and publisher execution APIs"
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.10"
+authors = [{ name = "Scrapy.io" }]
+keywords = ["scrapyio", "scrapy.io", "web-scraping", "sdk"]
+dependencies = [
+  "httpx>=0.27",
+]
+classifiers = [
+  "Development Status :: 4 - Beta",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: MIT License",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
+  "Typing :: Typed",
+]
+
+[project.urls]
+Homepage = "https://scrapy.io"
+Documentation = "https://docs.scrapy.io"
+Repository = "https://github.com/scrapyio/public-dd"
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "respx>=0.21"]
+
+[tool.hatch.build.targets.wheel]
+only-include = ["scrapyio_sdk"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/scrapyio_sdk", "README.md", "pyproject.toml", "tests"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

scrapyio_sdk-0.1.0/src/scrapyio_sdk/__init__.py

@@ -0,0 +1,20 @@
+"""Official Python SDK for Scrapy.io."""
+
+from .client import ScrapyIO
+from .errors import ScrapyIOError
+from .pagination import Page
+from .run import RunHandle
+from .tool import ToolHandle
+from .types import TERMINAL_RUN_STATUSES, is_terminal_run_status
+
+__all__ = [
+    "ScrapyIO",
+    "ScrapyIOError",
+    "Page",
+    "RunHandle",
+    "ToolHandle",
+    "TERMINAL_RUN_STATUSES",
+    "is_terminal_run_status",
+]
+
+__version__ = "0.1.0"

scrapyio_sdk-0.1.0/src/scrapyio_sdk/client.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import httpx
+
+from .errors import ScrapyIOError
+from .platform import PlatformClient
+from .run import RunHandle
+from .runs import RunsResource
+from .schedules import SchedulesResource
+from .tool import ToolHandle
+from .tools import ToolsResource
+
+DEFAULT_BASE_URL = "https://api.scrapy.io/v1"
+
+
+class ScrapyIO:
+    """Official Scrapy.io Python SDK.
+
+    API keys must stay server-side. Do not embed keys in client apps.
+    """
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        base_url: str = DEFAULT_BASE_URL,
+        transport: httpx.BaseTransport | None = None,
+        max_retries: int = 2,
+        retry_backoff_ms: int = 250,
+        timeout: float = 60.0,
+    ) -> None:
+        key = (api_key or "").strip()
+        if not key:
+            raise ScrapyIOError(
+                "ScrapyIO requires an api_key (e.g. os.environ['SCRAPYIO_API_KEY'])",
+                type="sdk_error",
+            )
+
+        self._api_key = key
+        self._transport = transport
+        self._platform = PlatformClient(
+            base_url=base_url,
+            api_key=key,
+            transport=transport,
+            max_retries=max_retries,
+            retry_backoff_ms=retry_backoff_ms,
+            timeout=timeout,
+        )
+        self.tools = ToolsResource(self._platform)
+        self.runs = RunsResource(self._platform)
+        self.schedules = SchedulesResource(self._platform)
+
+    def close(self) -> None:
+        self._platform.close()
+
+    def __enter__(self) -> ScrapyIO:
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+    def tool(self, ref: str) -> ToolHandle:
+        return ToolHandle(
+            self._platform,
+            api_key=self._api_key,
+            transport=self._transport,
+            ref=ref,
+        )
+
+    def run(self, run_id: str) -> RunHandle:
+        return RunHandle(self._platform, run_id)

scrapyio_sdk-0.1.0/src/scrapyio_sdk/errors.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+PlatformErrorType = Literal[
+    "unauthorized",
+    "forbidden",
+    "not_found",
+    "validation_error",
+    "insufficient_credits",
+    "conflict",
+    "rate_limit_exceeded",
+    "internal_error",
+    "timeout",
+    "publisher_error",
+    "sdk_error",
+]
+
+_PLATFORM_ERROR_TYPES = frozenset(
+    {
+        "unauthorized",
+        "forbidden",
+        "not_found",
+        "validation_error",
+        "insufficient_credits",
+        "conflict",
+        "rate_limit_exceeded",
+        "internal_error",
+    }
+)
+
+
+class ScrapyIOError(Exception):
+    def __init__(
+        self,
+        message: str,
+        *,
+        type: PlatformErrorType = "sdk_error",
+        status: int | None = None,
+        doc_url: str | None = None,
+        body: Any = None,
+    ) -> None:
+        super().__init__(message)
+        self.message = message
+        self.type = type
+        self.status = status
+        self.doc_url = doc_url
+        self.body = body
+
+    def __str__(self) -> str:
+        return self.message
+
+
+def is_platform_error_type(value: str) -> bool:
+    return value in _PLATFORM_ERROR_TYPES
+
+
+def error_from_platform_body(
+    status: int,
+    body: Any,
+    fallback_message: str = "Platform API request failed",
+) -> ScrapyIOError:
+    if isinstance(body, dict):
+        err = body.get("error")
+        if isinstance(err, dict):
+            type_raw = err.get("type") if isinstance(err.get("type"), str) else "internal_error"
+            err_type: PlatformErrorType = (
+                type_raw if is_platform_error_type(type_raw) else "internal_error"  # type: ignore[assignment]
+            )
+            msg = err.get("message")
+            return ScrapyIOError(
+                msg if isinstance(msg, str) and msg else fallback_message,
+                type=err_type,
+                status=status,
+                doc_url=err.get("doc_url") if isinstance(err.get("doc_url"), str) else None,
+                body=body,
+            )
+    return ScrapyIOError(
+        fallback_message,
+        type="unauthorized" if status == 401 else "internal_error",
+        status=status,
+        body=body,
+    )
+
+
+def error_from_publisher_body(
+    status: int,
+    body: Any,
+    fallback_message: str = "Publisher execution request failed",
+) -> ScrapyIOError:
+    if isinstance(body, dict):
+        if body.get("error") == "INSUFFICIENT_CREDITS":
+            msg = body.get("message")
+            return ScrapyIOError(
+                msg if isinstance(msg, str) and msg else "Insufficient credits",
+                type="insufficient_credits",
+                status=status or 402,
+                body=body,
+            )
+        err = body.get("error")
+        if isinstance(err, str) and err:
+            message = body.get("message")
+            full = f"{err}: {message}" if isinstance(message, str) and message else err
+            if status == 401:
+                err_type: PlatformErrorType = "unauthorized"
+            elif status == 404:
+                err_type = "not_found"
+            elif status == 400:
+                err_type = "validation_error"
+            elif status == 409:
+                err_type = "conflict"
+            else:
+                err_type = "publisher_error"
+            return ScrapyIOError(full, type=err_type, status=status, body=body)
+        if body.get("success") is False and isinstance(body.get("error"), str):
+            return ScrapyIOError(
+                body["error"],
+                type="unauthorized" if status == 401 else "publisher_error",
+                status=status,
+                body=body,
+            )
+    return ScrapyIOError(fallback_message, type="publisher_error", status=status, body=body)

scrapyio_sdk-0.1.0/src/scrapyio_sdk/execution.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+import httpx
+
+from .errors import ScrapyIOError, error_from_publisher_body
+from .types import JsonObject, RunStatus
+
+
+def unwrap_publisher_success(body: Any) -> Any:
+    current = body
+    for _ in range(4):
+        if not isinstance(current, dict) or "success" not in current:
+            return current
+        if current.get("success") is False:
+            raise error_from_publisher_body(400, current)
+        if current.get("success") is not True:
+            return current
+        current = current.get("data")
+    return current
+
+
+def normalize_publisher_status(raw: str | None) -> RunStatus:
+    if not raw:
+        return "queued"
+    if raw == "completed":
+        return "succeeded"
+    if raw in {"queued", "running", "succeeded", "failed", "cancelled", "partial"}:
+        return raw  # type: ignore[return-value]
+    return "queued"
+
+
+def execute_call(
+    *,
+    api_key: str,
+    api_url: str,
+    input: JsonObject,
+    transport: httpx.BaseTransport | None = None,
+    timeout: float = 120.0,
+) -> Any:
+    with httpx.Client(transport=transport, timeout=timeout) as client:
+        response = client.post(
+            api_url,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            },
+            json=input,
+        )
+    body = _read_json(response)
+    if not response.is_success:
+        raise error_from_publisher_body(response.status_code, body)
+    return unwrap_publisher_success(body)
+
+
+def execute_start(
+    *,
+    api_key: str,
+    scraper_url: str,
+    input: JsonObject,
+    transport: httpx.BaseTransport | None = None,
+    timeout: float = 120.0,
+) -> dict[str, Any]:
+    with httpx.Client(transport=transport, timeout=timeout) as client:
+        response = client.post(
+            scraper_url,
+            headers={
+                "Authorization": f"Bearer {api_key}",
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            },
+            json=input,
+        )
+    body = _read_json(response)
+    if not response.is_success:
+        raise error_from_publisher_body(response.status_code, body)
+    if not isinstance(body, dict) or not isinstance(body.get("taskId"), str) or not body["taskId"]:
+        raise ScrapyIOError(
+            "Publisher did not return a taskId",
+            type="publisher_error",
+            status=response.status_code,
+            body=body,
+        )
+    status = normalize_publisher_status(body.get("status") if isinstance(body.get("status"), str) else None)
+    return {
+        "id": body["taskId"],
+        "kind": "async_batch",
+        "status": status,
+        "toolId": "",
+        "runName": body.get("runName"),
+        "totalItems": body.get("totalItems"),
+        "processedItems": body.get("processedItems"),
+        "completedItems": body.get("completedItems"),
+        "failedItems": body.get("failedItems"),
+        "estimatedCost": body.get("estimatedCost"),
+        "billedAmount": None,
+        "publisher": None,
+        "toolSlug": None,
+        "createdAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+        "completedAt": None,
+    }
+
+
+def _read_json(response: httpx.Response) -> Any:
+    if not response.content:
+        return None
+    try:
+        return response.json()
+    except ValueError:
+        return response.text