scrapio-py 1.0.0__tar.gz

scrapio_py-1.0.0/.gitignore

@@ -0,0 +1,8 @@
+node_modules/
+dist/
+coverage/
+.env
+artifacts/
+logs/
+*.tsbuildinfo
+.DS_Store

scrapio_py-1.0.0/PKG-INFO

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: scrapio-py
+Version: 1.0.0
+Summary: Official Python SDK for the Scrapio
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: httpx>=0.25.0
+Requires-Dist: pydantic>=2.0.0
+Provides-Extra: dev
+Requires-Dist: anyio[trio]; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.21; extra == 'dev'
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scrapio
+
+Official Python SDK for [Scrapio](https://scrapio.dev) — fetch, crawl, search, and extract structured data from any URL.
+
+## Install
+
+```bash
+pip install scrapio-py
+```
+
+Requires Python 3.9 or later.
+
+## Quickstart
+
+```python
+from scrapio import ApiClient, FetchRequest
+
+client = ApiClient(api_key="YOUR_API_KEY")
+
+result = client.fetch.fetch(FetchRequest(
+    url="https://example.com",
+    output=["markdown"],
+))
+
+print(result.outputs["markdown"])
+```
+
+## Usage
+
+### Fetch a page
+
+```python
+result = client.fetch.fetch(FetchRequest(
+    url="https://news.ycombinator.com",
+    render_js=True,
+    output=["markdown"],
+))
+```
+
+### Google Search
+
+```python
+from scrapio import GoogleSearchParams
+
+results = client.google.search(GoogleSearchParams(
+    search="best web scraping API 2025",
+    country_code="us",
+))
+print(results.organic_results)
+```
+
+### Amazon product
+
+```python
+product = client.amazon.get_product("B08N5WRWNW")
+print(product.title, product.price)
+```
+
+### Walmart search
+
+```python
+items = client.walmart.search("headphones")
+```
+
+### YouTube transcript
+
+```python
+video = client.youtube.get_video("dQw4w9WgXcQ")
+```
+
+### Browser automation
+
+```python
+result = client.interact.interact({
+    "url": "https://example.com",
+    "actions": [
+        {"type": "click", "selector": "#login"},
+        {"type": "type", "selector": "#email", "text": "user@example.com"},
+    ],
+})
+```
+
+### Crawl a site
+
+```python
+result = client.crawl.crawl({
+    "seeds": ["https://docs.example.com"],
+    "max_pages": 50,
+})
+```
+
+### Async jobs
+
+```python
+from scrapio import CreateJobRequest
+
+job = client.jobs.create(CreateJobRequest(
+    job_type="fetch",
+    payload={"url": "https://example.com", "output": ["markdown"]},
+))
+result = client.jobs.wait_for_completion(job.job_id, poll_interval=2.0, timeout=120.0)
+```
+
+### Async client
+
+```python
+import asyncio
+from scrapio import AsyncApiClient, FetchRequest
+
+async def main():
+    async with AsyncApiClient(api_key="YOUR_API_KEY") as client:
+        result = await client.fetch.fetch(FetchRequest(
+            url="https://example.com",
+            output=["markdown"],
+        ))
+        print(result.outputs["markdown"])
+
+asyncio.run(main())
+```
+
+## Configuration
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `api_key` | `str` | required | Your API key |
+| `base_url` | `str` | `https://api.scrapio.dev` | Override for local/staging |
+| `timeout` | `float` | `30.0` | Per-request timeout (seconds) |
+| `max_retries` | `int` | `3` | Max retries on 429/503 |
+
+## Error handling
+
+```python
+from scrapio import (
+    ApiClient, FetchRequest,
+    AuthError, RateLimitError, CreditsExhaustedError, ApiError,
+)
+
+try:
+    client.fetch.fetch(FetchRequest(url="https://example.com"))
+except AuthError:
+    print("Invalid API key")
+except CreditsExhaustedError:
+    print("No credits remaining")
+except RateLimitError:
+    print("Rate limited — back off and retry")
+except ApiError as e:
+    print(f"API error {e.status_code}: {e}")
+```
+
+## Links
+
+- [Documentation](https://scrapio.dev/docs)
+- [API Reference](https://scrapio.dev/docs/api-reference/fetch)
+- [Dashboard](https://app.scrapio.dev)
+- [Get an API key](https://scrapio.dev#pricing)
+
+## License
+
+MIT

scrapio_py-1.0.0/README.md

@@ -0,0 +1,159 @@
+# scrapio
+
+Official Python SDK for [Scrapio](https://scrapio.dev) — fetch, crawl, search, and extract structured data from any URL.
+
+## Install
+
+```bash
+pip install scrapio-py
+```
+
+Requires Python 3.9 or later.
+
+## Quickstart
+
+```python
+from scrapio import ApiClient, FetchRequest
+
+client = ApiClient(api_key="YOUR_API_KEY")
+
+result = client.fetch.fetch(FetchRequest(
+    url="https://example.com",
+    output=["markdown"],
+))
+
+print(result.outputs["markdown"])
+```
+
+## Usage
+
+### Fetch a page
+
+```python
+result = client.fetch.fetch(FetchRequest(
+    url="https://news.ycombinator.com",
+    render_js=True,
+    output=["markdown"],
+))
+```
+
+### Google Search
+
+```python
+from scrapio import GoogleSearchParams
+
+results = client.google.search(GoogleSearchParams(
+    search="best web scraping API 2025",
+    country_code="us",
+))
+print(results.organic_results)
+```
+
+### Amazon product
+
+```python
+product = client.amazon.get_product("B08N5WRWNW")
+print(product.title, product.price)
+```
+
+### Walmart search
+
+```python
+items = client.walmart.search("headphones")
+```
+
+### YouTube transcript
+
+```python
+video = client.youtube.get_video("dQw4w9WgXcQ")
+```
+
+### Browser automation
+
+```python
+result = client.interact.interact({
+    "url": "https://example.com",
+    "actions": [
+        {"type": "click", "selector": "#login"},
+        {"type": "type", "selector": "#email", "text": "user@example.com"},
+    ],
+})
+```
+
+### Crawl a site
+
+```python
+result = client.crawl.crawl({
+    "seeds": ["https://docs.example.com"],
+    "max_pages": 50,
+})
+```
+
+### Async jobs
+
+```python
+from scrapio import CreateJobRequest
+
+job = client.jobs.create(CreateJobRequest(
+    job_type="fetch",
+    payload={"url": "https://example.com", "output": ["markdown"]},
+))
+result = client.jobs.wait_for_completion(job.job_id, poll_interval=2.0, timeout=120.0)
+```
+
+### Async client
+
+```python
+import asyncio
+from scrapio import AsyncApiClient, FetchRequest
+
+async def main():
+    async with AsyncApiClient(api_key="YOUR_API_KEY") as client:
+        result = await client.fetch.fetch(FetchRequest(
+            url="https://example.com",
+            output=["markdown"],
+        ))
+        print(result.outputs["markdown"])
+
+asyncio.run(main())
+```
+
+## Configuration
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `api_key` | `str` | required | Your API key |
+| `base_url` | `str` | `https://api.scrapio.dev` | Override for local/staging |
+| `timeout` | `float` | `30.0` | Per-request timeout (seconds) |
+| `max_retries` | `int` | `3` | Max retries on 429/503 |
+
+## Error handling
+
+```python
+from scrapio import (
+    ApiClient, FetchRequest,
+    AuthError, RateLimitError, CreditsExhaustedError, ApiError,
+)
+
+try:
+    client.fetch.fetch(FetchRequest(url="https://example.com"))
+except AuthError:
+    print("Invalid API key")
+except CreditsExhaustedError:
+    print("No credits remaining")
+except RateLimitError:
+    print("Rate limited — back off and retry")
+except ApiError as e:
+    print(f"API error {e.status_code}: {e}")
+```
+
+## Links
+
+- [Documentation](https://scrapio.dev/docs)
+- [API Reference](https://scrapio.dev/docs/api-reference/fetch)
+- [Dashboard](https://app.scrapio.dev)
+- [Get an API key](https://scrapio.dev#pricing)
+
+## License
+
+MIT

scrapio_py-1.0.0/pyproject.toml

@@ -0,0 +1,24 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scrapio-py"
+version = "1.0.0"
+description = "Official Python SDK for the Scrapio"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "MIT" }
+dependencies = [
+    "httpx>=0.25.0",
+    "pydantic>=2.0.0",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=7.0", "pytest-asyncio>=0.21", "anyio[trio]"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scrapio"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

scrapio_py-1.0.0/src/scrapio/__init__.py

@@ -0,0 +1,21 @@
+from .client import ApiClient, AsyncApiClient
+from .errors import ApiError, AuthError, RateLimitError, CreditsExhaustedError
+from .types import (
+    FetchRequest, FetchResponse,
+    CreateJobRequest, Job, JobResult,
+    GoogleSearchParams, GoogleSearchResponse,
+    AmazonProductResponse, AmazonSearchResponse,
+    WalmartProductResponse, WalmartSearchResponse,
+    YouTubeVideoResponse, YouTubeSearchResponse, YouTubeSubtitleResponse,
+)
+
+__all__ = [
+    "ApiClient", "AsyncApiClient",
+    "ApiError", "AuthError", "RateLimitError", "CreditsExhaustedError",
+    "FetchRequest", "FetchResponse",
+    "CreateJobRequest", "Job", "JobResult",
+    "GoogleSearchParams", "GoogleSearchResponse",
+    "AmazonProductResponse", "AmazonSearchResponse",
+    "WalmartProductResponse", "WalmartSearchResponse",
+    "YouTubeVideoResponse", "YouTubeSearchResponse", "YouTubeSubtitleResponse",
+]

scrapio_py-1.0.0/src/scrapio/_http.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any, Optional, Type, TypeVar
+
+import httpx
+from pydantic import BaseModel
+
+from .errors import ApiError, AuthError, CreditsExhaustedError, RateLimitError
+
+T = TypeVar("T", bound=BaseModel)
+
+RETRYABLE_STATUS = {429, 503}
+DEFAULT_TIMEOUT = 30.0
+DEFAULT_MAX_RETRIES = 3
+
+
+def _raise_for_status(status_code: int, body: dict[str, Any]) -> None:
+    code = body.get("error", {}).get("code", "")
+    if status_code == 401:
+        raise AuthError(body)
+    if status_code == 429:
+        raise RateLimitError(body)
+    if status_code == 402 or code == "credits_exhausted":
+        raise CreditsExhaustedError(body)
+    raise ApiError(status_code, body)
+
+
+class SyncHttpClient:
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ) -> None:
+        self._base_url = base_url.rstrip("/")
+        self._headers = {"Authorization": f"Bearer {api_key}"}
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._client = httpx.Client(
+            base_url=self._base_url,
+            headers=self._headers,
+            timeout=self._timeout,
+        )
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "SyncHttpClient":
+        return self
+
+    def __exit__(self, *args: Any) -> None:
+        self.close()
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Optional[dict[str, Any]] = None,
+        json: Optional[Any] = None,
+        response_model: Type[T],
+        timeout: Optional[float] = None,
+    ) -> T:
+        clean_params = {k: v for k, v in (params or {}).items() if v is not None}
+
+        for attempt in range(self._max_retries + 1):
+            res = self._client.request(
+                method,
+                path,
+                params=clean_params or None,
+                json=json,
+                timeout=timeout or self._timeout,
+            )
+            if res.is_success:
+                return response_model.model_validate(res.json())
+
+            body: dict[str, Any] = {}
+            try:
+                body = res.json()
+            except Exception:
+                body = {"request_id": "", "error": {"code": "unknown", "message": res.text}}
+
+            if res.status_code in RETRYABLE_STATUS and attempt < self._max_retries:
+                backoff = min(1.0 * (2**attempt), 8.0)
+                time.sleep(backoff)
+                continue
+
+            _raise_for_status(res.status_code, body)
+
+        raise RuntimeError("Unexpected end of retry loop")  # unreachable
+
+
+class AsyncHttpClient:
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ) -> None:
+        self._base_url = base_url.rstrip("/")
+        self._headers = {"Authorization": f"Bearer {api_key}"}
+        self._timeout = timeout
+        self._max_retries = max_retries
+        self._client = httpx.AsyncClient(
+            base_url=self._base_url,
+            headers=self._headers,
+            timeout=self._timeout,
+        )
+
+    async def aclose(self) -> None:
+        await self._client.aclose()
+
+    async def __aenter__(self) -> "AsyncHttpClient":
+        return self
+
+    async def __aexit__(self, *args: Any) -> None:
+        await self.aclose()
+
+    async def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Optional[dict[str, Any]] = None,
+        json: Optional[Any] = None,
+        response_model: Type[T],
+        timeout: Optional[float] = None,
+    ) -> T:
+        clean_params = {k: v for k, v in (params or {}).items() if v is not None}
+
+        for attempt in range(self._max_retries + 1):
+            res = await self._client.request(
+                method,
+                path,
+                params=clean_params or None,
+                json=json,
+                timeout=timeout or self._timeout,
+            )
+            if res.is_success:
+                return response_model.model_validate(res.json())
+
+            body: dict[str, Any] = {}
+            try:
+                body = res.json()
+            except Exception:
+                body = {"request_id": "", "error": {"code": "unknown", "message": res.text}}
+
+            if res.status_code in RETRYABLE_STATUS and attempt < self._max_retries:
+                backoff = min(1.0 * (2**attempt), 8.0)
+                await asyncio.sleep(backoff)
+                continue
+
+            _raise_for_status(res.status_code, body)
+
+        raise RuntimeError("Unexpected end of retry loop")  # unreachable

scrapio_py-1.0.0/src/scrapio/client.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+from typing import Optional
+from ._http import SyncHttpClient, AsyncHttpClient, DEFAULT_TIMEOUT, DEFAULT_MAX_RETRIES
+from .resources.fetch import FetchResource, AsyncFetchResource
+from .resources.jobs import JobsResource, AsyncJobsResource
+from .resources.google import GoogleResource, AsyncGoogleResource
+from .resources.amazon import AmazonResource, AsyncAmazonResource
+from .resources.walmart import WalmartResource, AsyncWalmartResource
+from .resources.youtube import YouTubeResource, AsyncYouTubeResource
+
+DEFAULT_BASE_URL = "https://api.webdataapi.com"
+
+
+class ApiClient:
+    fetch: FetchResource
+    jobs: JobsResource
+    google: GoogleResource
+    amazon: AmazonResource
+    walmart: WalmartResource
+    youtube: YouTubeResource
+
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ) -> None:
+        http = SyncHttpClient(base_url, api_key, timeout=timeout, max_retries=max_retries)
+        self.fetch = FetchResource(http)
+        self.jobs = JobsResource(http)
+        self.google = GoogleResource(http)
+        self.amazon = AmazonResource(http)
+        self.walmart = WalmartResource(http)
+        self.youtube = YouTubeResource(http)
+        self._http = http
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "ApiClient":
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+
+class AsyncApiClient:
+    fetch: AsyncFetchResource
+    jobs: AsyncJobsResource
+    google: AsyncGoogleResource
+    amazon: AsyncAmazonResource
+    walmart: AsyncWalmartResource
+    youtube: AsyncYouTubeResource
+
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        base_url: str = DEFAULT_BASE_URL,
+        timeout: float = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ) -> None:
+        http = AsyncHttpClient(base_url, api_key, timeout=timeout, max_retries=max_retries)
+        self.fetch = AsyncFetchResource(http)
+        self.jobs = AsyncJobsResource(http)
+        self.google = AsyncGoogleResource(http)
+        self.amazon = AsyncAmazonResource(http)
+        self.walmart = AsyncWalmartResource(http)
+        self.youtube = AsyncYouTubeResource(http)
+        self._http = http
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncApiClient":
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        await self.aclose()

scrapio_py-1.0.0/src/scrapio/errors.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+from typing import Any
+
+
+class ApiError(Exception):
+    status_code: int
+    request_id: str
+    code: str
+
+    def __init__(self, status_code: int, body: dict[str, Any]) -> None:
+        error = body.get("error", {})
+        super().__init__(error.get("message", "Unknown error"))
+        self.status_code = status_code
+        self.request_id = body.get("request_id", "")
+        self.code = error.get("code", "unknown")
+
+
+class AuthError(ApiError):
+    def __init__(self, body: dict[str, Any]) -> None:
+        super().__init__(401, body)
+
+
+class RateLimitError(ApiError):
+    def __init__(self, body: dict[str, Any]) -> None:
+        super().__init__(429, body)
+
+
+class CreditsExhaustedError(ApiError):
+    def __init__(self, body: dict[str, Any]) -> None:
+        super().__init__(402, body)

scrapio_py-1.0.0/src/scrapio/resources/amazon.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+from typing import Optional
+from .._http import SyncHttpClient, AsyncHttpClient
+from ..types import AmazonProductResponse, AmazonSearchResponse
+
+
+class AmazonResource:
+    def __init__(self, http: SyncHttpClient) -> None:
+        self._http = http
+
+    def get_product(self, asin: str, *, country: Optional[str] = None) -> AmazonProductResponse:
+        return self._http.request(
+            "GET", "/v1/amazon/product",
+            params={"asin": asin, "country": country},
+            response_model=AmazonProductResponse,
+        )
+
+    def search(self, query: str, *, country: Optional[str] = None, page: Optional[int] = None) -> AmazonSearchResponse:
+        return self._http.request(
+            "GET", "/v1/amazon/search",
+            params={"query": query, "country": country, "page": page},
+            response_model=AmazonSearchResponse,
+        )
+
+    def queue_search_crawl(self, query: str, *, country: Optional[str] = None) -> dict:
+        from pydantic import RootModel
+        from typing import Any
+        class _R(RootModel[dict[str, Any]]): pass
+        result = self._http.request(
+            "GET", "/v1/amazon/search/crawl",
+            params={"query": query, "country": country},
+            response_model=_R,
+        )
+        return result.root
+
+
+class AsyncAmazonResource:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def get_product(self, asin: str, *, country: Optional[str] = None) -> AmazonProductResponse:
+        return await self._http.request(
+            "GET", "/v1/amazon/product",
+            params={"asin": asin, "country": country},
+            response_model=AmazonProductResponse,
+        )
+
+    async def search(self, query: str, *, country: Optional[str] = None, page: Optional[int] = None) -> AmazonSearchResponse:
+        return await self._http.request(
+            "GET", "/v1/amazon/search",
+            params={"query": query, "country": country, "page": page},
+            response_model=AmazonSearchResponse,
+        )

scrapio_py-1.0.0/src/scrapio/resources/fetch.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+from .._http import SyncHttpClient, AsyncHttpClient
+from ..types import FetchRequest, FetchResponse
+
+
+class FetchResource:
+    def __init__(self, http: SyncHttpClient) -> None:
+        self._http = http
+
+    def fetch(self, request: FetchRequest, *, timeout: float | None = None) -> FetchResponse:
+        return self._http.request(
+            "POST", "/v1/fetch",
+            json=request.model_dump(exclude_none=True),
+            response_model=FetchResponse,
+            timeout=timeout,
+        )
+
+
+class AsyncFetchResource:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def fetch(self, request: FetchRequest, *, timeout: float | None = None) -> FetchResponse:
+        return await self._http.request(
+            "POST", "/v1/fetch",
+            json=request.model_dump(exclude_none=True),
+            response_model=FetchResponse,
+            timeout=timeout,
+        )

scrapio_py-1.0.0/src/scrapio/resources/google.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+from .._http import SyncHttpClient, AsyncHttpClient
+from ..types import GoogleSearchParams, GoogleSearchResponse
+
+
+class GoogleResource:
+    def __init__(self, http: SyncHttpClient) -> None:
+        self._http = http
+
+    def search(self, params: GoogleSearchParams) -> GoogleSearchResponse:
+        return self._http.request(
+            "GET", "/v1/google/search",
+            params=params.model_dump(exclude_none=True),
+            response_model=GoogleSearchResponse,
+        )
+
+
+class AsyncGoogleResource:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def search(self, params: GoogleSearchParams) -> GoogleSearchResponse:
+        return await self._http.request(
+            "GET", "/v1/google/search",
+            params=params.model_dump(exclude_none=True),
+            response_model=GoogleSearchResponse,
+        )

scrapio_py-1.0.0/src/scrapio/resources/jobs.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+import asyncio
+import time
+from .._http import SyncHttpClient, AsyncHttpClient
+from ..types import CreateJobRequest, Job, JobResult
+
+TERMINAL = {"completed", "partial", "failed", "cancelled"}
+
+
+class JobsResource:
+    def __init__(self, http: SyncHttpClient) -> None:
+        self._http = http
+
+    def create(self, request: CreateJobRequest) -> Job:
+        return self._http.request(
+            "POST", "/v1/jobs",
+            json=request.model_dump(exclude_none=True),
+            response_model=Job,
+        )
+
+    def get(self, job_id: str) -> Job:
+        return self._http.request("GET", f"/v1/jobs/{job_id}", response_model=Job)
+
+    def get_result(self, job_id: str) -> JobResult:
+        return self._http.request("GET", f"/v1/jobs/{job_id}/result", response_model=JobResult)
+
+    def wait_for_completion(
+        self,
+        job_id: str,
+        *,
+        poll_interval: float = 2.0,
+        timeout: float = 300.0,
+    ) -> JobResult:
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            job = self.get(job_id)
+            if job.status in TERMINAL:
+                return self.get_result(job_id)
+            time.sleep(poll_interval)
+        raise TimeoutError(f"Job {job_id} did not complete within {timeout}s")
+
+
+class AsyncJobsResource:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def create(self, request: CreateJobRequest) -> Job:
+        return await self._http.request(
+            "POST", "/v1/jobs",
+            json=request.model_dump(exclude_none=True),
+            response_model=Job,
+        )
+
+    async def get(self, job_id: str) -> Job:
+        return await self._http.request("GET", f"/v1/jobs/{job_id}", response_model=Job)
+
+    async def get_result(self, job_id: str) -> JobResult:
+        return await self._http.request("GET", f"/v1/jobs/{job_id}/result", response_model=JobResult)
+
+    async def wait_for_completion(
+        self,
+        job_id: str,
+        *,
+        poll_interval: float = 2.0,
+        timeout: float = 300.0,
+    ) -> JobResult:
+        deadline = asyncio.get_event_loop().time() + timeout
+        while asyncio.get_event_loop().time() < deadline:
+            job = await self.get(job_id)
+            if job.status in TERMINAL:
+                return await self.get_result(job_id)
+            await asyncio.sleep(poll_interval)
+        raise TimeoutError(f"Job {job_id} did not complete within {timeout}s")

scrapio_py-1.0.0/src/scrapio/resources/walmart.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+from typing import Optional
+from .._http import SyncHttpClient, AsyncHttpClient
+from ..types import WalmartProductResponse, WalmartSearchResponse
+
+
+class WalmartResource:
+    def __init__(self, http: SyncHttpClient) -> None:
+        self._http = http
+
+    def get_product(self, product_id: str, *, country: Optional[str] = None) -> WalmartProductResponse:
+        return self._http.request(
+            "GET", "/v1/walmart/product",
+            params={"product_id": product_id, "country": country},
+            response_model=WalmartProductResponse,
+        )
+
+    def search(self, query: str, *, country: Optional[str] = None, page: Optional[int] = None) -> WalmartSearchResponse:
+        return self._http.request(
+            "GET", "/v1/walmart/search",
+            params={"query": query, "country": country, "page": page},
+            response_model=WalmartSearchResponse,
+        )
+
+    def queue_search_crawl(self, query: str, *, country: Optional[str] = None) -> dict:
+        from pydantic import RootModel
+        from typing import Any
+        class _R(RootModel[dict[str, Any]]): pass
+        result = self._http.request(
+            "GET", "/v1/walmart/search/crawl",
+            params={"query": query, "country": country},
+            response_model=_R,
+        )
+        return result.root
+
+
+class AsyncWalmartResource:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def get_product(self, product_id: str, *, country: Optional[str] = None) -> WalmartProductResponse:
+        return await self._http.request(
+            "GET", "/v1/walmart/product",
+            params={"product_id": product_id, "country": country},
+            response_model=WalmartProductResponse,
+        )
+
+    async def search(self, query: str, *, country: Optional[str] = None, page: Optional[int] = None) -> WalmartSearchResponse:
+        return await self._http.request(
+            "GET", "/v1/walmart/search",
+            params={"query": query, "country": country, "page": page},
+            response_model=WalmartSearchResponse,
+        )

scrapio_py-1.0.0/src/scrapio/resources/youtube.py

@@ -0,0 +1,76 @@
+from __future__ import annotations
+from typing import Optional
+from .._http import SyncHttpClient, AsyncHttpClient
+from ..types import YouTubeVideoResponse, YouTubeSearchResponse, YouTubeSubtitleResponse
+
+
+class YouTubeResource:
+    def __init__(self, http: SyncHttpClient) -> None:
+        self._http = http
+
+    def search(self, query: str, *, page: Optional[int] = None, country: Optional[str] = None, language: Optional[str] = None) -> YouTubeSearchResponse:
+        return self._http.request(
+            "GET", "/v1/youtube/search",
+            params={"query": query, "page": page, "country": country, "language": language},
+            response_model=YouTubeSearchResponse,
+        )
+
+    def get_video(self, video_id: str) -> YouTubeVideoResponse:
+        return self._http.request(
+            "GET", f"/v1/youtube/videos/{video_id}",
+            response_model=YouTubeVideoResponse,
+        )
+
+    def get_subtitles(self, video_id: str, *, language: Optional[str] = None) -> YouTubeSubtitleResponse:
+        return self._http.request(
+            "GET", "/v1/youtube/subtitles",
+            params={"video_id": video_id, "language": language},
+            response_model=YouTubeSubtitleResponse,
+        )
+
+    def queue_search_crawl(self, query: str, *, page: Optional[int] = None) -> dict:
+        from pydantic import RootModel
+        from typing import Any
+        class _R(RootModel[dict[str, Any]]): pass
+        result = self._http.request(
+            "POST", "/v1/youtube/search/crawl",
+            json={"query": query, "page": page},
+            response_model=_R,
+        )
+        return result.root
+
+
+class AsyncYouTubeResource:
+    def __init__(self, http: AsyncHttpClient) -> None:
+        self._http = http
+
+    async def search(self, query: str, *, page: Optional[int] = None, country: Optional[str] = None, language: Optional[str] = None) -> YouTubeSearchResponse:
+        return await self._http.request(
+            "GET", "/v1/youtube/search",
+            params={"query": query, "page": page, "country": country, "language": language},
+            response_model=YouTubeSearchResponse,
+        )
+
+    async def get_video(self, video_id: str) -> YouTubeVideoResponse:
+        return await self._http.request(
+            "GET", f"/v1/youtube/videos/{video_id}",
+            response_model=YouTubeVideoResponse,
+        )
+
+    async def get_subtitles(self, video_id: str, *, language: Optional[str] = None) -> YouTubeSubtitleResponse:
+        return await self._http.request(
+            "GET", "/v1/youtube/subtitles",
+            params={"video_id": video_id, "language": language},
+            response_model=YouTubeSubtitleResponse,
+        )
+
+    async def queue_search_crawl(self, query: str, *, page: Optional[int] = None) -> dict:
+        from pydantic import RootModel
+        from typing import Any
+        class _R(RootModel[dict[str, Any]]): pass
+        result = await self._http.request(
+            "POST", "/v1/youtube/search/crawl",
+            json={"query": query, "page": page},
+            response_model=_R,
+        )
+        return result.root

scrapio_py-1.0.0/src/scrapio/types.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+from typing import Any, Literal, Optional, Union
+from pydantic import BaseModel
+
+
+# ---- Fetch ----
+
+class FetchSession(BaseModel):
+    id: str
+
+class FetchRequest(BaseModel):
+    url: str
+    render_js: Optional[bool] = None
+    device: Optional[Literal["desktop", "mobile", "tablet"]] = None
+    session: Optional[FetchSession] = None
+    output: Optional[list[str]] = None
+    extract: Optional[dict[str, Any]] = None
+    actions: Optional[list[Any]] = None
+    timeout: Optional[int] = None
+    proxy: Optional[str] = None
+    country: Optional[str] = None
+
+class FetchResponse(BaseModel):
+    request_id: str
+    url: str
+    status_code: int
+    outputs: dict[str, Any]
+    diagnostics: Optional[dict[str, Any]] = None
+
+
+# ---- Jobs ----
+
+JobStatus = Literal["queued", "running", "completed", "partial", "failed", "cancelled"]
+
+class CreateJobRequest(BaseModel):
+    job_type: str
+    payload: dict[str, Any]
+    webhook_url: Optional[str] = None
+
+class Job(BaseModel):
+    request_id: str
+    job_id: str
+    job_type: str
+    status: str
+    created_at: str
+    updated_at: Optional[str] = None
+    webhook_url: Optional[str] = None
+
+class JobError(BaseModel):
+    code: str
+    message: str
+
+class JobResult(Job):
+    result: Optional[Any] = None
+    error: Optional[JobError] = None
+
+
+# ---- Google ----
+
+GoogleSearchType = Literal["classic", "news", "maps", "images", "lens", "shopping", "ai_mode", "ads"]
+GoogleDevice = Literal["desktop", "mobile"]
+GoogleDateRange = Literal["past_hour", "past_day", "past_week", "past_month", "past_year"]
+GoogleSortBy = Literal["relevance", "reviews", "price_asc", "price_desc"]
+
+class GoogleSearchParams(BaseModel):
+    search: str
+    search_type: Optional[GoogleSearchType] = None
+    country_code: Optional[str] = None
+    language: Optional[str] = None
+    device: Optional[GoogleDevice] = None
+    page: Optional[Union[int, str]] = None
+    date_range: Optional[GoogleDateRange] = None
+    latitude: Optional[Union[float, str]] = None
+    longitude: Optional[Union[float, str]] = None
+    radius: Optional[Union[float, str]] = None
+    sort_by: Optional[GoogleSortBy] = None
+
+class GoogleSearchResponse(BaseModel):
+    request_id: str
+    results: list[Any]
+    pagination: Optional[dict[str, Any]] = None
+
+
+# ---- Amazon ----
+
+class AmazonProductResponse(BaseModel):
+    provider: str
+    asin: str
+    title: str
+    brand: Optional[str] = None
+    price: Optional[float] = None
+    currency: Optional[str] = None
+    availability: Optional[str] = None
+    rating: Optional[float] = None
+    review_count: Optional[int] = None
+    images: Optional[list[str]] = None
+    bullet_points: Optional[list[str]] = None
+    url: str
+    model_config = {"extra": "allow"}
+
+class AmazonSearchResponse(BaseModel):
+    request_id: str
+    results: list[AmazonProductResponse]
+
+
+# ---- Walmart ----
+
+class WalmartProductResponse(BaseModel):
+    provider: str
+    product_id: str
+    title: str
+    brand: Optional[str] = None
+    price: Optional[float] = None
+    availability: Optional[str] = None
+    url: str
+    model_config = {"extra": "allow"}
+
+class WalmartSearchResponse(BaseModel):
+    request_id: str
+    results: list[WalmartProductResponse]
+
+
+# ---- YouTube ----
+
+class YouTubeVideoResponse(BaseModel):
+    request_id: str
+    video: dict[str, Any]
+
+class YouTubeSearchResponse(BaseModel):
+    request_id: str
+    results: list[Any]
+
+class YouTubeSubtitleResponse(BaseModel):
+    request_id: str
+    video_id: str
+    subtitles: list[Any]

scrapio_py-1.0.0/tests/test_integration.py

@@ -0,0 +1,177 @@
+"""
+SDK integration tests — require a running API stack.
+
+Set WDA_API_KEY and WDA_BASE_URL, then run:
+    WDA_API_KEY=sk-test WDA_BASE_URL=http://localhost:3000 pytest tests/
+"""
+
+import os
+import pytest
+import pytest_asyncio
+
+API_KEY = os.environ.get("WDA_API_KEY", "")
+BASE_URL = os.environ.get("WDA_BASE_URL", "http://localhost:3000")
+
+skip_if_no_key = pytest.mark.skipif(not API_KEY, reason="WDA_API_KEY not set")
+
+
+# ---- Sync client tests ----
+
+@skip_if_no_key
+class TestSyncFetch:
+    def setup_method(self):
+        from webdataapi import ApiClient
+        self.client = ApiClient(api_key=API_KEY, base_url=BASE_URL)
+
+    def teardown_method(self):
+        self.client.close()
+
+    def test_fetch_returns_markdown(self):
+        from webdataapi import FetchRequest
+        res = self.client.fetch.fetch(FetchRequest(url="https://example.com", output=["markdown"]))
+        assert res.request_id
+        assert res.status_code == 200
+        assert "markdown" in res.outputs
+
+
+@skip_if_no_key
+class TestSyncJobs:
+    def setup_method(self):
+        from webdataapi import ApiClient
+        self.client = ApiClient(api_key=API_KEY, base_url=BASE_URL)
+
+    def teardown_method(self):
+        self.client.close()
+
+    def test_create_and_poll_job(self):
+        from webdataapi import CreateJobRequest
+        job = self.client.jobs.create(
+            CreateJobRequest(job_type="fetch", payload={"url": "https://example.com", "output": ["markdown"]})
+        )
+        assert job.job_id
+        assert job.status in ("queued", "running", "completed", "partial")
+
+        polled = self.client.jobs.get(job.job_id)
+        assert polled.job_id == job.job_id
+
+    def test_wait_for_completion(self):
+        from webdataapi import CreateJobRequest
+        job = self.client.jobs.create(
+            CreateJobRequest(job_type="fetch", payload={"url": "https://example.com", "output": ["markdown"]})
+        )
+        result = self.client.jobs.wait_for_completion(job.job_id, poll_interval=1.0, timeout=60.0)
+        assert result.status in ("completed", "partial", "failed", "cancelled")
+
+
+@skip_if_no_key
+class TestSyncGoogle:
+    def setup_method(self):
+        from webdataapi import ApiClient
+        self.client = ApiClient(api_key=API_KEY, base_url=BASE_URL)
+
+    def teardown_method(self):
+        self.client.close()
+
+    def test_search_returns_results(self):
+        from webdataapi import GoogleSearchParams
+        res = self.client.google.search(GoogleSearchParams(search="python web scraping"))
+        assert res.request_id
+        assert isinstance(res.results, list)
+
+
+@skip_if_no_key
+class TestSyncAmazon:
+    def setup_method(self):
+        from webdataapi import ApiClient
+        self.client = ApiClient(api_key=API_KEY, base_url=BASE_URL)
+
+    def teardown_method(self):
+        self.client.close()
+
+    def test_search_amazon(self):
+        res = self.client.amazon.search("laptop")
+        assert res.request_id
+        assert isinstance(res.results, list)
+
+
+@skip_if_no_key
+class TestSyncWalmart:
+    def setup_method(self):
+        from webdataapi import ApiClient
+        self.client = ApiClient(api_key=API_KEY, base_url=BASE_URL)
+
+    def teardown_method(self):
+        self.client.close()
+
+    def test_search_walmart(self):
+        res = self.client.walmart.search("headphones")
+        assert res.request_id
+        assert isinstance(res.results, list)
+
+
+@skip_if_no_key
+class TestSyncYouTube:
+    def setup_method(self):
+        from webdataapi import ApiClient
+        self.client = ApiClient(api_key=API_KEY, base_url=BASE_URL)
+
+    def teardown_method(self):
+        self.client.close()
+
+    def test_search_youtube(self):
+        res = self.client.youtube.search("python tutorial")
+        assert res.request_id
+        assert isinstance(res.results, list)
+
+
+# ---- Error types (sync) ----
+
+class TestSyncErrors:
+    def test_auth_error_on_bad_key(self):
+        from webdataapi import ApiClient, AuthError, FetchRequest
+        client = ApiClient(api_key="invalid", base_url=BASE_URL)
+        with pytest.raises(AuthError) as exc_info:
+            client.fetch.fetch(FetchRequest(url="https://example.com"))
+        assert exc_info.value.status_code == 401
+
+    def test_credits_exhausted_error_is_catchable(self):
+        from webdataapi import CreditsExhaustedError
+        body = {"request_id": "r1", "error": {"code": "credits_exhausted", "message": "no credits"}}
+        err = CreditsExhaustedError(body)
+        assert err.status_code == 402
+        assert isinstance(err, CreditsExhaustedError)
+
+    def test_rate_limit_error_is_catchable(self):
+        from webdataapi import RateLimitError
+        body = {"request_id": "r2", "error": {"code": "rate_limited", "message": "slow down"}}
+        err = RateLimitError(body)
+        assert err.status_code == 429
+        assert isinstance(err, RateLimitError)
+
+
+# ---- Async client tests ----
+
+@skip_if_no_key
+@pytest.mark.asyncio
+class TestAsyncFetch:
+    async def test_fetch_returns_markdown(self):
+        from webdataapi import AsyncApiClient, FetchRequest
+        async with AsyncApiClient(api_key=API_KEY, base_url=BASE_URL) as client:
+            res = await client.fetch.fetch(FetchRequest(url="https://example.com", output=["markdown"]))
+            assert res.request_id
+            assert res.status_code == 200
+
+    async def test_wait_for_completion_async(self):
+        from webdataapi import AsyncApiClient, CreateJobRequest
+        async with AsyncApiClient(api_key=API_KEY, base_url=BASE_URL) as client:
+            job = await client.jobs.create(
+                CreateJobRequest(job_type="fetch", payload={"url": "https://example.com", "output": ["markdown"]})
+            )
+            result = await client.jobs.wait_for_completion(job.job_id, poll_interval=1.0, timeout=60.0)
+            assert result.status in ("completed", "partial", "failed", "cancelled")
+
+    async def test_auth_error_async(self):
+        from webdataapi import AsyncApiClient, AuthError, FetchRequest
+        async with AsyncApiClient(api_key="invalid", base_url=BASE_URL) as client:
+            with pytest.raises(AuthError):
+                await client.fetch.fetch(FetchRequest(url="https://example.com"))