okdata 0.1.0__py3-none-any.whl

okdata/__init__.py

@@ -0,0 +1,28 @@
+"""okdata: social data for AI agents, paid per call.
+
+    >>> from okdata import OkData
+    >>> ok = OkData()  # reads OKDATA_API_KEY
+    >>> page = ok.douyin.videos.search(query="露营装备", max_price="0.05")
+
+One typed method per endpoint (generated from the catalog), sync (OkData) and async (AsyncOkData),
+an automatic Idempotency-Key per call, retries driven by the error's ``retry`` field, and
+``pages()`` / ``items()`` that stop before ``max_spend``.
+"""
+from ._async_client import AsyncOkData
+from ._base import DEFAULT_BASE_URL, ENV_API_KEY, ENV_BASE_URL, endpoint
+from ._client import OkData
+from ._errors import OkDataConnectionError, OkDataError, RetryAdvice
+from ._generated._endpoints import CATALOG_REVISION, ENDPOINTS, PRICE_BOOK
+from ._generated._errors import ERROR_CODES, ERRORS
+from ._generated._models import *  # noqa: F401,F403  (Post, Account, Comment, ...)
+from ._paging import AsyncItemPager, AsyncPager, ItemPager, Pager, StopReason
+from ._types import Batch, Billing, CallWarning, EndpointInfo, Fields, Item, Meta, Missing, Page, Price, Result, View
+from ._version import __version__
+
+__all__ = [
+    "OkData", "AsyncOkData", "OkDataError", "OkDataConnectionError", "RetryAdvice",
+    "Page", "Item", "Batch", "Result", "Meta", "Billing", "CallWarning", "Missing", "EndpointInfo",
+    "View", "Fields", "Price", "Pager", "ItemPager", "AsyncPager", "AsyncItemPager", "StopReason",
+    "ENDPOINTS", "CATALOG_REVISION", "PRICE_BOOK", "ERROR_CODES", "ERRORS", "endpoint",
+    "DEFAULT_BASE_URL", "ENV_API_KEY", "ENV_BASE_URL", "__version__",
+]

okdata/_async_client.py

@@ -0,0 +1,119 @@
+"""AsyncOkData: the asyncio client, the same surface as OkData with ``await``."""
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Awaitable, Callable, Dict, Mapping, Optional, Sequence, Union
+
+import httpx
+
+from ._base import (
+    call_request, check_request_id, endpoint, make_config, merge_params, result_query, usage_query,
+)
+from ._generated._async import AsyncEndpoints
+from ._paging import AsyncItemPager, AsyncPager, _Budget
+from ._result import to_result
+from ._transport import Send, send_async
+from ._types import Price, Result, View
+
+
+class _AsyncCalls:
+    def __init__(self, ok: "AsyncOkData") -> None:
+        self._ok = ok
+
+    async def get(self, request_id: str) -> Dict[str, Any]:
+        return await self._ok.request("GET", f"/v1/calls/{check_request_id(request_id)}")
+
+    async def result(self, request_id: str, *, view: Optional[View] = None, fields: Optional[Union[str, Sequence[str]]] = None) -> Result:
+        body = await send_async(self._ok._cfg, self._ok._http, self._ok._sleep,
+                                Send("GET", f"/v1/calls/{check_request_id(request_id)}/result", result_query(view, fields)))
+        return to_result(body, None)
+
+    async def by_key(self, idempotency_key: str) -> Dict[str, Any]:
+        return await self._ok.request("GET", "/v1/calls", query=[("idempotency_key", idempotency_key)])
+
+
+class AsyncOkData(AsyncEndpoints):
+    """Async client for the okdata API.
+
+    >>> async with AsyncOkData() as ok:
+    ...     page = await ok.douyin.videos.search(query="露营装备")
+    ...     async for item in ok.items("douyin.videos.search", query="露营装备", max_spend="0.1"):
+    ...         ...
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: float = 65.0,
+        max_retries: int = 2,
+        retry_base_delay: float = 0.5,
+        max_retry_delay: float = 20.0,
+        entry: str = "sdk",
+        source: Optional[str] = None,
+        end_user: Optional[str] = None,
+        user_agent: Optional[str] = None,
+        http_client: Optional[httpx.AsyncClient] = None,
+        sleep: Optional[Callable[[float], Awaitable[None]]] = None,
+    ) -> None:
+        self._cfg = make_config(api_key, base_url, timeout, max_retries, retry_base_delay, max_retry_delay, entry, source, end_user, user_agent)
+        self.base_url = self._cfg.base_url
+        self._owns_http = http_client is None
+        self._http = http_client or httpx.AsyncClient()
+        self._sleep: Callable[[float], Awaitable[None]] = sleep or asyncio.sleep
+        self.calls = _AsyncCalls(self)
+        self._init_endpoints(self)
+
+    @property
+    def has_key(self) -> bool:
+        return self._cfg.api_key is not None
+
+    def __repr__(self) -> str:
+        return f"AsyncOkData(base_url={self.base_url!r}, key={'set' if self.has_key else 'missing'})"
+
+    async def __aenter__(self) -> "AsyncOkData":
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        if self._owns_http:
+            await self._http.aclose()
+
+    async def _call(self, endpoint_id: str, params: Mapping[str, Any], *, idempotency_key: Optional[str] = None, timeout: Optional[float] = None) -> Result:
+        req = call_request(endpoint_id, params, idempotency_key, timeout)
+        return to_result(await send_async(self._cfg, self._http, self._sleep, req), req.idempotency_key)
+
+    async def call(self, endpoint_id: str, params: Optional[Mapping[str, Any]] = None, *, idempotency_key: Optional[str] = None,
+                   timeout: Optional[float] = None, **kwargs: Any) -> Result:
+        return await self._call(endpoint_id, merge_params(params, kwargs), idempotency_key=idempotency_key, timeout=timeout)
+
+    def pages(self, endpoint_id: str, params: Optional[Mapping[str, Any]] = None, *, max_pages: Optional[int] = None,
+              max_spend: Optional[Price] = None, timeout: Optional[float] = None, **kwargs: Any) -> AsyncPager[Dict[str, Any]]:
+        """``async for page in ok.pages(...)``: see OkData.pages."""
+        budget = _Budget(endpoint_id, endpoint(endpoint_id), merge_params(params, kwargs), max_pages, max_spend)
+        return AsyncPager(budget, lambda p: self._call(endpoint_id, p, timeout=timeout))
+
+    def items(self, endpoint_id: str, params: Optional[Mapping[str, Any]] = None, *, max_items: Optional[int] = None,
+              max_pages: Optional[int] = None, max_spend: Optional[Price] = None, timeout: Optional[float] = None,
+              **kwargs: Any) -> AsyncItemPager[Dict[str, Any]]:
+        return AsyncItemPager(self.pages(endpoint_id, params, max_pages=max_pages, max_spend=max_spend, timeout=timeout, **kwargs), max_items)
+
+    async def account(self) -> Dict[str, Any]:
+        return await self.request("GET", "/v1/account")
+
+    async def catalog(self) -> Dict[str, Any]:
+        return await self.request("GET", "/v1/catalog", auth=False)
+
+    async def status(self) -> Dict[str, Any]:
+        return await self.request("GET", "/v1/catalog/status", auth=False)
+
+    async def usage(self, *, group_by: Optional[Sequence[str]] = None, since: Optional[str] = None, until: Optional[str] = None) -> Dict[str, Any]:
+        return await self.request("GET", "/v1/usage", query=usage_query(group_by, since, until))
+
+    async def request(self, method: str, path: str, *, query: Sequence[Any] = (), body: Optional[Dict[str, Any]] = None, auth: bool = True) -> Dict[str, Any]:
+        envelope = await send_async(self._cfg, self._http, self._sleep, Send(method, path, tuple(query), body, None, None, auth))
+        data = envelope.get("data")
+        return data if isinstance(data, dict) else {}

okdata/_base.py

@@ -0,0 +1,88 @@
+"""What OkData and AsyncOkData share: options, headers, and turning a call into a request."""
+from __future__ import annotations
+
+import os
+import re
+import uuid
+from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
+
+from ._generated._endpoints import ENDPOINTS
+from ._request import build_request, field_list
+from ._result import RetryPolicy
+from ._transport import Config, Send
+from ._types import EndpointInfo
+from ._version import __version__
+
+DEFAULT_BASE_URL = "https://api.okdata.online"
+ENV_API_KEY = "OKDATA_API_KEY"
+ENV_BASE_URL = "OKDATA_BASE_URL"
+_LABEL = re.compile(r"[\x21-\x7e]{1,128}")
+_KEY = re.compile(r"[\x21-\x7e]+")
+_REQUEST_ID = re.compile(r"[A-Za-z0-9-]{1,64}")
+
+
+def endpoint(endpoint_id: str) -> Optional[EndpointInfo]:
+    """The catalog entry this SDK was generated with, or None for an id it does not know."""
+    return ENDPOINTS.get(endpoint_id)
+
+
+def _label(value: Optional[str], name: str, most: int = 128) -> Optional[str]:
+    if value is None:
+        return None
+    if not _LABEL.fullmatch(value) or len(value) > most:
+        raise ValueError(f"{name} must be 1-{most} visible ASCII characters, no spaces")
+    return value
+
+
+def make_config(
+    api_key: Optional[str], base_url: Optional[str], timeout: float, max_retries: int, retry_base_delay: float,
+    max_retry_delay: float, entry: str, source: Optional[str], end_user: Optional[str], user_agent: Optional[str],
+) -> Config:
+    key = (api_key if api_key is not None else os.environ.get(ENV_API_KEY, "")).strip()
+    if key and not _KEY.fullmatch(key):
+        raise ValueError("the API key has spaces or control characters in it; copy it again (keys start with okd_)")
+    url = (base_url or os.environ.get(ENV_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
+    if not re.match(r"https?://[^/]+", url):
+        raise ValueError(f"base_url must start with https:// or http://, got {url!r}")
+    if isinstance(max_retries, bool) or not isinstance(max_retries, int) or max_retries < 0:
+        raise ValueError("max_retries must be a whole number, 0 or more")
+    headers = {"X-Entry": _label(entry, "entry", 32) or "sdk", "User-Agent": " ".join(filter(None, [f"okdata-sdk-python/{__version__}", user_agent]))}
+    for header, value, name in (("X-Source", source, "source"), ("X-End-User", end_user, "end_user")):
+        if _label(value, name) is not None:
+            headers[header] = value  # type: ignore[assignment]
+    return Config(url, key or None, timeout, RetryPolicy(max_retries, retry_base_delay, max_retry_delay), headers)
+
+
+def call_request(endpoint_id: str, params: Mapping[str, Any], idempotency_key: Optional[str], timeout: Optional[float]) -> Send:
+    req = build_request(endpoint_id, ENDPOINTS.get(endpoint_id), params)
+    return Send(req.method, req.path, tuple(req.query), req.body, idempotency_key or str(uuid.uuid4()), req.max_price, True, timeout)
+
+
+def merge_params(params: Optional[Mapping[str, Any]], kwargs: Mapping[str, Any]) -> Dict[str, Any]:
+    return {**(params or {}), **kwargs}
+
+
+def check_request_id(request_id: str) -> str:
+    if not _REQUEST_ID.fullmatch(request_id):
+        raise ValueError("request_id must be the request_id of an earlier call")
+    return request_id
+
+
+def result_query(view: Optional[str], fields: Optional[Union[str, Sequence[str]]]) -> Tuple[Tuple[str, str], ...]:
+    query: List[Tuple[str, str]] = []
+    if view is not None:
+        query.append(("view", view))
+    if fields is not None:
+        query.append(("fields", ",".join(field_list(fields))))
+    return tuple(query)
+
+
+def usage_query(group_by: Optional[Sequence[str]], since: Optional[str], until: Optional[str]) -> Tuple[Tuple[str, str], ...]:
+    query: List[Tuple[str, str]] = []
+    if group_by:
+        query.append(("group_by", ",".join(group_by)))
+    if since:
+        query.append(("since", since))
+    if until:
+        query.append(("until", until))
+    return tuple(query)

okdata/_client.py

@@ -0,0 +1,135 @@
+"""OkData: the synchronous client."""
+from __future__ import annotations
+
+import time
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Union
+
+import httpx
+
+from ._base import (
+    call_request, check_request_id, endpoint, make_config, merge_params, result_query, usage_query,
+)
+from ._generated._sync import SyncEndpoints
+from ._paging import ItemPager, Pager, _Budget
+from ._result import to_result
+from ._transport import Send, send_sync
+from ._types import Price, Result, View
+
+
+class _Calls:
+    """Earlier calls: their record, their kept result (free for 24 h), where an Idempotency-Key stands."""
+
+    def __init__(self, ok: "OkData") -> None:
+        self._ok = ok
+
+    def get(self, request_id: str) -> Dict[str, Any]:
+        return self._ok.request("GET", f"/v1/calls/{check_request_id(request_id)}")
+
+    def result(self, request_id: str, *, view: Optional[View] = None, fields: Optional[Union[str, Sequence[str]]] = None) -> Result:
+        body = send_sync(self._ok._cfg, self._ok._http, self._ok._sleep,
+                         Send("GET", f"/v1/calls/{check_request_id(request_id)}/result", result_query(view, fields)))
+        return to_result(body, None)
+
+    def by_key(self, idempotency_key: str) -> Dict[str, Any]:
+        return self._ok.request("GET", "/v1/calls", query=[("idempotency_key", idempotency_key)])
+
+
+class OkData(SyncEndpoints):
+    """Client for the okdata API.
+
+    >>> ok = OkData()  # reads OKDATA_API_KEY
+    >>> page = ok.douyin.videos.search(query="露营装备", max_price="0.05")
+    >>> page.items[0]["text"], page.cost
+
+    Every paid call carries an Idempotency-Key, reused by its retries, so a retry is never charged
+    twice. Failed calls are not charged.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: float = 65.0,
+        max_retries: int = 2,
+        retry_base_delay: float = 0.5,
+        max_retry_delay: float = 20.0,
+        entry: str = "sdk",
+        source: Optional[str] = None,
+        end_user: Optional[str] = None,
+        user_agent: Optional[str] = None,
+        http_client: Optional[httpx.Client] = None,
+        sleep: Optional[Callable[[float], None]] = None,
+    ) -> None:
+        self._cfg = make_config(api_key, base_url, timeout, max_retries, retry_base_delay, max_retry_delay, entry, source, end_user, user_agent)
+        self.base_url = self._cfg.base_url
+        self._owns_http = http_client is None
+        self._http = http_client or httpx.Client()
+        self._sleep: Callable[[float], None] = sleep or time.sleep
+        self.calls = _Calls(self)
+        self._init_endpoints(self)
+
+    @property
+    def has_key(self) -> bool:
+        """Whether a key is set; the key itself is never shown."""
+        return self._cfg.api_key is not None
+
+    def __repr__(self) -> str:
+        return f"OkData(base_url={self.base_url!r}, key={'set' if self.has_key else 'missing'})"
+
+    def __enter__(self) -> "OkData":
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        if self._owns_http:
+            self._http.close()
+
+    def _call(self, endpoint_id: str, params: Mapping[str, Any], *, idempotency_key: Optional[str] = None, timeout: Optional[float] = None) -> Result:
+        req = call_request(endpoint_id, params, idempotency_key, timeout)
+        return to_result(send_sync(self._cfg, self._http, self._sleep, req), req.idempotency_key)
+
+    def call(self, endpoint_id: str, params: Optional[Mapping[str, Any]] = None, *, idempotency_key: Optional[str] = None,
+             timeout: Optional[float] = None, **kwargs: Any) -> Result:
+        """Any endpoint by id: ``call("weibo.trends.list")``. Ids newer than this SDK work too (GET)."""
+        return self._call(endpoint_id, merge_params(params, kwargs), idempotency_key=idempotency_key, timeout=timeout)
+
+    def pages(self, endpoint_id: str, params: Optional[Mapping[str, Any]] = None, *, max_pages: Optional[int] = None,
+              max_spend: Optional[Price] = None, timeout: Optional[float] = None, **kwargs: Any) -> Pager[Dict[str, Any]]:
+        """Pages of a list endpoint, following next_cursor, within max_pages / max_spend (US dollars).
+
+        Before each page the pager adds what it could cost at most and stops if that would pass
+        max_spend; each call also carries X-Max-Price = what is left, so the server refuses dearer ones.
+        """
+        budget = _Budget(endpoint_id, endpoint(endpoint_id), merge_params(params, kwargs), max_pages, max_spend)
+        return Pager(budget, lambda p: self._call(endpoint_id, p, timeout=timeout))
+
+    def items(self, endpoint_id: str, params: Optional[Mapping[str, Any]] = None, *, max_items: Optional[int] = None,
+              max_pages: Optional[int] = None, max_spend: Optional[Price] = None, timeout: Optional[float] = None,
+              **kwargs: Any) -> ItemPager[Dict[str, Any]]:
+        """The items of a list endpoint across pages, within max_items / max_pages / max_spend."""
+        return ItemPager(self.pages(endpoint_id, params, max_pages=max_pages, max_spend=max_spend, timeout=timeout, **kwargs), max_items)
+
+    def account(self) -> Dict[str, Any]:
+        """Balance, this key with its spending limits, and the account's rate limits. Free."""
+        return self.request("GET", "/v1/account")
+
+    def catalog(self) -> Dict[str, Any]:
+        """Every endpoint with its parameters and price. Free, no key needed."""
+        return self.request("GET", "/v1/catalog", auth=False)
+
+    def status(self) -> Dict[str, Any]:
+        """Which endpoints are up right now. Free, no key needed."""
+        return self.request("GET", "/v1/catalog/status", auth=False)
+
+    def usage(self, *, group_by: Optional[Sequence[str]] = None, since: Optional[str] = None, until: Optional[str] = None) -> Dict[str, Any]:
+        """Calls, settled calls and charges grouped as asked (30 days by default). Free."""
+        return self.request("GET", "/v1/usage", query=usage_query(group_by, since, until))
+
+    def request(self, method: str, path: str, *, query: Sequence[Any] = (), body: Optional[Dict[str, Any]] = None, auth: bool = True) -> Dict[str, Any]:
+        """A raw request to any route; returns the envelope's data."""
+        envelope = send_sync(self._cfg, self._http, self._sleep, Send(method, path, tuple(query), body, None, None, auth))
+        data = envelope.get("data")
+        return data if isinstance(data, dict) else {}

okdata/_errors.py

@@ -0,0 +1,88 @@
+"""Errors the client raises. Every API error says whether it was charged and whether to retry;
+branch on ``code`` (and ``reason``), never on ``message``."""
+from __future__ import annotations
+
+from typing import Any, Dict, List, Literal, Mapping, Optional
+
+RetryAdvice = Literal["no", "yes", "once", "after_fix", "same_key"]
+"""no: fails the same way again. yes: retry. once: one more try. after_fix: change what problems says.
+same_key: resend with the same Idempotency-Key (never charged twice)."""
+
+RETRY_ADVICE = ("no", "yes", "once", "after_fix", "same_key")
+
+
+def _text(body: Mapping[str, Any], key: str) -> Optional[str]:
+    value = body.get(key)
+    return value if isinstance(value, str) else None
+
+
+class OkDataError(Exception):
+    """An error answer from the API, or a call that got none.
+
+    ``charged`` is False for almost every error, True for a not_found the endpoint charges (then
+    ``amount`` says how much), and None when it is not known (a lost answer, settlement_unknown):
+    look the call up with ``ok.calls.get(request_id)`` or ``ok.calls.by_key(idempotency_key)``.
+    """
+
+    def __init__(
+        self,
+        code: str,
+        message: str,
+        *,
+        status: Optional[int] = None,
+        request_id: Optional[str] = None,
+        charged: Optional[bool] = False,
+        retry: RetryAdvice = "no",
+        retry_after_seconds: Optional[float] = None,
+        idempotency_key: Optional[str] = None,
+        body: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        super().__init__(f"{code}: {message}")
+        data: Mapping[str, Any] = body or {}
+        self.code = code
+        self.message = message
+        self.status = status
+        self.request_id = request_id
+        self.charged = charged
+        self.retry: RetryAdvice = retry
+        self.retry_after_seconds = retry_after_seconds
+        self.idempotency_key = idempotency_key
+        self.body: Optional[Dict[str, Any]] = dict(body) if body is not None else None
+        self.message_zh = _text(data, "message_zh")
+        problems = data.get("problems")
+        self.problems: List[str] = [str(p) for p in problems] if isinstance(problems, list) else []
+        self.param = _text(data, "param")
+        self.reason = _text(data, "reason")
+        self.docs_url = _text(data, "docs_url")
+        self.amount = _text(data, "amount")
+        self.required = _text(data, "required")
+        self.available = _text(data, "available")
+        self.top_up_url = _text(data, "top_up_url")
+        self.next_step = _text(data, "next_step")
+        self.next_step_zh = _text(data, "next_step_zh")
+
+    def with_charged(self, charged: Optional[bool]) -> "OkDataError":
+        """The same error with another charge state."""
+        return OkDataError(
+            self.code, self.message, status=self.status, request_id=self.request_id, charged=charged,
+            retry=self.retry, retry_after_seconds=self.retry_after_seconds,
+            idempotency_key=self.idempotency_key, body=self.body,
+        )
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            **(self.body or {}),
+            "code": self.code, "message": self.message, "status": self.status, "request_id": self.request_id,
+            "charged": self.charged, "retry": self.retry, "retry_after_seconds": self.retry_after_seconds,
+            "idempotency_key": self.idempotency_key,
+        }
+
+    def __repr__(self) -> str:
+        return f"OkDataError(code={self.code!r}, status={self.status!r}, charged={self.charged!r}, retry={self.retry!r}, request_id={self.request_id!r})"
+
+
+class OkDataConnectionError(OkDataError):
+    """No answer came back (network failure or the client's own timeout); whether the call ran is unknown."""
+
+    def __init__(self, message: str, code: Literal["connection_error", "client_timeout"], idempotency_key: Optional[str]) -> None:
+        super().__init__(code, message, charged=None, retry="same_key", idempotency_key=idempotency_key)

okdata/_generated/__init__.py

@@ -0,0 +1 @@
+"""Generated by scripts/generate.ts from @okdata/catalog. Do not edit by hand."""