xippios-core 0.1.0__py3-none-any.whl

xippios/core/__init__.py

@@ -0,0 +1,80 @@
+"""xippios-core: the shared foundation of the xippios Python SDK family.
+
+Only names re-exported here are public API. The optional adapters live in their own
+subpackages and are imported explicitly: ``xippios.core.mcp``, ``xippios.core.http``,
+``xippios.core.testing``.
+"""
+
+from xippios.core._loop import sync
+from xippios.core._version import __version__
+from xippios.core.client import (
+    AsyncClient,
+    BasicKeyAuth,
+    BearerAuth,
+    CacheBackend,
+    Client,
+    ClientConfig,
+    EnvKeyAuth,
+    HeaderKeyAuth,
+    MemoryCache,
+    QueryKeyAuth,
+    RateLimit,
+    RetryPolicy,
+    Timeouts,
+)
+from xippios.core.config import SourceSettings, resolve_api_key
+from xippios.core.errors import (
+    AuthError,
+    ConfigError,
+    InvalidRequestError,
+    NotFoundError,
+    RateLimitError,
+    TransportError,
+    UpstreamError,
+    ValidationError,
+    XippiosError,
+)
+from xippios.core.models import Result, ResultMeta, UpstreamModel, parse, parse_decimal
+from xippios.core.pagination import collect, paginate_cursor, paginate_offset, paginate_pages
+from xippios.core.reference import load_csv, load_json, load_table
+
+__all__ = [
+    "AsyncClient",
+    "AuthError",
+    "BasicKeyAuth",
+    "BearerAuth",
+    "CacheBackend",
+    "Client",
+    "ClientConfig",
+    "ConfigError",
+    "EnvKeyAuth",
+    "HeaderKeyAuth",
+    "InvalidRequestError",
+    "MemoryCache",
+    "NotFoundError",
+    "QueryKeyAuth",
+    "RateLimit",
+    "RateLimitError",
+    "Result",
+    "ResultMeta",
+    "RetryPolicy",
+    "SourceSettings",
+    "Timeouts",
+    "TransportError",
+    "UpstreamError",
+    "UpstreamModel",
+    "ValidationError",
+    "XippiosError",
+    "__version__",
+    "collect",
+    "load_csv",
+    "load_json",
+    "load_table",
+    "paginate_cursor",
+    "paginate_offset",
+    "paginate_pages",
+    "parse",
+    "parse_decimal",
+    "resolve_api_key",
+    "sync",
+]

xippios/core/_loop.py

@@ -0,0 +1,113 @@
+"""A background event loop so synchronous callers can run the async core (ADR-0002).
+
+One daemon thread owns one event loop for the whole process. Sync facades submit
+coroutines to it and block until they finish, which works from plain scripts, from
+Jupyter, and from inside another running loop. Nothing here is public API except
+through ``xippios.core.sync``.
+
+Not fork-safe: a process forked after the loop thread started (``multiprocessing`` with
+the ``fork`` start method) inherits a loop whose thread does not exist. Use the
+``spawn`` start method, or fork before the first sync call.
+"""
+
+import asyncio
+import functools
+import inspect
+import threading
+from collections.abc import Awaitable, Callable, Coroutine
+from typing import Any, Protocol, cast
+
+__all__ = ["SyncFunction", "as_async", "run", "sync"]
+
+_LOOP_THREAD_NAME = "xippios-core-loop"
+
+
+class SyncFunction[**P, T](Protocol):
+    """A synchronous function derived by ``sync`` from an async one."""
+
+    __name__: str
+    __qualname__: str
+    __doc__: str | None
+    __wrapped__: Callable[P, Coroutine[Any, Any, T]]
+    async_fn: Callable[P, Coroutine[Any, Any, T]]
+
+    def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: ...
+
+
+class _Runner:
+    def __init__(self) -> None:
+        self._loop: asyncio.AbstractEventLoop | None = None
+        self._thread: threading.Thread | None = None
+        self._lock = threading.Lock()
+
+    def loop(self) -> asyncio.AbstractEventLoop:
+        with self._lock:
+            if self._loop is None or self._loop.is_closed():
+                loop = asyncio.new_event_loop()
+                thread = threading.Thread(
+                    target=loop.run_forever, name=_LOOP_THREAD_NAME, daemon=True
+                )
+                thread.start()
+                self._loop, self._thread = loop, thread
+            return self._loop
+
+    def run[T](self, coro: Coroutine[Any, Any, T]) -> T:
+        loop = self.loop()
+        if threading.current_thread() is self._thread:
+            coro.close()
+            raise RuntimeError(
+                "a sync facade was called from inside the xippios event loop; "
+                "await the async function instead"
+            )
+        return asyncio.run_coroutine_threadsafe(coro, loop).result()
+
+
+_runner = _Runner()
+
+
+def run[T](coro: Coroutine[Any, Any, T]) -> T:
+    """Run ``coro`` to completion on the background loop and return its result."""
+    return _runner.run(coro)
+
+
+def sync[**P, T](fn: Callable[P, Coroutine[Any, Any, T]]) -> SyncFunction[P, T]:
+    """Derive a synchronous function from an async one.
+
+    The result has the same signature and docstring, runs the coroutine on the
+    background loop, and exposes the original as ``.async_fn`` so adapters can
+    await it natively instead of blocking their own loop. A trailing ``_async`` is
+    dropped from the name, so ``get_series = sync(get_series_async)`` is named
+    ``get_series`` wherever the name is shown, such as in MCP tool names. The loop
+    is not fork-safe; see the module docstring.
+
+    Args:
+        fn: An ``async def`` function.
+    """
+
+    @functools.wraps(fn)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+        return run(fn(*args, **kwargs))
+
+    wrapper.__name__ = fn.__name__.removesuffix("_async")
+    wrapper.__qualname__ = fn.__qualname__.removesuffix("_async")
+    wrapper.async_fn = fn  # type: ignore[attr-defined]
+    return cast(SyncFunction[P, T], wrapper)
+
+
+def as_async(fn: Callable[..., Any]) -> Callable[..., Awaitable[Any]]:
+    """Return an awaitable form of ``fn`` for use inside an adapter's event loop.
+
+    Prefers the ``async_fn`` twin set by ``sync``; returns ``fn`` itself when it is
+    already a coroutine function; otherwise wraps a plain function in a worker thread.
+    """
+    twin = getattr(fn, "async_fn", None)
+    if twin is not None and inspect.iscoroutinefunction(twin):
+        return cast(Callable[..., Awaitable[Any]], twin)
+    if inspect.iscoroutinefunction(fn):
+        return cast(Callable[..., Awaitable[Any]], fn)
+
+    @functools.wraps(fn)
+    async def wrapper(*args: Any, **kwargs: Any) -> Any:
+        return await asyncio.to_thread(fn, *args, **kwargs)
+
+    return wrapper

xippios/core/_version.py

@@ -0,0 +1,10 @@
+"""Package version, read from the installed distribution metadata."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("xippios-core")
+except PackageNotFoundError:  # pragma: no cover - only in an unbuilt checkout
+    __version__ = "0.0.0"
+
+__all__ = ["__version__"]

xippios/core/client/__init__.py

@@ -0,0 +1,119 @@
+"""HTTP client for upstream APIs (conventions §6): httpx, async core, sync facade."""
+
+from collections.abc import Mapping
+from types import TracebackType
+from typing import Any, Self
+
+import httpx
+
+from xippios.core._loop import run
+from xippios.core.client._core import AsyncClient, Params
+from xippios.core.client.auth import (
+    BasicKeyAuth,
+    BearerAuth,
+    EnvKeyAuth,
+    HeaderKeyAuth,
+    QueryKeyAuth,
+)
+from xippios.core.client.cache import CacheBackend, CachedResponse, MemoryCache, cache_key
+from xippios.core.client.config import ClientConfig, RateLimit, RetryPolicy, Timeouts
+from xippios.core.client.ratelimit import TokenBucket
+from xippios.core.client.retry import send_with_retries
+
+__all__ = [
+    "AsyncClient",
+    "BasicKeyAuth",
+    "BearerAuth",
+    "CacheBackend",
+    "CachedResponse",
+    "Client",
+    "ClientConfig",
+    "EnvKeyAuth",
+    "HeaderKeyAuth",
+    "MemoryCache",
+    "Params",
+    "QueryKeyAuth",
+    "RateLimit",
+    "RetryPolicy",
+    "Timeouts",
+    "TokenBucket",
+    "cache_key",
+    "send_with_retries",
+]
+
+
+class Client:
+    """Synchronous facade over ``AsyncClient`` (ADR-0002).
+
+    Every call runs on the family's background event loop, so it works from scripts,
+    notebooks and inside other event loops. ``async_client`` exposes the underlying
+    instance for code that wants to await natively.
+
+    Args:
+        config: Base URL, timeouts, retry policy, rate limit, cache and headers.
+        auth: An ``httpx.Auth`` adding credentials.
+        transport: Replacement transport, for tests.
+    """
+
+    def __init__(
+        self,
+        config: ClientConfig,
+        *,
+        auth: httpx.Auth | None = None,
+        transport: httpx.AsyncBaseTransport | None = None,
+    ) -> None:
+        self.async_client = AsyncClient(config, auth=auth, transport=transport)
+
+    @property
+    def config(self) -> ClientConfig:
+        """The configuration in use."""
+        return self.async_client.config
+
+    def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Params = None,
+        json: Any = None,
+        headers: Mapping[str, str] | None = None,
+        raise_on_error: bool = True,
+    ) -> httpx.Response:
+        """See ``AsyncClient.request``."""
+        return run(
+            self.async_client.request(
+                method,
+                path,
+                params=params,
+                json=json,
+                headers=headers,
+                raise_on_error=raise_on_error,
+            )
+        )
+
+    def get(
+        self, path: str, *, params: Params = None, headers: Mapping[str, str] | None = None
+    ) -> httpx.Response:
+        """See ``AsyncClient.get``."""
+        return run(self.async_client.get(path, params=params, headers=headers))
+
+    def get_json(
+        self, path: str, *, params: Params = None, headers: Mapping[str, str] | None = None
+    ) -> Any:
+        """See ``AsyncClient.get_json``."""
+        return run(self.async_client.get_json(path, params=params, headers=headers))
+
+    def close(self) -> None:
+        """Close the background loop's connection pool."""
+        run(self.async_client.aclose())
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.close()

xippios/core/client/_core.py

@@ -0,0 +1,188 @@
+"""The async client: one request path with retries, rate limiting, caching and errors."""
+
+import asyncio
+import threading
+import time
+import weakref
+from collections.abc import Awaitable, Callable, Mapping
+from types import TracebackType
+from typing import Any, Self
+
+import httpx
+
+from xippios.core.client.cache import CachedResponse, cache_key
+from xippios.core.client.config import ClientConfig
+from xippios.core.client.ratelimit import TokenBucket
+from xippios.core.client.retry import send_with_retries
+from xippios.core.errors import BODY_LIMIT, ValidationError, raise_for_status
+
+__all__ = ["AsyncClient", "Params"]
+
+Params = Mapping[str, Any] | None
+
+
+class AsyncClient:
+    """Async HTTP client for one upstream (conventions §6).
+
+    One instance can be shared by sync facades and async servers alike: it keeps one
+    ``httpx`` connection pool per event loop, while the rate limiter and the cache are
+    shared across all of them.
+
+    Args:
+        config: Base URL, timeouts, retry policy, rate limit, cache and headers.
+        auth: An ``httpx.Auth`` adding credentials, e.g. a query key or basic auth.
+        transport: Replacement transport, for tests.
+        sleep: Async sleep used by retries and the rate limiter, replaceable in tests.
+    """
+
+    def __init__(
+        self,
+        config: ClientConfig,
+        *,
+        auth: httpx.Auth | None = None,
+        transport: httpx.AsyncBaseTransport | None = None,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    ) -> None:
+        self.config = config
+        self._auth = auth
+        self._transport = transport
+        self._sleep = sleep
+        self._bucket = TokenBucket(config.rate_limit) if config.rate_limit is not None else None
+        self._pools: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, httpx.AsyncClient] = (
+            weakref.WeakKeyDictionary()
+        )
+        self._lock = threading.Lock()
+
+    @property
+    def source(self) -> str:
+        """Source slug from the configuration."""
+        return self.config.source
+
+    def _pool(self) -> httpx.AsyncClient:
+        loop = asyncio.get_running_loop()
+        with self._lock:
+            pool = self._pools.get(loop)
+            if pool is None:
+                pool = httpx.AsyncClient(
+                    base_url=self.config.base_url,
+                    headers={"User-Agent": self.config.user_agent, **self.config.headers},
+                    timeout=self.config.timeouts.to_httpx(),
+                    auth=self._auth,
+                    transport=self._transport,
+                )
+                self._pools[loop] = pool
+            return pool
+
+    def _cache_key(self, method: str, url: httpx.URL) -> str | None:
+        if method != "GET" or self.config.cache is None:
+            return None
+        return cache_key(method, url)
+
+    async def request(
+        self,
+        method: str,
+        path: str,
+        *,
+        params: Params = None,
+        json: Any = None,
+        headers: Mapping[str, str] | None = None,
+        raise_on_error: bool = True,
+    ) -> httpx.Response:
+        """Send one request and return the response.
+
+        Retries, rate limiting and caching apply as configured. The source's
+        ``classify`` hook, if any, sees the response first; otherwise a 4xx/5xx that
+        survives the retry policy is raised as the matching error from the taxonomy; a
+        transport failure as ``TransportError``.
+
+        Args:
+            method: HTTP method.
+            path: Path joined onto the base URL, e.g. ``"/api/api_GET"``.
+            params: Query parameters.
+            json: JSON body for write methods.
+            headers: Extra headers for this request only.
+            raise_on_error: ``False`` returns 4xx/5xx responses instead of raising, for
+                callers that record fixtures or inspect error bodies.
+        """
+        pool = self._pool()
+        method = method.upper()
+        probe = pool.build_request(method, path, params=params)
+        key = self._cache_key(method, probe.url)
+        if key is not None and self.config.cache is not None:
+            cached = self.config.cache.get(key)
+            if cached is not None:
+                return _from_cache(cached, probe)
+
+        async def send() -> httpx.Response:
+            if self._bucket is not None:
+                await self._bucket.acquire(self._sleep)
+            request = pool.build_request(method, path, params=params, json=json, headers=headers)
+            return await pool.send(request)
+
+        response = await send_with_retries(
+            send, policy=self.config.retry, method=method, source=self.source, sleep=self._sleep
+        )
+        if raise_on_error:
+            if self.config.classify is not None:
+                classified = self.config.classify(response)
+                if classified is not None:
+                    raise classified
+            raise_for_status(response, source=self.source)
+        if key is not None and self.config.cache is not None and response.status_code == 200:
+            self.config.cache.set(
+                key,
+                CachedResponse(
+                    status=response.status_code,
+                    headers=tuple(response.headers.multi_items()),
+                    content=response.content,
+                    stored_at=time.time(),
+                ),
+            )
+        return response
+
+    async def get(
+        self, path: str, *, params: Params = None, headers: Mapping[str, str] | None = None
+    ) -> httpx.Response:
+        """``request("GET", ...)``."""
+        return await self.request("GET", path, params=params, headers=headers)
+
+    async def get_json(
+        self, path: str, *, params: Params = None, headers: Mapping[str, str] | None = None
+    ) -> Any:
+        """GET and decode a JSON body, raising ``ValidationError`` when it is not JSON."""
+        response = await self.get(path, params=params, headers=headers)
+        try:
+            return response.json()
+        except ValueError as exc:
+            raise ValidationError(
+                f"{self.source}: response from {path} is not JSON",
+                raw=response.text[:BODY_LIMIT],
+                source=self.source,
+            ) from exc
+
+    async def aclose(self) -> None:
+        """Close the connection pool bound to the current event loop."""
+        loop = asyncio.get_running_loop()
+        with self._lock:
+            pool = self._pools.pop(loop, None)
+        if pool is not None:
+            await pool.aclose()
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.aclose()
+
+
+def _from_cache(cached: CachedResponse, request: httpx.Request) -> httpx.Response:
+    response = httpx.Response(
+        cached.status, headers=list(cached.headers), content=cached.content, request=request
+    )
+    response.extensions["xippios_cache"] = "hit"
+    return response

xippios/core/client/auth.py

@@ -0,0 +1,147 @@
+"""Authentication helpers that read the key from the environment on first use.
+
+Every upstream sends its key differently: USDA NASS as a query parameter, UN Comtrade
+in a header, CashCtrl as the HTTP basic user name. These ``httpx.Auth`` classes cover
+those shapes and resolve the key lazily through ``resolve_api_key``, so a package
+imports without credentials and fails with ``ConfigError`` on the first request
+instead of at import time.
+"""
+
+import base64
+import threading
+from collections.abc import Generator
+
+import httpx
+from pydantic import SecretStr
+
+from xippios.core.config import resolve_api_key
+
+__all__ = ["BasicKeyAuth", "BearerAuth", "EnvKeyAuth", "HeaderKeyAuth", "QueryKeyAuth"]
+
+
+class EnvKeyAuth(httpx.Auth):
+    """Base class: resolve the key once, then let a subclass attach it to each request.
+
+    Subclass and implement ``apply`` for a shape not covered below.
+
+    Args:
+        env_var: Variable that holds the key, named as the upstream documents it.
+        key: Explicit key; wins over the environment (conventions §7).
+        source: Source slug for the ``ConfigError`` message.
+    """
+
+    requires_request_body = False
+
+    def __init__(
+        self, env_var: str, *, key: str | SecretStr | None = None, source: str | None = None
+    ) -> None:
+        self.env_var = env_var
+        self.source = source
+        self._override = key
+        self._key: str | None = None
+        self._lock = threading.Lock()
+
+    def key(self) -> str:
+        """The key, resolved on first use and cached for the life of the object."""
+        if self._key is None:
+            with self._lock:
+                if self._key is None:
+                    secret = resolve_api_key(self.env_var, self._override, source=self.source)
+                    self._key = secret.get_secret_value()
+        return self._key
+
+    def apply(self, request: httpx.Request, key: str) -> None:
+        """Attach ``key`` to ``request``. Implemented by subclasses."""
+        raise NotImplementedError
+
+    def auth_flow(self, request: httpx.Request) -> Generator[httpx.Request, httpx.Response, None]:
+        """The httpx hook: one request, the key attached."""
+        self.apply(request, self.key())
+        yield request
+
+
+class QueryKeyAuth(EnvKeyAuth):
+    """Send the key as a query parameter, e.g. ``?key=...`` for USDA NASS.
+
+    Args:
+        param: Query parameter name.
+        env_var: Variable that holds the key.
+        key: Explicit key; wins over the environment.
+        source: Source slug for error messages.
+    """
+
+    def __init__(
+        self,
+        param: str,
+        env_var: str,
+        *,
+        key: str | SecretStr | None = None,
+        source: str | None = None,
+    ) -> None:
+        super().__init__(env_var, key=key, source=source)
+        self.param = param
+
+    def apply(self, request: httpx.Request, key: str) -> None:
+        """Merge the key into the query string."""
+        request.url = request.url.copy_merge_params({self.param: key})
+
+
+class HeaderKeyAuth(EnvKeyAuth):
+    """Send the key in a header, e.g. ``Ocp-Apim-Subscription-Key`` for UN Comtrade.
+
+    Args:
+        header: Header name.
+        env_var: Variable that holds the key.
+        key: Explicit key; wins over the environment.
+        source: Source slug for error messages.
+    """
+
+    def __init__(
+        self,
+        header: str,
+        env_var: str,
+        *,
+        key: str | SecretStr | None = None,
+        source: str | None = None,
+    ) -> None:
+        super().__init__(env_var, key=key, source=source)
+        self.header = header
+
+    def apply(self, request: httpx.Request, key: str) -> None:
+        """Set the header."""
+        request.headers[self.header] = key
+
+
+class BearerAuth(EnvKeyAuth):
+    """Send the key as ``Authorization: Bearer <key>``."""
+
+    def apply(self, request: httpx.Request, key: str) -> None:
+        """Set the Authorization header."""
+        request.headers["Authorization"] = f"Bearer {key}"
+
+
+class BasicKeyAuth(EnvKeyAuth):
+    """Send the key as the HTTP basic user name, as CashCtrl expects.
+
+    Args:
+        env_var: Variable that holds the key.
+        password: Basic-auth password; empty for key-as-user-name schemes.
+        key: Explicit key; wins over the environment.
+        source: Source slug for error messages.
+    """
+
+    def __init__(
+        self,
+        env_var: str,
+        *,
+        password: str = "",
+        key: str | SecretStr | None = None,
+        source: str | None = None,
+    ) -> None:
+        super().__init__(env_var, key=key, source=source)
+        self.password = password
+
+    def apply(self, request: httpx.Request, key: str) -> None:
+        """Set the Authorization header."""
+        token = base64.b64encode(f"{key}:{self.password}".encode()).decode("ascii")
+        request.headers["Authorization"] = f"Basic {token}"