ballotdata 1.0.0__py3-none-any.whl

ballotdata/_transport.py

@@ -0,0 +1,170 @@
+"""The HTTP layer: one attempt at a time, retries, timeouts and error mapping."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from collections.abc import Mapping
+from typing import Any, Literal
+
+import httpx
+
+from ._base import BaseSettings, backoff, encode_query, retryable_status
+from .errors import APIConnectionError, APIError, APITimeoutError, error_from_response, parse_retry_after
+
+Method = Literal["GET", "POST", "PUT", "DELETE"]
+
+
+class RequestOptions:
+    """Per-call overrides, taken by every method as keyword arguments."""
+
+    __slots__ = ("extra_headers", "max_retries", "timeout")
+
+    def __init__(
+        self,
+        timeout: float | None = None,
+        max_retries: int | None = None,
+        extra_headers: Mapping[str, str] | None = None,
+    ) -> None:
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.extra_headers = extra_headers
+
+
+def _wrap_transport_error(err: httpx.TransportError) -> APIConnectionError:
+    if isinstance(err, httpx.TimeoutException):
+        return APITimeoutError(f"No response from BallotData in time: {err}")
+    return APIConnectionError(f"Could not reach BallotData: {err}")
+
+
+def _decode(response: httpx.Response) -> Any:
+    if response.status_code == 204 or not response.content:
+        return None
+    return response.json()
+
+
+class _Common(BaseSettings):
+    def build(
+        self,
+        method: Method,
+        path: str,
+        query: Mapping[str, Any] | None,
+        body: Any,
+        options: RequestOptions,
+        timeout: float,
+    ) -> dict[str, Any]:
+        extra: dict[str, str] = {}
+        content = None
+        if body is not None:
+            extra["Content-Type"] = "application/json"
+            content = json.dumps(body).encode()
+        if options.extra_headers:
+            extra.update(options.extra_headers)
+        return {
+            "method": method,
+            "url": self.url(path),
+            "params": encode_query(query),
+            "headers": self.headers(extra),
+            "content": content,
+            "timeout": timeout,
+        }
+
+    def outcome(self, method: Method, response: httpx.Response) -> tuple[bool, float | None, APIError]:
+        """For an error response: (retryable, retry_after, error)."""
+        error = error_from_response(response)
+        retry_after = parse_retry_after(response.headers.get("retry-after"))
+        return retryable_status(response.status_code, method, retry_after), retry_after, error
+
+
+class SyncTransport(_Common):
+    """Sends requests with an ``httpx.Client``. Shared by every resource of one client."""
+
+    def __init__(self, *, http_client: httpx.Client | None, **settings: Any) -> None:
+        super().__init__(**settings)
+        self._own_client = http_client is None
+        self.client = http_client or httpx.Client(follow_redirects=True)
+
+    def close(self) -> None:
+        if self._own_client:
+            self.client.close()
+
+    def request(
+        self,
+        method: Method,
+        path: str,
+        *,
+        query: Mapping[str, Any] | None = None,
+        body: Any = None,
+        options: RequestOptions | None = None,
+    ) -> Any:
+        """Make a request, retrying where it's safe, and return the parsed JSON (None for 204)."""
+        options = options or RequestOptions()
+        max_retries = self.max_retries if options.max_retries is None else options.max_retries
+        timeout = self.timeout if options.timeout is None else options.timeout
+        args = self.build(method, path, query, body, options, timeout)
+        attempt = 0
+        while True:
+            try:
+                response = self.client.request(**args)
+            except httpx.TransportError as err:
+                # A request that never got a full answer may or may not have been processed: only retry idempotent ones.
+                if method == "POST" or attempt >= max_retries:
+                    raise _wrap_transport_error(err) from err
+                time.sleep(backoff(attempt))
+                attempt += 1
+                continue
+            self.observe(response)
+            if response.is_success:
+                return _decode(response)
+            retryable, retry_after, error = self.outcome(method, response)
+            if not retryable or attempt >= max_retries:
+                raise error
+            time.sleep(retry_after if retry_after is not None else backoff(attempt))
+            attempt += 1
+
+
+class AsyncTransport(_Common):
+    """Sends requests with an ``httpx.AsyncClient``. Shared by every resource of one client."""
+
+    def __init__(self, *, http_client: httpx.AsyncClient | None, **settings: Any) -> None:
+        super().__init__(**settings)
+        self._own_client = http_client is None
+        self.client = http_client or httpx.AsyncClient(follow_redirects=True)
+
+    async def close(self) -> None:
+        if self._own_client:
+            await self.client.aclose()
+
+    async def request(
+        self,
+        method: Method,
+        path: str,
+        *,
+        query: Mapping[str, Any] | None = None,
+        body: Any = None,
+        options: RequestOptions | None = None,
+    ) -> Any:
+        """Make a request, retrying where it's safe, and return the parsed JSON (None for 204)."""
+        options = options or RequestOptions()
+        max_retries = self.max_retries if options.max_retries is None else options.max_retries
+        timeout = self.timeout if options.timeout is None else options.timeout
+        args = self.build(method, path, query, body, options, timeout)
+        attempt = 0
+        while True:
+            try:
+                response = await self.client.request(**args)
+            except httpx.TransportError as err:
+                if method == "POST" or attempt >= max_retries:
+                    raise _wrap_transport_error(err) from err
+                await asyncio.sleep(backoff(attempt))
+                attempt += 1
+                continue
+            self.observe(response)
+            if response.is_success:
+                return _decode(response)
+            retryable, retry_after, error = self.outcome(method, response)
+            if not retryable or attempt >= max_retries:
+                raise error
+            await asyncio.sleep(retry_after if retry_after is not None else backoff(attempt))
+            attempt += 1

ballotdata/_version.py

@@ -0,0 +1 @@
+__version__ = "1.0.0"

ballotdata/errors.py

@@ -0,0 +1,207 @@
+"""Exceptions raised by the client.
+
+Every error the API answers with becomes an :class:`APIError` (or one of its
+subclasses, chosen by HTTP status), carrying the API's ``code``, its message
+and the request id. Problems reaching the API at all become
+:class:`APIConnectionError` or :class:`APITimeoutError`::
+
+    from ballotdata import APIError, NotFoundError
+
+    try:
+        client.contests.get(4410)
+    except NotFoundError:
+        ...
+    except APIError as err:
+        if err.code == "outside_purchase":
+            ...
+        raise
+"""
+
+from __future__ import annotations
+
+import email.utils
+import time
+from collections.abc import Mapping
+
+import httpx
+
+__all__ = [
+    "BallotDataError",
+    "APIError",
+    "BadRequestError",
+    "AuthenticationError",
+    "PermissionDeniedError",
+    "NotFoundError",
+    "ConflictError",
+    "CursorExpiredError",
+    "PayloadTooLargeError",
+    "RateLimitError",
+    "InternalServerError",
+    "APIConnectionError",
+    "APITimeoutError",
+]
+
+
+class BallotDataError(Exception):
+    """Base class for everything this library raises on purpose."""
+
+
+class APIError(BallotDataError):
+    """The API answered with an error: ``{"error": {"code", "message", "request_id"}}``."""
+
+    status: int
+    """HTTP status."""
+    code: str
+    """Machine-readable code, e.g. ``"outside_purchase"``. See :data:`ballotdata.types.ErrorCode`."""
+    message: str
+    """What went wrong, in English."""
+    request_id: str | None
+    """Request id from the error body or the X-Request-Id header. Quote it when you write to BallotData."""
+    headers: httpx.Headers
+    """The response headers."""
+
+    def __init__(
+        self,
+        status: int,
+        code: str,
+        message: str,
+        request_id: str | None = None,
+        headers: Mapping[str, str] | httpx.Headers | None = None,
+    ) -> None:
+        super().__init__(message)
+        self.status = status
+        self.code = code
+        self.message = message
+        self.request_id = request_id
+        self.headers = httpx.Headers(headers or {})
+
+    def __str__(self) -> str:
+        suffix = f" (request {self.request_id})" if self.request_id else ""
+        return f"{self.status} {self.code}: {self.message}{suffix}"
+
+    def __repr__(self) -> str:
+        return (
+            f"{type(self).__name__}(status={self.status!r}, code={self.code!r}, message={self.message!r}, request_id={self.request_id!r})"
+        )
+
+
+class BadRequestError(APIError):
+    """400: a parameter is missing or not valid (``invalid_parameter``, ``invalid_cursor``, ``invalid_filter``, ``invalid_json``, …)."""
+
+
+class AuthenticationError(APIError):
+    """401: no API key, or one that isn't valid, was revoked or expired."""
+
+
+class PermissionDeniedError(APIError):
+    """403: the key can't be used for this.
+
+    A missing scope (``insufficient_scope``), an IP allowlist (``ip_not_allowed``),
+    an account that isn't active, or data the account hasn't bought (``no_access``,
+    ``outside_purchase``).
+    """
+
+
+class NotFoundError(APIError):
+    """404: nothing has that id or slug (``not_found``, ``subscription_not_found``)."""
+
+
+class ConflictError(APIError):
+    """409: a subscription with that name already exists."""
+
+
+class CursorExpiredError(APIError):
+    """410 ``cursor_expired``: the position is older than the stream keeps.
+
+    Catch up with ``client.endorsements.list()``, then start again with ``from_="latest"``.
+    """
+
+
+class PayloadTooLargeError(APIError):
+    """413: the request body is too large."""
+
+
+class RateLimitError(APIError):
+    """429: over a rate limit (``rate_limit_exceeded``, ``daily_limit_exceeded``) or holding too many streams (``too_many_streams``)."""
+
+    @property
+    def retry_after(self) -> float | None:
+        """Seconds the API asked you to wait (the Retry-After header), or None."""
+        return parse_retry_after(self.headers.get("retry-after"))
+
+
+class InternalServerError(APIError):
+    """500 or above: something went wrong at BallotData. 503 ``warming_up`` and ``server_busy`` are retried for you first."""
+
+
+class APIConnectionError(BallotDataError):
+    """The API couldn't be reached: DNS, TLS, a refused or dropped connection."""
+
+
+class APITimeoutError(APIConnectionError):
+    """The API didn't answer within the ``timeout``."""
+
+
+def error_for(
+    status: int,
+    code: str,
+    message: str,
+    request_id: str | None = None,
+    headers: Mapping[str, str] | httpx.Headers | None = None,
+) -> APIError:
+    """Build the right :class:`APIError` subclass for a status."""
+    cls: type[APIError]
+    if status == 400:
+        cls = BadRequestError
+    elif status == 401:
+        cls = AuthenticationError
+    elif status == 403:
+        cls = PermissionDeniedError
+    elif status == 404:
+        cls = NotFoundError
+    elif status == 409:
+        cls = ConflictError
+    elif status == 410 or code == "cursor_expired":
+        cls = CursorExpiredError
+    elif status == 413:
+        cls = PayloadTooLargeError
+    elif status == 429:
+        cls = RateLimitError
+    elif status >= 500:
+        cls = InternalServerError
+    else:
+        cls = APIError
+    return cls(status, code, message, request_id, headers)
+
+
+def error_from_response(response: httpx.Response) -> APIError:
+    """Turn an error response (with its body read) into an :class:`APIError`."""
+    header_id = response.headers.get("x-request-id")
+    code = "http_error"
+    message = response.reason_phrase or f"HTTP {response.status_code}"
+    request_id = header_id
+    try:
+        body = response.json()
+    except ValueError:
+        body = None  # Not JSON (a proxy's error page, say): keep the status text.
+    error = body.get("error") if isinstance(body, dict) else None
+    if isinstance(error, dict) and error.get("code"):
+        code = str(error["code"])
+        message = str(error.get("message") or message)
+        request_id = error.get("request_id") or header_id
+    return error_for(response.status_code, code, message, request_id, response.headers)
+
+
+def parse_retry_after(value: str | None) -> float | None:
+    """Seconds from a Retry-After header (seconds or an HTTP date), or None."""
+    if not value:
+        return None
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        pass
+    try:
+        when = email.utils.parsedate_to_datetime(value)
+    except (TypeError, ValueError):
+        return None
+    return max(0.0, when.timestamp() - time.time())

ballotdata/helpers.py

@@ -0,0 +1,88 @@
+"""Small helpers for working with ballot data."""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Iterable, Sequence
+from typing import Any, TypedDict, TypeVar
+
+from .types import (
+    DateString,
+    Endorsement,
+    EndorsementContestRef,
+    EndorsementElectionRef,
+    EndorsementEndorserRef,
+    MeasureAnswer,
+)
+
+__all__ = ["ContestEndorsements", "EndorsementPick", "EndorsementPosition", "group_by_contest", "sort_picks"]
+
+P = TypeVar("P", bound=Any)
+
+
+class EndorsementPick(TypedDict):
+    """One choice an endorser picked, from :func:`group_by_contest`."""
+
+    endorsement_id: int
+    """The endorsement row: ``client.endorsements.get(id)`` has its statement and source."""
+    choice_id: int
+    label: str
+    measure_answer: MeasureAnswer | None
+    rank: int | None
+    """Ranked choice: 1 for first choice, 2 for second, …; None otherwise."""
+    published_on: DateString | None
+
+
+class EndorsementPosition(TypedDict):
+    """One endorser's picks in a contest, from :func:`group_by_contest`."""
+
+    endorser: EndorsementEndorserRef
+    picks: list[EndorsementPick]
+
+
+class ContestEndorsements(TypedDict):
+    """A contest and every endorser's position in it, from :func:`group_by_contest`."""
+
+    contest: EndorsementContestRef
+    election: EndorsementElectionRef
+    positions: list[EndorsementPosition]
+
+
+def group_by_contest(endorsements: Iterable[Endorsement]) -> list[ContestEndorsements]:
+    """Group endorsement rows (from ``client.endorsements.list()``) by contest and
+    endorser: one position per endorser, with its picks ranked first-to-last for
+    ranked choice, like the positions ``client.contests.get()`` returns.
+
+    Example::
+
+        rows = client.endorsements.list(endorsers="sf-labor-council", elections=slug, limit=500).to_list()
+        for group in group_by_contest(rows):
+            print(group["contest"]["title"], [p["label"] for p in group["positions"][0]["picks"]])
+    """
+    contests: dict[int, tuple[Endorsement, dict[int, EndorsementPosition]]] = {}
+    for e in endorsements:
+        _first, by_endorser = contests.setdefault(e["contest"]["id"], (e, {}))
+        position = by_endorser.setdefault(e["endorser"]["id"], {"endorser": e["endorser"], "picks": []})
+        position["picks"].append(
+            {
+                "endorsement_id": e["id"],
+                "choice_id": e["choice"]["id"],
+                "label": e["choice"]["label"],
+                "measure_answer": e["choice"]["measure_answer"],
+                "rank": e["rank"],
+                "published_on": e["published_on"],
+            }
+        )
+    return [
+        {
+            "contest": first["contest"],
+            "election": first["election"],
+            "positions": [{"endorser": p["endorser"], "picks": sort_picks(p["picks"])} for p in by_endorser.values()],
+        }
+        for first, by_endorser in contests.values()
+    ]
+
+
+def sort_picks(picks: Sequence[P]) -> list[P]:
+    """Picks in ballot-marking order: by ``rank`` for ranked choice, otherwise as given."""
+    return sorted(picks, key=lambda p: math.inf if p.get("rank") is None else p["rank"])

ballotdata/pagination.py

@@ -0,0 +1,152 @@
+"""Paged lists.
+
+List endpoints answer a page at a time: ``{"data": [...], "next_cursor": "..."}``.
+
+With the sync client, a list method returns the first :class:`SyncPage`.
+Iterating it walks every item on every page, fetching pages as needed::
+
+    page = client.endorsements.list(states="CA")       # the first page
+    page.data, page.next_cursor
+    for endorsement in client.endorsements.list(states="CA"):  # every item
+        ...
+
+With the async client, a list method returns an :class:`AsyncPaginator`:
+await it for the first page, or iterate it with ``async for`` for every item::
+
+    page = await client.endorsements.list(states="CA")
+    async for endorsement in client.endorsements.list(states="CA"):
+        ...
+"""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterator
+from typing import Any, Generic, TypeVar
+
+T = TypeVar("T")
+
+__all__ = ["SyncPage", "AsyncPage", "AsyncPaginator"]
+
+
+class _Page(Generic[T]):
+    data: list[T]
+    """This page's items."""
+    next_cursor: str | None
+    """Pass as ``cursor`` for the next page; None on the last page."""
+
+    def __init__(self, body: Any) -> None:
+        body = body or {}
+        self.data = list(body.get("data") or [])
+        self.next_cursor = body.get("next_cursor") or None
+
+    def has_next_page(self) -> bool:
+        """Whether there is another page after this one."""
+        return self.next_cursor is not None
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(data=<{len(self.data)} items>, next_cursor={self.next_cursor!r})"
+
+
+class SyncPage(_Page[T]):
+    """One page of a list. Iterate it for every item across every page."""
+
+    def __init__(self, body: Any, fetch: Callable[[str], Any]) -> None:
+        super().__init__(body)
+        self._fetch = fetch
+
+    def next_page(self) -> SyncPage[T] | None:
+        """The next page, or None after the last."""
+        if self.next_cursor is None:
+            return None
+        return SyncPage(self._fetch(self.next_cursor), self._fetch)
+
+    def iter_pages(self) -> Iterator[SyncPage[T]]:
+        """This page and every page after it."""
+        page: SyncPage[T] | None = self
+        while page is not None:
+            yield page
+            page = page.next_page()
+
+    def __iter__(self) -> Iterator[T]:
+        """Every item on this page and every page after it."""
+        for page in self.iter_pages():
+            yield from page.data
+
+    def to_list(self, max: int | None = None) -> list[T]:
+        """Every item collected into a list. Mind large lists: pass ``max`` to stop after that many."""
+        out: list[T] = []
+        if max is not None and max <= 0:
+            return out
+        for item in self:
+            out.append(item)
+            if max is not None and len(out) >= max:
+                break
+        return out
+
+
+class AsyncPage(_Page[T]):
+    """One page of a list. Iterate it with ``async for`` for every item across every page."""
+
+    def __init__(self, body: Any, fetch: Callable[[str], Awaitable[Any]]) -> None:
+        super().__init__(body)
+        self._fetch = fetch
+
+    async def next_page(self) -> AsyncPage[T] | None:
+        """The next page, or None after the last."""
+        if self.next_cursor is None:
+            return None
+        return AsyncPage(await self._fetch(self.next_cursor), self._fetch)
+
+    async def iter_pages(self) -> AsyncIterator[AsyncPage[T]]:
+        """This page and every page after it."""
+        page: AsyncPage[T] | None = self
+        while page is not None:
+            yield page
+            page = await page.next_page()
+
+    async def __aiter__(self) -> AsyncIterator[T]:
+        """Every item on this page and every page after it."""
+        async for page in self.iter_pages():
+            for item in page.data:
+                yield item
+
+
+class AsyncPaginator(Generic[T]):
+    """What an async list method returns. Await it for the first page, or
+    iterate it with ``async for`` for every item on every page."""
+
+    def __init__(self, fetch: Callable[[str | None], Awaitable[Any]], start_cursor: str | None = None) -> None:
+        self._fetch = fetch
+        self._start = start_cursor
+        self._first: AsyncPage[T] | None = None
+
+    async def _first_page(self) -> AsyncPage[T]:
+        if self._first is None:
+            self._first = AsyncPage(await self._fetch(self._start), self._fetch)
+        return self._first
+
+    def __await__(self) -> Generator[Any, None, AsyncPage[T]]:
+        return self._first_page().__await__()
+
+    async def iter_pages(self) -> AsyncIterator[AsyncPage[T]]:
+        """Every page in turn, starting with the first."""
+        first = await self._first_page()
+        async for page in first.iter_pages():
+            yield page
+
+    async def __aiter__(self) -> AsyncIterator[T]:
+        """Every item on every page."""
+        async for page in self.iter_pages():
+            for item in page.data:
+                yield item
+
+    async def to_list(self, max: int | None = None) -> list[T]:
+        """Every item collected into a list. Mind large lists: pass ``max`` to stop after that many."""
+        out: list[T] = []
+        if max is not None and max <= 0:
+            return out
+        async for item in self:
+            out.append(item)
+            if max is not None and len(out) >= max:
+                break
+        return out