fleetpull 0.1.0__py3-none-any.whl

fleetpull/network/client/transport.py

@@ -0,0 +1,381 @@
+# src/fleetpull/network/client/transport.py
+"""The transport client: HTTP transport, retry, rate limiting, page decoding.
+
+Owns one pooled httpx.Client and runs the per-attempt pipeline and the page
+loop. Provider-blind, state-blind, storage-blind, name-blind: every provider
+difference is absorbed by the injected ProviderProfile (auth + classifier) and
+the per-endpoint page decoder. The client runs the decoder over each envelope
+to emit a FetchedPage of records plus an opaque durable_progress cursor; it
+interprets neither the records' field shapes nor the cursor.
+"""
+
+import json
+import logging
+from collections.abc import Iterator
+from dataclasses import replace
+from types import TracebackType
+from typing import Self
+
+import httpx
+
+from fleetpull.exceptions import (
+    AuthenticationError,
+    ProviderResponseError,
+    RetriesExhaustedError,
+)
+from fleetpull.network.client.page import FetchedPage
+from fleetpull.network.client.profile import ProviderProfile
+from fleetpull.network.client.runtime import ClientRuntime
+from fleetpull.network.contract import (
+    ClassifiedResponse,
+    DecodedPage,
+    PageDecoder,
+    RequestSpec,
+)
+from fleetpull.network.posture import new_http_client
+from fleetpull.network.retry import RetryDecision, decide_retry
+from fleetpull.vocabulary import JsonValue, ResponseCategory
+
+__all__: list[str] = ['TransportClient']
+
+logger = logging.getLogger(__name__)
+
+# The most of a non-JSON body a raised detail may carry: enough to identify
+# a proxy block page or an HTML landing page, never the whole document.
+_BODY_EXCERPT_LIMIT: int = 80
+
+
+def _safe_body_excerpt(body_text: str) -> str:
+    """Whitespace-collapsed head of a response body, safe for an error detail.
+
+    Args:
+        body_text: The raw response body.
+
+    Returns:
+        At most ``_BODY_EXCERPT_LIMIT`` characters, whitespace collapsed.
+    """
+    return ' '.join(body_text.split())[:_BODY_EXCERPT_LIMIT]
+
+
+class TransportClient:
+    """
+    Pulls one endpoint's pages over HTTP with retry and rate limiting.
+
+    Owns one pooled ``httpx.Client``, reused across every attempt and (since
+    httpx clients are thread-safe) across every concurrent ``fetch_pages``
+    call sharing this instance. Reentrant: ``fetch_pages`` keeps its entire
+    working set local, so many threads may run it on one client without
+    interference. Use as a context manager so the connection pool is closed.
+
+    Invariants this client must never violate:
+        - Every HTTP attempt consumes exactly one limiter token; every page
+          is an attempt; ``request_slot()`` wraps exactly one HTTP call,
+          never the page loop or the retry loop.
+        - ``auth.prepare`` runs OUTSIDE the limiter slot: a GeoTab session
+          refresh during prepare consumes the auth scope's token, not the
+          data endpoint's, and a failed prepare wastes no data-scope token.
+        - The limiter owns all rate-limit waiting. RATE_LIMITED penalizes the
+          scope; the next ``request_slot()`` waits it out. The client never
+          sleeps for a 429.
+        - Per-page consecutive failure counters reset every page by
+          construction (they are locals of ``_fetch_single_page``).
+    """
+
+    def __init__(self, profile: ProviderProfile, runtime: ClientRuntime) -> None:
+        """
+        Args:
+            profile: The per-provider auth strategy and classifier.
+            runtime: Process-global transport infrastructure shared across
+                every provider's client.
+
+        Side Effects:
+            Constructs one pooled ``httpx.Client`` held for this client's
+            lifetime; close it via the context-manager protocol.
+        """
+        self._profile: ProviderProfile = profile
+        self._runtime: ClientRuntime = runtime
+        self._http_client: httpx.Client = new_http_client(runtime.http_config)
+
+    def __enter__(self) -> Self:
+        """Return self; the pool is already open."""
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        traceback: TracebackType | None,
+    ) -> None:
+        """Close the connection pool. Exceptions propagate (returns None)."""
+        self._http_client.close()
+
+    def fetch_pages(
+        self,
+        spec: RequestSpec,
+        page_decoder: PageDecoder,
+        quota_scope: str,
+    ) -> Iterator[FetchedPage]:
+        """
+        Yield every page of one endpoint, page decoding transparent.
+
+        One visible loop, terminating when the decoder returns no next spec.
+        The decoder builds each next request and extracts each page's records;
+        the client only sends the request and emits the records.
+
+        Args:
+            spec: The endpoint definition's credential-less base request.
+            page_decoder: The endpoint's page decoder (per-endpoint, NOT
+                per-provider).
+            quota_scope: The endpoint's rate-limit scope key.
+
+        Yields:
+            One ``FetchedPage`` per page, in order, each carrying its records
+            and its ``durable_progress`` (including the terminal page).
+
+        Raises:
+            RetriesExhaustedError: A retryable category exhausted its budget
+                on a single page.
+            AuthenticationError: Credentials are unfixable, or a second
+                consecutive auth failure on one page.
+            ProviderResponseError: A FATAL response, a structurally
+                violating envelope raised by the page decoder, or a
+                SUCCESS-classified response whose body is not JSON.
+        """
+        sent: RequestSpec | None = page_decoder.first_request(spec)
+        while sent is not None:
+            envelope: JsonValue = self._fetch_single_page(sent, quota_scope)
+            decoded: DecodedPage = page_decoder.decode_page(sent, envelope)
+            yield FetchedPage(
+                records=decoded.records,
+                durable_progress=decoded.advance.durable_progress,
+            )
+            sent = decoded.advance.next_spec
+
+    def fetch_envelope(self, spec: RequestSpec, quota_scope: str) -> JsonValue:
+        """
+        Execute one request outside any page loop; return its envelope.
+
+        The single-request surface for non-paging calls that must still
+        ride the whole per-attempt pipeline -- auth prepare, one limiter
+        token per attempt, retry, classification (the completeness
+        guard's ``GetCountOf`` is the first consumer). Interpretation of
+        the envelope is the caller's; the client stays shape-blind.
+
+        Args:
+            spec: The credential-less request to execute.
+            quota_scope: The rate-limit scope key the attempt spends from.
+
+        Returns:
+            The parsed success envelope.
+
+        Raises:
+            RetriesExhaustedError: A retryable category exhausted its
+                budget.
+            AuthenticationError: Credentials are unfixable, or a second
+                consecutive auth failure.
+            ProviderResponseError: A FATAL response, or a
+                SUCCESS-classified response whose body is not JSON.
+        """
+        return self._fetch_single_page(spec, quota_scope)
+
+    def _fetch_single_page(self, sent: RequestSpec, quota_scope: str) -> JsonValue:
+        """
+        Drive the retry loop for ONE page; return its success envelope.
+
+        The three failure counters are locals — they re-zero on entry, which
+        IS the reset-on-success semantics: a budget is "N consecutive
+        failures of this category on this page", so a long healthy fetch
+        never accumulates toward exhaustion and a short fetch is not handed a
+        long fetch's budget. The match over ``ResponseCategory`` is
+        exhaustive; do not add ``case _``.
+
+        Args:
+            sent: The fully decorated spec for this page.
+            quota_scope: The endpoint's rate-limit scope key.
+
+        Returns:
+            The parsed success envelope for this page.
+
+        Raises:
+            RetriesExhaustedError, AuthenticationError, ProviderResponseError
+            as documented on ``fetch_pages``.
+        """
+        transient_failures: int = 0
+        rate_limited_failures: int = 0
+        auth_failures: int = 0
+        while True:
+            classified: ClassifiedResponse = self._attempt(sent, quota_scope)
+            match classified.category:
+                case ResponseCategory.SUCCESS:
+                    # _attempt guarantees parsed_body is the envelope on
+                    # SUCCESS; the guard narrows JsonValue | None and fires
+                    # only on a contract violation.
+                    envelope: JsonValue | None = classified.parsed_body
+                    if envelope is None:
+                        raise ProviderResponseError(
+                            detail='classifier reported SUCCESS with no parsed body'
+                        )
+                    return envelope
+                case ResponseCategory.TRANSIENT:
+                    transient_failures += 1
+                    transient_decision: RetryDecision = self._retry_or_exhaust(
+                        ResponseCategory.TRANSIENT, transient_failures
+                    )
+                    self._runtime.sleeper.sleep(transient_decision.local_delay_seconds)
+                case ResponseCategory.RATE_LIMITED:
+                    rate_limited_failures += 1
+                    self._retry_or_exhaust(
+                        ResponseCategory.RATE_LIMITED, rate_limited_failures
+                    )
+                    self._penalize_scope(quota_scope, classified)
+                case ResponseCategory.AUTH_FAILURE:
+                    auth_failures += 1
+                    # Sessions get exactly one refresh-and-retry per page; a
+                    # static key (on_auth_failure False) or a second failure
+                    # in a row is unfixable.
+                    if auth_failures > 1 or not self._profile.auth.on_auth_failure():
+                        raise AuthenticationError(detail=classified.detail)
+                case ResponseCategory.FATAL:
+                    raise ProviderResponseError(detail=classified.detail)
+
+    def _retry_or_exhaust(
+        self, category: ResponseCategory, failure_count: int
+    ) -> RetryDecision:
+        """
+        Decide one more retry for a retryable category, or raise.
+
+        Raising on exhaustion happens BEFORE the caller's category-specific
+        action (sleep or penalize), so the exhausted attempt pays no delay
+        and applies no penalty — pinned by the transport tests.
+
+        Args:
+            category: The retryable category being budgeted.
+            failure_count: Consecutive failures of this category on this
+                page, including the one just observed.
+
+        Returns:
+            The retry decision; the caller applies its category-specific
+            action.
+
+        Raises:
+            RetriesExhaustedError: The category's failure budget is spent.
+        """
+        decision: RetryDecision = decide_retry(
+            category,
+            failure_count,
+            self._runtime.retry_config,
+            self._runtime.random_source,
+        )
+        if not decision.should_retry:
+            raise RetriesExhaustedError(
+                category=category,
+                attempt_count=failure_count,
+            )
+        return decision
+
+    def _attempt(self, sent: RequestSpec, quota_scope: str) -> ClassifiedResponse:
+        """
+        Run one attempt: prepare (outside the slot), then send (inside it).
+
+        Returns one classification. prepare-time and send-time transport
+        failures both route through ``classify_transport_exception`` (always
+        TRANSIENT). Non-transport errors from the authenticator
+        (AuthenticationError, ProviderResponseError) are not caught and
+        propagate untouched. On SUCCESS the parsed envelope is guaranteed on
+        ``parsed_body``: a status-only classifier (Motive/Samsara) leaves it
+        None, so the body is parsed here; a classifier that already parsed
+        (GeoTab) is left untouched, so the body is parsed at most once.
+
+        Args:
+            sent: The fully decorated spec for this attempt.
+            quota_scope: The endpoint's rate-limit scope key.
+
+        Returns:
+            The classification of this attempt's response or transport failure.
+
+        Side Effects:
+            Consumes one limiter token for the data request; may drive a
+            network Authenticate inside ``prepare`` (GeoTab), which consumes
+            the auth scope's own token via the authenticator.
+        """
+        try:
+            prepared: RequestSpec = self._profile.auth.prepare(sent)
+        except httpx.TransportError as prepare_transport_error:
+            return self._profile.classifier.classify_transport_exception(
+                prepare_transport_error
+            )
+
+        with self._runtime.limiter_registry.get(quota_scope).request_slot():
+            try:
+                response: httpx.Response = self._http_client.request(
+                    method=prepared.method,
+                    url=prepared.url,
+                    headers=dict(prepared.headers),
+                    params=dict(prepared.params)
+                    if prepared.params is not None
+                    else None,
+                    json=prepared.json_body,
+                )
+            except httpx.TransportError as send_transport_error:
+                return self._profile.classifier.classify_transport_exception(
+                    send_transport_error
+                )
+
+        # Classification runs after the slot releases — it is CPU, not a
+        # request; the slot wrapped exactly the HTTP call.
+        body_text: str = response.text
+        classified: ClassifiedResponse = self._profile.classifier.classify_response(
+            response.status_code, response.headers, body_text
+        )
+        if (
+            classified.category is ResponseCategory.SUCCESS
+            and classified.parsed_body is None
+        ):
+            try:
+                parsed_envelope: JsonValue = json.loads(body_text)
+            except json.JSONDecodeError:
+                # A 200 serving non-JSON (a TLS-intercepting proxy's block
+                # page, a base_url pointing at a web page) is sustained, not
+                # transient: name it rather than burn the retry budget. The
+                # detail carries a sanitized excerpt only -- a block page may
+                # contain environment details that must never reach logs.
+                raise ProviderResponseError(
+                    detail=(
+                        f'classifier reported SUCCESS but the body is not '
+                        f'JSON (content-type '
+                        f'{response.headers.get("content-type", "")!r}): '
+                        f'{_safe_body_excerpt(body_text)!r}'
+                    )
+                ) from None
+            classified = replace(classified, parsed_body=parsed_envelope)
+        return classified
+
+    def _penalize_scope(self, quota_scope: str, classified: ClassifiedResponse) -> None:
+        """
+        Penalize the whole quota scope for a 429, clamping to the fallback.
+
+        ``penalize`` rejects non-positive seconds, so a missing or
+        non-positive parsed Retry-After clamps to ``fallback_penalty_seconds``
+        and the parsed value is logged for diagnosis.
+
+        Args:
+            quota_scope: The endpoint's rate-limit scope key.
+            classified: The RATE_LIMITED verdict carrying the parsed
+                Retry-After, when the provider sent a usable one.
+
+        Side Effects:
+            Extends the scope-wide pause; logs a WARNING when no usable
+            Retry-After was present.
+        """
+        retry_after_seconds: float | None = classified.retry_after_seconds
+        if retry_after_seconds is not None and retry_after_seconds > 0:
+            penalty_seconds: float = retry_after_seconds
+        else:
+            logger.warning(
+                'Rate-limited with no usable Retry-After (parsed=%r); applying '
+                'fallback penalty of %.2f seconds.',
+                retry_after_seconds,
+                self._runtime.retry_config.fallback_penalty_seconds,
+            )
+            penalty_seconds = self._runtime.retry_config.fallback_penalty_seconds
+        self._runtime.limiter_registry.get(quota_scope).penalize(penalty_seconds)

fleetpull/network/contract/__init__.py

@@ -0,0 +1,50 @@
+# src/fleetpull/network/contract/__init__.py
+"""The request/response contract: the shared vocabulary and protocols
+every network layer speaks. Provider implementations (classifiers,
+page decoders, auth strategies) live in sibling packages and import
+this surface through this face."""
+
+from fleetpull.network.contract.auth import AuthStrategy
+from fleetpull.network.contract.classifier import (
+    SERVER_ERROR_FLOOR,
+    SUCCESS_STATUS_RANGE,
+    ResponseClassifier,
+    body_snippet,
+    retry_after_seconds_from_headers,
+)
+from fleetpull.network.contract.envelope_fetcher import EnvelopeFetcher
+from fleetpull.network.contract.envelopes import (
+    StrictEnvelopeSlice,
+    require_child_object,
+    require_record_list,
+    unwrap_record_objects,
+    validated_envelope_slice,
+)
+from fleetpull.network.contract.outcome import ClassifiedResponse
+from fleetpull.network.contract.page_decoder import (
+    DecodedPage,
+    PageAdvance,
+    PageDecoder,
+)
+from fleetpull.network.contract.request import HttpMethod, RequestSpec
+
+__all__: list[str] = [
+    'SERVER_ERROR_FLOOR',
+    'SUCCESS_STATUS_RANGE',
+    'AuthStrategy',
+    'ClassifiedResponse',
+    'DecodedPage',
+    'EnvelopeFetcher',
+    'HttpMethod',
+    'PageAdvance',
+    'PageDecoder',
+    'RequestSpec',
+    'ResponseClassifier',
+    'StrictEnvelopeSlice',
+    'body_snippet',
+    'require_child_object',
+    'require_record_list',
+    'retry_after_seconds_from_headers',
+    'unwrap_record_objects',
+    'validated_envelope_slice',
+]

fleetpull/network/contract/auth.py

@@ -0,0 +1,48 @@
+# src/fleetpull/network/contract/auth.py
+"""The auth-strategy protocol: pure shape, the contract every strategy implements.
+
+Implementations live in ``network/auth/strategies.py`` (beside the
+``GeotabSessionManager`` that ``GeotabSessionAuth`` wraps); keeping them
+out of this surface module is what lets the contract face stay free of
+any ``network/auth`` dependency.
+"""
+
+from typing import Protocol
+
+from fleetpull.network.contract.request import RequestSpec
+
+__all__: list[str] = ['AuthStrategy']
+
+
+class AuthStrategy(Protocol):
+    """Provider-agnostic credential injection (DESIGN.md §8)."""
+
+    def prepare(self, spec: RequestSpec) -> RequestSpec:
+        """
+        Inject credentials and return a new spec.
+
+        Called fresh for EVERY HTTP attempt — every page, every retry
+        (the prepare-per-attempt contract, symmetric with
+        token-per-attempt). After a successful ``on_auth_failure``, the
+        retry must carry the fresh credentials, so re-preparing is
+        mandatory; implementations must keep the hot path cheap.
+
+        Args:
+            spec: The credential-less request description.
+
+        Returns:
+            A new spec carrying credentials; the input is unchanged.
+        """
+        ...
+
+    def on_auth_failure(self) -> bool:
+        """
+        Answer "did I change anything that makes one retry worthwhile?"
+
+        Performs the fix (if any) as a side effect.
+
+        Returns:
+            True when fresh credentials are now available and one retry
+            is worthwhile; False when nothing can be fixed by retrying.
+        """
+        ...

fleetpull/network/contract/classifier.py

@@ -0,0 +1,189 @@
+# src/fleetpull/network/contract/classifier.py
+"""ResponseClassifier ABC and the shared helpers every classifier uses.
+
+This module is the only one in the contract package allowed to import
+httpx: the shared transport-exception mapping must name real exception
+types, and the classifier is the transport boundary.
+"""
+
+import math
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+from typing import Final
+
+import httpx
+
+from fleetpull.network.contract.outcome import ClassifiedResponse
+from fleetpull.vocabulary import ResponseCategory
+
+__all__: list[str] = [
+    'SERVER_ERROR_FLOOR',
+    'SUCCESS_STATUS_RANGE',
+    'ResponseClassifier',
+    'body_snippet',
+    'retry_after_seconds_from_headers',
+]
+
+# Cap on body text carried into ClassifiedResponse.detail. Every
+# classifier truncates through body_snippet; no inline magic numbers.
+_BODY_SNIPPET_MAX_CHARS: Final[int] = 200
+
+# Generic HTTP status boundaries shared by all classifiers — protocol
+# semantics, not provider behavior, so they live in the base module.
+# These stay as a range and a bare int deliberately: the tempting
+# alternative, HTTPStatus(code).is_success / .is_server_error,
+# constructs the enum from an arbitrary code and raises ValueError on
+# nonstandard statuses (e.g. Cloudflare's 522). A classifier must
+# classify every status it meets, never crash on one. HTTPStatus
+# members are reserved for equality against specific well-known codes
+# in the provider classifiers; band membership is checked against
+# these constants.
+SUCCESS_STATUS_RANGE: Final[range] = range(200, 300)
+SERVER_ERROR_FLOOR: Final[int] = 500
+
+
+def _find_header(headers: Mapping[str, str], name: str) -> str | None:
+    """
+    Case-insensitive header lookup.
+
+    HTTP headers are case-insensitive (captures show ``retry-after``
+    lowercase from GeoTab), and this contract must not depend on
+    httpx's case-insensitive mapping being the one passed in.
+
+    Args:
+        headers: The response headers.
+        name: Header name, any casing.
+
+    Returns:
+        The header value, or None when absent.
+    """
+    normalized_name: str = name.lower()
+    for header_name, header_value in headers.items():
+        if header_name.lower() == normalized_name:
+            return header_value
+    return None
+
+
+def _parse_retry_after_seconds(value: str) -> float | None:
+    """
+    Parse the numeric-seconds form of a Retry-After value.
+
+    Samsara sends fractional seconds (e.g. ``0.40235``); GeoTab sends
+    integers (e.g. ``58``). The HTTP-date form is deliberately unparsed
+    in v1. The consumer is the limiter's ``penalize(seconds)``, which
+    raises on ``seconds <= 0`` — this helper must never hand it an
+    invalid value, so anything that is not a finite positive number
+    maps to None, which safely triggers the client's fallback penalty.
+
+    Args:
+        value: The raw header value.
+
+    Returns:
+        The positive, finite seconds value, or None for everything
+        else (non-numeric strings, HTTP dates, zero, negatives, NaN,
+        infinity).
+    """
+    try:
+        seconds: float = float(value)
+    except ValueError:
+        return None
+    if not math.isfinite(seconds) or seconds <= 0:
+        return None
+    return seconds
+
+
+def retry_after_seconds_from_headers(headers: Mapping[str, str]) -> float | None:
+    """
+    Extract and parse the ``Retry-After`` header's numeric-seconds form.
+
+    Composes the module's header-lookup and seconds-parsing helpers —
+    the composition is HTTP protocol semantics shared by every provider,
+    not provider behavior, so it lives in the base module.
+
+    Args:
+        headers: The response headers.
+
+    Returns:
+        The positive, finite seconds value, or None when the header is
+        absent or unusable (the client then applies its fallback penalty).
+    """
+    retry_after_value: str | None = _find_header(headers, 'Retry-After')
+    if retry_after_value is None:
+        return None
+    return _parse_retry_after_seconds(retry_after_value)
+
+
+def body_snippet(body_text: str) -> str:
+    """
+    Truncate body text for use in ``ClassifiedResponse.detail``.
+
+    Args:
+        body_text: The raw response body.
+
+    Returns:
+        The text unchanged when within the cap; otherwise the first
+        ``_BODY_SNIPPET_MAX_CHARS`` characters with a ``…`` marker.
+    """
+    if len(body_text) <= _BODY_SNIPPET_MAX_CHARS:
+        return body_text
+    return body_text[:_BODY_SNIPPET_MAX_CHARS] + '…'
+
+
+class ResponseClassifier(ABC):
+    """
+    Per-provider response classifier — the SOLE producer of the
+    classification vocabulary; the client only consumes it.
+
+    ``classify_response`` is abstract because provider envelopes differ
+    (GeoTab returns JSON-RPC errors inside HTTP 200, so a
+    status-code-only client cannot see failure). The transport-exception
+    mapping is concrete in the base because timeouts and connection
+    failures sit below any provider envelope and must not vary per
+    provider (Protocol-for-shape / ABC-for-substance, DESIGN.md §8).
+    """
+
+    @abstractmethod
+    def classify_response(
+        self,
+        status_code: int,
+        headers: Mapping[str, str],
+        body_text: str,
+    ) -> ClassifiedResponse:
+        """
+        Classify one provider response.
+
+        Args:
+            status_code: The HTTP status code.
+            headers: The response headers.
+            body_text: The raw response body.
+
+        Returns:
+            The classified outcome the client dispatches on.
+        """
+
+    def classify_transport_exception(self, exception: Exception) -> ClassifiedResponse:
+        """
+        Classify an exception raised by the transport, shared across
+        all providers.
+
+        Args:
+            exception: The exception the HTTP attempt raised.
+
+        Returns:
+            TRANSIENT for any ``httpx.TransportError`` (the hierarchy
+            makes timeouts and connect/read/write failures subclasses,
+            so one check covers them all), with the concrete exception
+            class name in ``detail``.
+
+        Raises:
+            Exception: Any non-transport exception is re-raised
+                untouched — a ValueError or KeyError reaching the
+                classifier is a programming error, and classifying it
+                would silence a bug.
+        """
+        if isinstance(exception, httpx.TransportError):
+            return ClassifiedResponse(
+                category=ResponseCategory.TRANSIENT,
+                detail=f'transport error: {type(exception).__name__}',
+            )
+        raise exception