fleetpull 0.1.0__py3-none-any.whl

fleetpull/orchestrator/backfill.py

@@ -0,0 +1,127 @@
+# src/fleetpull/orchestrator/backfill.py
+"""Window decomposition: a windowed run's range into per-chunk work units.
+
+The caller-side planning the work-unit store defers to its driver (DESIGN §5).
+Every windowed run -- the daily pull and a long backfill alike -- tiles its
+(provider, endpoint) window into whole-UTC-day chunks, one unit per chunk (a
+window smaller than one chunk is a single unit). A unit fans the whole roster
+at execution, so it carries no member -- one planner serves fan-out and
+non-fan-out watermark endpoints alike, the fan-out distinction being the
+driver's, not the unit's. Pure functions only: the runner resolves the window,
+calls these to plan, and drives the queue.
+"""
+
+from datetime import datetime, timedelta
+
+from fleetpull.incremental import DateWindow
+from fleetpull.state import WorkUnitSpec
+from fleetpull.vocabulary import Provider
+
+__all__: list[str] = ['plan_backfill_units']
+
+
+def _is_utc_midnight(value: datetime) -> bool:
+    """True when ``value`` is exactly midnight UTC -- a whole-UTC-day boundary.
+
+    A date-partition boundary: a zero UTC offset and a zeroed time of day. Naive
+    and non-UTC datetimes are rejected (``utcoffset`` is ``None`` or nonzero), so
+    this also gates the timezone validity ``DateWindow`` does not.
+    """
+    return (
+        value.utcoffset() == timedelta(0)
+        and value.hour == 0
+        and value.minute == 0
+        and value.second == 0
+        and value.microsecond == 0
+    )
+
+
+def _date_chunks(span: DateWindow, chunk: timedelta) -> list[tuple[datetime, datetime]]:
+    """Tile a half-open span into contiguous whole-UTC-day chunks.
+
+    Splits ``[span.start, span.end)`` into half-open chunks of ``chunk`` width,
+    left to right; the final chunk runs to ``span.end`` and so may be shorter
+    (but still a whole number of days). A chunk is emitted only while the cursor
+    is strictly before ``span.end``, so no zero-width chunk is produced and every
+    bound pair satisfies the work-unit store's ``chunk_start < chunk_end``.
+
+    Both span bounds must be midnight UTC and ``chunk`` a positive whole number of
+    days, so every chunk bound lands on midnight UTC. The date-partitioned
+    watermark writer replaces whole date partitions, so a chunk boundary mid-day
+    would drive partial-day replacement and silently corrupt the partitions it
+    touches -- hence the guards rather than a permissive ``timedelta``.
+
+    Args:
+        span: The half-open range to tile; both bounds midnight UTC.
+        chunk: The width of each chunk; a positive whole number of days.
+
+    Returns:
+        The chunk bounds in order, each ``(chunk_start, chunk_end)`` on midnight
+        UTC with ``chunk_start < chunk_end``; contiguous (each chunk's end is the
+        next's start), the first at ``span.start`` and the last at ``span.end``.
+
+    Raises:
+        ValueError: When ``chunk`` is not a positive whole number of days, or a
+            span bound is not midnight UTC -- caller bugs, kept stdlib.
+    """
+    if chunk <= timedelta(0) or chunk % timedelta(days=1) != timedelta(0):
+        raise ValueError(
+            f'backfill chunk must be a positive whole number of days: {chunk!r}'
+        )
+    if not _is_utc_midnight(span.start) or not _is_utc_midnight(span.end):
+        raise ValueError(
+            'backfill chunks require whole-UTC-day span bounds (midnight UTC): '
+            f'start={span.start!r}, end={span.end!r}'
+        )
+    chunks: list[tuple[datetime, datetime]] = []
+    start = span.start
+    while start < span.end:
+        end = min(start + chunk, span.end)
+        chunks.append((start, end))
+        start = end
+    return chunks
+
+
+def plan_backfill_units(
+    provider: Provider,
+    endpoint: str,
+    span: DateWindow,
+    chunk: timedelta,
+) -> list[WorkUnitSpec]:
+    """Decompose a backfill into one work unit per whole-UTC-day chunk.
+
+    The caller-side decomposition the work-unit store leaves to its driver
+    (DESIGN §5): tile the span into whole-UTC-day chunks, one unit per chunk. A
+    unit fans the whole roster at execution, so it carries no member
+    (``partition_key=None``); the unit loop drives each chunk through the
+    endpoint's already-resolved driver. So one planner serves fan-out and
+    non-fan-out watermark endpoints alike; the fan-out distinction is the
+    driver's, not the unit's. Pure: it returns the specs; enqueuing them
+    idempotently is the caller's, kept separate so the plan can be inspected and
+    the enqueue stays the store's one write.
+
+    Args:
+        provider: The provider being backfilled.
+        endpoint: The endpoint being backfilled.
+        span: The full backfill range, half-open and midnight-UTC on both bounds
+            (the runner's residual-window resolution builds it from the resume
+            precedence against the trailing edge).
+        chunk: The width of each date chunk; a positive whole number of days.
+
+    Returns:
+        One ``WorkUnitSpec`` per chunk, in chronological order, ready to enqueue.
+
+    Raises:
+        ValueError: When the span bounds are not midnight UTC or ``chunk`` is not a
+            positive whole number of days (from :func:`_date_chunks`).
+    """
+    return [
+        WorkUnitSpec(
+            provider=provider,
+            endpoint=endpoint,
+            partition_key=None,
+            chunk_start=chunk_start,
+            chunk_end=chunk_end,
+        )
+        for chunk_start, chunk_end in _date_chunks(span, chunk)
+    ]

fleetpull/orchestrator/batch.py

@@ -0,0 +1,155 @@
+# src/fleetpull/orchestrator/batch.py
+"""Per-batch transform: validate and frame one record batch, and (watermark only)
+guard, window-filter, and fold its event time.
+
+``process_batch`` is the shared per-batch step both run-executor arms drive: the
+snapshot arm passes ``context=None`` (validate + frame only); the watermark arm
+passes a ``WindowContext`` to additionally apply the future-event guard, filter the
+frame to the resume window, and produce the batch's in-window fold candidate.
+``combine_latest_event_time`` folds those candidates across batches. Pure
+transforms -- the caller writes the frame and commits the watermark (DESIGN §14).
+"""
+
+from dataclasses import dataclass
+from datetime import datetime
+
+import polars as pl
+
+from fleetpull.endpoints.shared import EndpointDefinition
+from fleetpull.exceptions import ProviderResponseError
+from fleetpull.incremental import DateWindow
+from fleetpull.model_contract import ResponseModel
+from fleetpull.records import latest_event_time, models_to_dataframe, validate_records
+from fleetpull.storage import in_window
+from fleetpull.vocabulary import JsonObject
+
+__all__: list[str] = [
+    'ProcessedBatch',
+    'WindowContext',
+    'combine_latest_event_time',
+    'process_batch',
+]
+
+
+@dataclass(frozen=True, slots=True)
+class WindowContext:
+    """The watermark arm's per-batch context: window, clock, and event column.
+
+    Present only on the watermark path; ``process_batch`` takes
+    ``WindowContext | None`` and treats ``None`` as the snapshot path (no
+    guard, no window filter, no fold candidate).
+
+    Attributes:
+        window: The resolved half-open ``[start, end)`` resume window.
+        now: The run's clock instant, for the future-event guard.
+        event_time_column: The response model's UTC datetime field the window
+            filter and fold read.
+    """
+
+    window: DateWindow
+    now: datetime
+    event_time_column: str
+
+
+@dataclass(frozen=True, slots=True)
+class ProcessedBatch:
+    """One processed batch: the frame to write and its fold candidate.
+
+    Attributes:
+        frame: The frame to hand to ``writer.write`` -- the validated, framed
+            batch on the snapshot path; that frame filtered to the resume
+            window on the watermark path. Its ``height`` is the run's per-batch
+            row count (one row per validated model; the ledger counts the rows
+            written for the window).
+        latest_event_time: The maximum in-window event time in this batch, or
+            ``None`` (snapshot path, or an empty/all-filtered batch). The
+            cross-batch fold input.
+    """
+
+    frame: pl.DataFrame
+    latest_event_time: datetime | None
+
+
+def process_batch(
+    batch: list[JsonObject],
+    definition: EndpointDefinition[ResponseModel],
+    context: WindowContext | None,
+) -> ProcessedBatch:
+    """Validate, frame, and (watermark only) guard-and-window one batch.
+
+    The shared per-batch transform both runner arms drive. Snapshot path
+    (``context is None``): validate the raw records against the response model
+    and frame them; the frame is written as-is and carries no fold candidate.
+    Watermark path: additionally apply the future-event guard to the raw frame,
+    filter the frame to the resume window, and fold the in-window maximum event
+    time.
+
+    The guard runs on the *raw* frame, before the window filter: the window's
+    end is at or before ``now`` (the trailing edge is held back), so a
+    future-dated record falls outside the window and the filter would silently
+    drop it -- guarding the raw frame surfaces the anomaly instead. The fold
+    uses the *filtered* frame: an event time past ``window.end`` would otherwise
+    advance the watermark past the trailing edge and skip the next run's cutoff
+    holdback.
+
+    Args:
+        batch: One batch of raw response records from the driver.
+        definition: The endpoint binding (response model for validation).
+        context: The watermark per-batch context, or ``None`` for snapshot.
+
+    Returns:
+        The frame to write and its fold candidate.
+
+    Raises:
+        ProviderResponseError: A raw event time exceeds ``context.now`` -- a
+            contract violation (watermark path only). Validation and framing
+            errors propagate from ``validate_records`` / ``models_to_dataframe``
+            unchanged.
+
+    Side Effects:
+        None -- pure transform; the caller writes the frame.
+    """
+    models = validate_records(batch, definition.response_model)
+    frame = models_to_dataframe(models, definition.response_model)
+    if context is None:
+        return ProcessedBatch(frame=frame, latest_event_time=None)
+    observed_raw = latest_event_time(frame, context.event_time_column)
+    if observed_raw is not None and observed_raw > context.now:
+        raise ProviderResponseError(
+            provider=definition.provider.value,
+            endpoint=definition.name,
+            detail=(
+                f'observed event time {observed_raw.isoformat()} is after the '
+                f'run clock {context.now.isoformat()}'
+            ),
+        )
+    in_scope = frame.filter(in_window(context.event_time_column, context.window))
+    return ProcessedBatch(
+        frame=in_scope,
+        latest_event_time=latest_event_time(in_scope, context.event_time_column),
+    )
+
+
+def combine_latest_event_time(
+    running: datetime | None, candidate: datetime | None
+) -> datetime | None:
+    """Fold a batch's in-window max into the running watermark candidate.
+
+    None-tolerant: an empty or all-filtered batch contributes ``None`` and
+    leaves the running maximum unchanged.
+
+    Args:
+        running: The accumulated maximum so far, or ``None``.
+        candidate: This batch's in-window maximum, or ``None``.
+
+    Returns:
+        The greater of the two, or whichever is non-``None``, or ``None``.
+
+    Side Effects:
+        None.
+    """
+    if running is None:
+        return candidate
+    if candidate is None:
+        return running
+    return max(running, candidate)

fleetpull/orchestrator/bisection.py

@@ -0,0 +1,271 @@
+# src/fleetpull/orchestrator/bisection.py
+"""The bisecting request driver: complete fetches from capped, unsortable Gets.
+
+The third ``RequestDriver`` (the request-cardinality seam,
+``orchestrator/drivers.py``): for endpoints declaring the
+``BisectedWindowFetch`` request shape, the unit's resume window is
+fetched whole; a
+response of exactly the declared ``results_limit`` is the overflow
+signal — the page is discarded, the window halves at a whole-second
+midpoint, and both halves recurse left-to-right; a floor-width window
+still returning a full page raises loudly (the data is denser than
+windowed fetching can enumerate — the provider's feed transport is the
+escape for such streams). Overflow is a return-type condition (page
+length), never an exception.
+
+Fetch grain thereby decouples from write grain: work units and the
+delete-by-window merge stay whole-window while only the wire requests
+narrow. Every sub-request rides ``client.fetch_pages`` — one limiter
+token per attempt, exactly like any page walk — and a mid-recursion
+crash re-claims the whole unit, idempotent under delete-by-window
+(no bisection state is ever persisted).
+
+Under overlap-matched retrieval a record straddling an internal split
+boundary is returned by both neighboring leaves, so each emitted page
+is filtered to the records ANCHORED in its own leaf window (the
+binding's ``event_time_wire_key``): leaves partition the unit, every
+record has exactly one owning leaf, and write-time dedup stays hygiene
+rather than a correctness mechanism. Midpoints are computed at whole
+seconds because fractional-second search bounds are unprobed on the
+wire.
+"""
+
+import logging
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+
+from fleetpull.endpoints.shared import (
+    BisectedWindowFetch,
+    EndpointDefinition,
+    ResumeValue,
+    require_date_window,
+)
+from fleetpull.exceptions import ProviderResponseError
+from fleetpull.incremental import DateWindow
+from fleetpull.model_contract import ResponseModel
+from fleetpull.network.client import FetchedPage, TransportClient
+from fleetpull.vocabulary import JsonObject
+
+__all__: list[str] = ['BisectingWindowDriver']
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(slots=True)
+class _BisectionTally:
+    """Mutable leaf/overflow counters threaded through one unit's recursion.
+
+    Narration state only -- the fetch and filter semantics never read it.
+    ``record_batches`` folds it into the unit-level INFO line so an
+    operator sees that (and how hard) bisection engaged, without a log
+    line per leaf.
+
+    Attributes:
+        leaves: Non-overflowing windows fetched and yielded.
+        overflows: Windows that returned exactly ``results_limit`` records
+            and were halved.
+    """
+
+    leaves: int = 0
+    overflows: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class BisectingWindowDriver:
+    """Fetch a windowed endpoint completely by adaptive window bisection.
+
+    Attributes:
+        shape: The endpoint's declared ``BisectedWindowFetch`` facts — the
+            overflow threshold, the floor width, and the wire key that
+            anchors each record to its one owning leaf window.
+    """
+
+    shape: BisectedWindowFetch
+
+    def record_batches(
+        self,
+        definition: EndpointDefinition[ResponseModel],
+        client: TransportClient,
+        resume: ResumeValue,
+    ) -> Iterator[FetchedPage]:
+        """Yield one ownership-filtered batch per non-overflowing leaf window.
+
+        Args:
+            definition: The endpoint being run (its ``spec_builder``,
+                ``page_decoder``, and ``quota_scope``).
+            client: The transport client for this endpoint's provider.
+            resume: The run's resume window. Must be a ``DateWindow`` — a
+                watermark endpoint always resumes from one; any other
+                value is a wiring bug.
+
+        Yields:
+            One ``FetchedPage`` per leaf window, left-to-right, each
+            holding only the records anchored in that leaf.
+
+        Raises:
+            TypeError: ``resume`` is not a ``DateWindow``.
+            ProviderResponseError: A floor-width window still returned a
+                full page (the loud no-narrower failure), or a record
+                arrived without a parseable anchor timestamp.
+        """
+        window = require_date_window(resume, type(self).__name__)
+        tally = _BisectionTally()
+        yield from self._drive_window(definition, client, window, tally)
+        if tally.overflows:
+            logger.info(
+                'bisection complete: provider=%s endpoint=%s window_start=%s '
+                'window_end=%s leaves=%d overflows=%d',
+                definition.provider.value,
+                definition.name,
+                window.start.isoformat(),
+                window.end.isoformat(),
+                tally.leaves,
+                tally.overflows,
+            )
+
+    def _drive_window(
+        self,
+        definition: EndpointDefinition[ResponseModel],
+        client: TransportClient,
+        window: DateWindow,
+        tally: _BisectionTally,
+    ) -> Iterator[FetchedPage]:
+        """Fetch one window; recurse on overflow, yield the leaf otherwise.
+
+        Args:
+            definition: The endpoint being run.
+            client: The transport client for this endpoint's provider.
+            window: The window to fetch — the unit's resume window at the
+                top of the recursion, a half of a parent below it.
+            tally: The recursion's shared leaf/overflow counters, folded
+                into the unit-level narration by ``record_batches``.
+
+        Yields:
+            The leaf batches under this window, left-to-right.
+
+        Raises:
+            ProviderResponseError: Per ``record_batches``.
+        """
+        spec = definition.spec_builder.build_spec(resume=window, member_values={})
+        pages = list(
+            client.fetch_pages(
+                spec, definition.page_decoder, definition.quota_scope.value
+            )
+        )
+        # The endpoint's decoder is single-page (terminal on the first
+        # page), so the chain is exactly one page.
+        records = [record for page in pages for record in page.records]
+        if len(records) < self.shape.results_limit:
+            tally.leaves += 1
+            yield FetchedPage(
+                records=self._anchored_in(records, window, definition),
+                durable_progress=None,
+            )
+            return
+        width = window.end - window.start
+        if width <= self.shape.floor:
+            raise ProviderResponseError(
+                provider=definition.provider.value,
+                endpoint=definition.name,
+                detail=(
+                    f'a {width} window starting {window.start.isoformat()} '
+                    f'still returned {len(records)} records — the window '
+                    f'cannot be narrowed under the provider record cap. '
+                    f'The stream is denser than windowed fetching can '
+                    f'enumerate; the provider feed transport is the '
+                    f'escape for this endpoint at this density.'
+                ),
+            )
+        tally.overflows += 1
+        logger.debug(
+            'window overflowed: provider=%s endpoint=%s window_start=%s '
+            'window_end=%s records=%d; halving',
+            definition.provider.value,
+            definition.name,
+            window.start.isoformat(),
+            window.end.isoformat(),
+            len(records),
+        )
+        midpoint = _whole_second_midpoint(window.start, window.end)
+        yield from self._drive_window(
+            definition, client, DateWindow(start=window.start, end=midpoint), tally
+        )
+        yield from self._drive_window(
+            definition, client, DateWindow(start=midpoint, end=window.end), tally
+        )
+
+    def _anchored_in(
+        self,
+        records: list[JsonObject],
+        window: DateWindow,
+        definition: EndpointDefinition[ResponseModel],
+    ) -> list[JsonObject]:
+        """Keep the records whose anchor timestamp falls in the window.
+
+        Under overlap-matched retrieval a straddler of an internal split
+        boundary is fetched by both neighboring leaves; anchoring gives it
+        exactly one owner. Records anchored outside the unit's whole
+        window (overlap edge returns) drop here too — the same records
+        the runner's window filter would drop after modeling.
+
+        Args:
+            records: The leaf page's raw records.
+            window: The leaf window that owns anchored records.
+            definition: The endpoint being run, for the error detail.
+
+        Returns:
+            The records anchored in ``[window.start, window.end)``.
+
+        Raises:
+            ProviderResponseError: A record's anchor key is missing or
+                unparseable — the routing anchor is load-bearing, so a
+                record without one fails loudly rather than being
+                silently kept or dropped.
+        """
+        wire_key = self.shape.event_time_wire_key
+        anchored: list[JsonObject] = []
+        for record in records:
+            raw_value = record.get(wire_key)
+            if not isinstance(raw_value, str):
+                raise ProviderResponseError(
+                    provider=definition.provider.value,
+                    endpoint=definition.name,
+                    detail=(
+                        f'record is missing the anchor timestamp '
+                        f'{wire_key!r} bisection routes by.'
+                    ),
+                )
+            try:
+                anchor = datetime.fromisoformat(raw_value)
+            except ValueError as error:
+                raise ProviderResponseError(
+                    provider=definition.provider.value,
+                    endpoint=definition.name,
+                    detail=(
+                        f'unparseable anchor timestamp {raw_value!r} under '
+                        f'{wire_key!r}.'
+                    ),
+                ) from error
+            if window.start <= anchor < window.end:
+                anchored.append(record)
+        return anchored
+
+
+def _whole_second_midpoint(start: datetime, end: datetime) -> datetime:
+    """The window's midpoint, floored to a whole second.
+
+    Fractional-second search bounds are unprobed on the wire, so splits
+    stay second-granular; the halves differ by at most one second, which
+    the recursion absorbs.
+
+    Args:
+        start: The window's inclusive start.
+        end: The window's exclusive end.
+
+    Returns:
+        The floored midpoint, strictly between ``start`` and ``end`` for
+        any window wider than one second.
+    """
+    half_seconds = int((end - start).total_seconds() // 2)
+    return start + timedelta(seconds=half_seconds)