fleetpull 0.1.0__py3-none-any.whl

fleetpull/orchestrator/feed_drive.py

@@ -0,0 +1,266 @@
+# src/fleetpull/orchestrator/feed_drive.py
+"""The feed drive: the run executor's per-page version-token arm.
+
+``FeedDrive`` drives one feed endpoint's version-token stream (DESIGN
+sections 4/14, built 2026-07-21): resume is the stored ``FeedToken``, or a
+``FeedSeed`` at the sync-wide cold-start anchor when none is stored (the
+seed rides ONLY the tokenless first run -- I4); each page then commits
+independently in the per-page crash order parquet BEFORE token (I1/I2) --
+the append writer lands the page's rows durably, then the page's
+``toVersion`` commits through the store's kind-guarded feed write -- so a
+crash between the two refetches exactly one page on the next run and its
+rows land again as new appended rows, harmless under the stored-as-emitted
+contract. The drive consumes the page stream directly
+(``stream_processed_batches`` deliberately drops ``durable_progress`` and
+stays the non-feed pipe).
+"""
+
+import logging
+from collections.abc import Iterator
+
+from fleetpull.endpoints.shared import EndpointDefinition
+from fleetpull.exceptions import ConfigurationError
+from fleetpull.incremental import FeedResume, FeedSeed, FeedToken
+from fleetpull.model_contract import ResponseModel
+from fleetpull.network.client import FetchedPage
+from fleetpull.orchestrator.batch import process_batch
+from fleetpull.orchestrator.drivers import RequestDriver
+from fleetpull.orchestrator.outcome import Executed, RunOutcome
+from fleetpull.orchestrator.recording import recorded_run
+from fleetpull.orchestrator.resume import resolve_feed_resume
+from fleetpull.orchestrator.spine import RunnerSpine
+from fleetpull.orchestrator.streaming import BatchObserver
+from fleetpull.storage import DatasetWriter
+from fleetpull.timing import to_iso8601
+from fleetpull.vocabulary import Provider
+
+__all__: list[str] = ['FeedDrive']
+
+logger = logging.getLogger(__name__)
+
+
+def _feed_resume_label(resume: FeedResume) -> str:
+    """The ledger's ``from_version`` text for a feed run's resume value.
+
+    A resumed run records the token verbatim; a seeded run records the
+    self-describing ``seed:<iso8601>`` marker (the ``runs`` table requires a
+    non-null ``from_version`` on a feed row, and the seed date IS what the
+    run resumed from -- the convention is recorded on
+    ``RunLedger.start_feed_run``).
+
+    Args:
+        resume: The run's resolved feed resume value.
+
+    Returns:
+        The token, or ``seed:<iso8601 start>`` for a seeded run.
+    """
+    match resume:
+        case FeedToken(from_version=from_version):
+            return from_version
+        case FeedSeed(start=start):
+            return f'seed:{to_iso8601(start)}'
+
+
+def _log_feed_resume(provider: Provider, endpoint: str, resume: FeedResume) -> None:
+    """Narrate a feed run's resume point at INFO: seeded or resumed.
+
+    Args:
+        provider: The endpoint's provider.
+        endpoint: The endpoint name.
+        resume: The run's resolved feed resume value.
+
+    Side Effects:
+        Emits one INFO line.
+    """
+    match resume:
+        case FeedToken(from_version=from_version):
+            logger.info(
+                'feed run resumed: provider=%s endpoint=%s from_version=%s',
+                provider.value,
+                endpoint,
+                from_version,
+            )
+        case FeedSeed(start=start):
+            logger.info(
+                'feed run seeded: provider=%s endpoint=%s from_date=%s',
+                provider.value,
+                endpoint,
+                to_iso8601(start),
+            )
+
+
+class FeedDrive:
+    """Drives one feed endpoint's version-token stream, page by page.
+
+    Constructed once by ``EndpointRunner.__init__`` from the shared spine;
+    ``run`` takes the endpoint, its request driver, and the optional
+    per-frame observer.
+    """
+
+    def __init__(self, spine: RunnerSpine) -> None:
+        """
+        Args:
+            spine: The shared drive kit -- collaborators plus the runner's
+                writer-factory and projection seams.
+        """
+        self._spine = spine
+
+    def run(
+        self,
+        definition: EndpointDefinition[ResponseModel],
+        driver: RequestDriver,
+        observer: BatchObserver | None,
+    ) -> RunOutcome:
+        """Run the feed arm: drive the version-token stream, page by page.
+
+        Resume is the stored ``FeedToken`` used directly, or a ``FeedSeed``
+        at the sync-wide cold-start anchor when no token is stored -- the
+        seed rides ONLY the tokenless first run (I4; ``resolve_feed_resume``
+        makes it structural). The client is resolved before the run is
+        opened, so an unconfigured provider opens no dangling run. Each page
+        then commits independently through ``_consume_feed_pages`` in the
+        per-page crash order (parquet before token -- I1/I2); the ledger row
+        closes with the run's total row count and final ``toVersion`` after
+        the stream drains. A crash mid-stream leaves the run ``running``
+        (diagnostic only, the §5 stance) with every completed page's parquet
+        and token already committed -- the next run resumes from the last
+        committed token and refetches exactly one page, whose rows append
+        again as duplicates the stored-as-emitted contract absorbs (§4).
+
+        Args:
+            definition: The feed endpoint to run.
+            driver: The request driver supplying the run's pages (a
+                ``SingleRequestDriver`` -- feeds are single-chain).
+            observer: The optional per-frame hook, handed each
+                post-validation frame as the run streams.
+
+        Returns:
+            ``Executed`` with the fetched-row count and the append report.
+
+        Raises:
+            ConfigurationError: A watermark cursor is stored for this feed
+                endpoint (cross-mode corruption, from ``resolve_feed_resume``
+                -- raised before any run is opened), or a page carried no
+                durable progress (a non-feed decoder wired to a feed
+                endpoint).
+            FleetpullError: A fetch, validation, write, or completion failure
+                -- the run is recorded failed and the original error
+                re-raised; pages committed before it stand.
+        """
+        provider = definition.provider
+        name = definition.name
+        state = self._spine.state
+        client = self._spine.clients.client_for(provider)
+        resume = resolve_feed_resume(
+            state.cursors.get_cursor(provider, name),
+            self._spine.sync.default_start_datetime,
+            provider,
+            name,
+        )
+        _log_feed_resume(provider, name, resume)
+        run_id = state.recorder.start_feed_run(
+            provider, name, from_version=_feed_resume_label(resume)
+        )
+        with recorded_run(state.recorder, run_id):
+            writer = self._spine.make_writer(definition)
+            pages = driver.record_batches(definition, client, resume)
+            records_fetched, page_count, last_token = self._consume_feed_pages(
+                definition, pages, writer, observer
+            )
+            write = writer.finalize()
+            state.recorder.complete_run(
+                run_id, row_count=records_fetched, to_version=last_token
+            )
+        logger.info(
+            'feed complete: provider=%s endpoint=%s pages=%d records=%d to_version=%s',
+            provider.value,
+            name,
+            page_count,
+            records_fetched,
+            last_token,
+        )
+        outcome = Executed(records_fetched=records_fetched, write=write)
+        self._spine.projection.project(definition, outcome, window=None)
+        return outcome
+
+    def _consume_feed_pages(
+        self,
+        definition: EndpointDefinition[ResponseModel],
+        pages: Iterator[FetchedPage],
+        writer: DatasetWriter,
+        observer: BatchObserver | None,
+    ) -> tuple[int, int, str]:
+        """Consume the feed stream: per page, parquet BEFORE token (I1/I2).
+
+        The per-page transaction (DESIGN section 14): validate and frame the
+        page (``process_batch`` with no window context -- the feed has no
+        window, no future-event guard, no fold; whatever the stream emits is
+        stored), hand the frame to the observer where one rides, append it
+        durably (the feed writer's ``write`` is durable on return -- the
+        append-log cell's contract), and only THEN commit the page's
+        ``toVersion``. The token therefore never moves past unwritten data:
+        a crash between the two loses only the token, and the next run
+        refetches that one page. An empty page (the at-head terminal)
+        appends nothing and re-commits its unchanged token -- the feed
+        always has a cursor to write (section 5).
+
+        Args:
+            definition: The feed endpoint being run.
+            pages: The driver's fetched-page stream, ready to consume.
+            writer: The run's append writer, handed each frame in order.
+            observer: The optional per-frame hook.
+
+        Returns:
+            The fetched-row count, the page count, and the final committed
+            ``toVersion``.
+
+        Raises:
+            ConfigurationError: A page carried no ``durable_progress`` -- a
+                non-feed decoder is wired to a feed endpoint, a construction
+                bug surfaced before the page's rows are written.
+            RuntimeError: The stream yielded no pages, violating the
+                client's at-least-one-page contract.
+
+        Side Effects:
+            Appends part files and commits the feed cursor, page by page.
+        """
+        records_fetched = 0
+        page_count = 0
+        last_token: str | None = None
+        for page in pages:
+            token = page.durable_progress
+            if token is None:
+                raise ConfigurationError(
+                    'feed page carries no durable progress',
+                    provider=definition.provider.value,
+                    endpoint=definition.name,
+                    detail=(
+                        'the endpoint declares FeedMode but its page decoder '
+                        'yielded no resume token -- a non-feed decoder is wired '
+                        'to a feed endpoint'
+                    ),
+                )
+            processed = process_batch(page.records, definition, None)
+            if observer is not None:
+                observer(processed.frame)
+            writer.write(processed.frame)
+            self._spine.state.cursors.commit_feed_token(
+                definition.provider, definition.name, token
+            )
+            records_fetched += processed.frame.height
+            page_count += 1
+            last_token = token
+            logger.debug(
+                'feed page appended: provider=%s endpoint=%s page=%d records=%d '
+                'to_version=%s',
+                definition.provider.value,
+                definition.name,
+                page_count,
+                processed.frame.height,
+                token,
+            )
+        if last_token is None:
+            raise RuntimeError(
+                'feed drive yielded no pages -- fetch_pages always drives at least one'
+            )
+        return records_fetched, page_count, last_token

fleetpull/orchestrator/metadata_projection.py

@@ -0,0 +1,195 @@
+# src/fleetpull/orchestrator/metadata_projection.py
+"""The ``metadata.json`` projection: a committed run's facts, humanly readable.
+
+``MetadataProjection`` writes the per-endpoint ``metadata.json`` snapshot
+after a successful run fully commits (DESIGN §3) -- post-commit and
+best-effort, never part of the run's transaction: the outcome's counts, the
+run's resolved window, and a cursor read-back from the store flatten into a
+``MetadataSnapshot`` the storage face renders and atomically writes. The
+program never reads the file back (SQLite stays the single source of truth,
+§5). ``sync_mode_label`` also serves the runner's endpoint-start narration,
+so renaming a label changes both surfaces together.
+"""
+
+import logging
+from typing import Protocol
+
+from fleetpull.endpoints.shared import (
+    EndpointDefinition,
+    FeedMode,
+    SnapshotMode,
+    SyncMode,
+    WatermarkMode,
+)
+from fleetpull.incremental import (
+    DateWatermark,
+    DateWindow,
+    FeedToken,
+    IncrementalCursor,
+)
+from fleetpull.model_contract import ResponseModel
+from fleetpull.orchestrator.outcome import Executed
+from fleetpull.paths import PathInput, endpoint_directory
+from fleetpull.state import CursorKind
+from fleetpull.storage import (
+    MetadataSnapshot,
+    render_metadata_json,
+    write_metadata_json,
+)
+from fleetpull.timing import Clock, to_iso8601
+from fleetpull.vocabulary import Provider
+
+__all__: list[str] = ['CursorReader', 'MetadataProjection', 'sync_mode_label']
+
+logger = logging.getLogger(__name__)
+
+
+class CursorReader(Protocol):
+    """The one-method cursor read-back the projection needs."""
+
+    def get_cursor(self, provider: Provider, endpoint: str) -> IncrementalCursor | None:
+        """Return the persisted cursor for a (provider, endpoint), or None."""
+        ...
+
+
+def sync_mode_label(sync_mode: SyncMode) -> str:
+    """The sync mode's human-readable label.
+
+    Shared by the ``metadata.json`` projection and the endpoint-start
+    narration line -- renaming a label changes both surfaces.
+
+    Args:
+        sync_mode: The endpoint's declared sync mode.
+
+    Returns:
+        ``'snapshot'``, ``'watermark'``, or ``'feed'``.
+    """
+    match sync_mode:
+        case SnapshotMode():
+            return 'snapshot'
+        case WatermarkMode():
+            return 'watermark'
+        case FeedMode():
+            return 'feed'
+
+
+def _serialize_cursor(
+    cursor: IncrementalCursor | None,
+) -> tuple[str | None, str | None]:
+    """Serialize a stored cursor to the metadata projection's plain pair.
+
+    The storage face never sees the cursor union (the §11 storage/state
+    boundary), so the projection flattens it here; the kind labels are the
+    cursor store's own ``CursorKind`` discriminators.
+
+    Args:
+        cursor: The stored cursor, or ``None`` when none is persisted.
+
+    Returns:
+        ``(kind, value)`` -- ``('date_watermark', <iso8601>)``,
+        ``('feed_token', <token>)``, or ``(None, None)``.
+    """
+    match cursor:
+        case DateWatermark(watermark=watermark):
+            return (CursorKind.DATE_WATERMARK.value, to_iso8601(watermark))
+        case FeedToken(from_version=from_version):
+            return (CursorKind.FEED_TOKEN.value, from_version)
+        case None:
+            return (None, None)
+
+
+class MetadataProjection:
+    """Projects one committed run's facts into its ``metadata.json``.
+
+    Constructed once beside the runner with the cursor read-back surface,
+    the clock, and the dataset root; ``project`` runs after each arm's
+    successful commit.
+    """
+
+    def __init__(
+        self, cursors: CursorReader, clock: Clock, dataset_root: PathInput
+    ) -> None:
+        """
+        Args:
+            cursors: The cursor store's read surface, for the post-commit
+                cursor read-back.
+            clock: Supplies the ``generated_at`` instant.
+            dataset_root: Where the endpoint output directories live.
+        """
+        self._cursors = cursors
+        self._clock = clock
+        self._dataset_root = dataset_root
+
+    def project(
+        self,
+        definition: EndpointDefinition[ResponseModel],
+        outcome: Executed,
+        *,
+        window: DateWindow | None,
+    ) -> None:
+        """Project a committed run's facts into the endpoint's ``metadata.json``.
+
+        Runs only after a successful run has fully committed (parquet, the
+        ledger rows, the unit done-marks, the watermark prefix): the
+        outcome's counts, the run's resolved window,
+        and a cursor read-back from the store flatten into a
+        ``MetadataSnapshot`` the storage face renders and atomically writes
+        (DESIGN §3).
+
+        Args:
+            definition: The endpoint that just ran.
+            outcome: The run's merged ``Executed`` outcome.
+            window: The run's resolved window, or ``None`` when it had none
+                (a snapshot or feed run, or a watermark run that only
+                re-drove leftover units).
+
+        Side Effects:
+            Writes ``metadata.json`` in the endpoint's output directory; on
+            an ``OSError``, logs at ERROR and continues.
+        """
+        cursor_kind, cursor_value = _serialize_cursor(
+            self._cursors.get_cursor(definition.provider, definition.name)
+        )
+        snapshot = MetadataSnapshot(
+            provider=definition.provider.value,
+            endpoint=definition.name,
+            sync_mode=sync_mode_label(definition.sync_mode),
+            generated_at=self._clock.now_utc(),
+            records_fetched=outcome.records_fetched,
+            rows_written=outcome.write.rows_written,
+            duplicates_dropped=outcome.write.duplicates_dropped,
+            files_written=outcome.write.files_written,
+            deleted_partitions=outcome.write.deleted_partitions,
+            window_start=None if window is None else window.start,
+            window_end=None if window is None else window.end,
+            cursor_kind=cursor_kind,
+            cursor_value=cursor_value,
+        )
+        directory = endpoint_directory(
+            self._dataset_root, definition.provider.value, definition.name
+        )
+        # Only the file write is guarded, and only for OSError: the run is
+        # already committed (parquet, ledger, units, watermark), and the file is a
+        # cosmetic projection the next successful run rewrites -- failing a
+        # committed run over it would be worse than a stale file. A render
+        # failure is a bug and propagates. An absent endpoint directory is
+        # a healthy no-data state (a seeded-at-head feed, an empty
+        # watermark cold run) -- there is nothing to project onto, so the
+        # write is skipped quietly rather than alarmed over.
+        if not directory.exists():
+            logger.debug(
+                'metadata.json skipped: provider=%s endpoint=%s '
+                '(no data has ever landed)',
+                definition.provider.value,
+                definition.name,
+            )
+            return
+        text = render_metadata_json(snapshot)
+        try:
+            write_metadata_json(directory, text)
+        except OSError:
+            logger.exception(
+                'metadata.json write failed: provider=%s endpoint=%s',
+                definition.provider.value,
+                definition.name,
+            )

fleetpull/orchestrator/outcome.py

@@ -0,0 +1,53 @@
+# src/fleetpull/orchestrator/outcome.py
+"""The run executor's result carrier: what one endpoint run produced.
+
+A frozen tagged union the run executor returns instead of ``None``, so the caller
+(the orchestration entry, ``entry.py``) dispatches on the outcome rather than
+inferring it. ``Executed`` carries the fetched-row count and the write report;
+``CaughtUp`` is the no-op marker for a run whose resume window resolved to
+nothing -- no fetch, no writer, no ledger row. ``CaughtUp`` is reachable only on
+the watermark arm, whose window resolution can find nothing to drive; a snapshot
+always executes.
+"""
+
+from dataclasses import dataclass
+from datetime import datetime
+
+from fleetpull.storage import WriteResult
+
+__all__: list[str] = ['CaughtUp', 'Executed', 'RunOutcome']
+
+
+@dataclass(frozen=True, slots=True)
+class Executed:
+    """A run that fetched and wrote.
+
+    Attributes:
+        records_fetched: The count of records fetched across the run -- the
+            ledger's row count, distinct from ``write.rows_written`` (which dedup
+            and partitioning can make a different number).
+        write: The storage layer's write report for the run.
+        latest_observed: The folded in-window maximum event time, or ``None``
+            when the run observed no in-window event (an empty unit) or has no
+            event-time axis (a snapshot). The prefix-advance watermark rule's
+            per-unit datum (DESIGN section 5): the unit loop records it at
+            ``mark_done`` and the watermark advances only across the
+            contiguous done-prefix -- never here, never per-unit in isolation.
+    """
+
+    records_fetched: int
+    write: WriteResult
+    latest_observed: datetime | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class CaughtUp:
+    """A run that did nothing because its resume window resolved to empty.
+
+    No fetch, no writer, no ledger row; carries no fields (the run executor logs the
+    detail). Reachable only on the watermark arm, whose window resolution can
+    resolve to nothing.
+    """
+
+
+type RunOutcome = Executed | CaughtUp

fleetpull/orchestrator/recording.py

@@ -0,0 +1,85 @@
+# src/fleetpull/orchestrator/recording.py
+"""Record-failure-without-masking: the orchestration layer's shared stance.
+
+Every failure-recording write in this layer -- the run executor marking a run
+failed, the roster coordinator marking a harvest run failed, the unit loop
+marking a work unit failed -- touches SQLite, which can itself fail (a locked
+or unwritable database); if it does, that secondary failure must never
+replace the error that actually ended the work. ``record_failure_safely``
+states that stance once: run the recording call, and on any recording failure
+log it and swallow it so the original propagates. ``recorded_run`` is the
+run-scoped composition every run-opening arm wraps its protected block in:
+on any failure inside the block, record the run failed (safely) and re-raise
+the original.
+"""
+
+import logging
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from functools import partial
+from typing import Protocol
+
+__all__: list[str] = ['FailureRecorder', 'record_failure_safely', 'recorded_run']
+
+logger = logging.getLogger(__name__)
+
+
+class FailureRecorder(Protocol):
+    """The one-method fail-a-run surface ``recorded_run`` needs."""
+
+    def fail_run(self, run_id: int, *, error_detail: str) -> None:
+        """Close a run as failed with an error detail."""
+        ...
+
+
+def record_failure_safely(record: Callable[[], None], subject: str) -> None:
+    """Run a failure-recording call without masking the original error.
+
+    Args:
+        record: The recording call (``fail_run`` / ``mark_failed``, already
+            bound to its arguments).
+        subject: What is being marked failed, for the log line (e.g.
+            ``'run 5'``, ``'work unit 7'``).
+
+    Side Effects:
+        Runs ``record``; on a recording failure, logs it with its traceback
+        and swallows it so the caller's original error propagates.
+    """
+    try:
+        record()
+    except Exception:
+        logger.exception(
+            'failed to record %s as failed after an earlier error', subject
+        )
+
+
+@contextmanager
+def recorded_run(recorder: FailureRecorder, run_id: int) -> Iterator[None]:
+    """Run the block; on any failure, record the run failed and re-raise.
+
+    The shared spine of every run-opening arm's protected block: a failure
+    inside marks the run failed through ``record_failure_safely`` (so a
+    recording failure never masks the original) and the original error
+    propagates unchanged.
+
+    Args:
+        recorder: The ledger surface whose ``fail_run`` closes the run.
+        run_id: The open run to mark failed on a block failure.
+
+    Yields:
+        Nothing -- the protected block runs in the ``with`` body.
+
+    Raises:
+        Exception: Whatever the block raised, unchanged.
+
+    Side Effects:
+        On a block failure, records the run failed (best-effort).
+    """
+    try:
+        yield
+    except Exception as error:
+        record_failure_safely(
+            partial(recorder.fail_run, run_id, error_detail=str(error)),
+            f'run {run_id}',
+        )
+        raise