fleetpull 0.1.0__py3-none-any.whl

fleetpull/storage/metadata.py

@@ -0,0 +1,161 @@
+# src/fleetpull/storage/metadata.py
+"""The per-endpoint ``metadata.json`` projection: render and write the snapshot.
+
+``metadata.json`` is a generated human-readable snapshot of one endpoint's
+last successful run, written beside its data after the run commits (DESIGN
+§3). It is never read by the program — SQLite stays the single source of
+truth (§5) — so this module is a pure projection surface: a frozen carrier
+of already-committed facts (``MetadataSnapshot``), a deterministic JSON
+render, and an atomic file write. The orchestrator supplies every fact as a
+plain value (strings, counts, datetimes); no SQLite, no state types, no
+cursor union crosses this boundary, which is what keeps the §11
+storage/state separation intact.
+
+The write is temp-then-rename atomic (the same same-filesystem rename
+doctrine as ``atomic.py``) and deliberately does not create the endpoint
+directory: an absent directory means no data ever landed there, an upstream
+bug that must surface as the ``OSError`` the caller's posture handles, not
+be papered over with a metadata file for a dataset that does not exist.
+"""
+
+import json
+from dataclasses import dataclass
+from datetime import date, datetime
+from pathlib import Path
+
+from fleetpull.storage.atomic import atomic_write_text
+from fleetpull.timing import to_iso8601
+
+__all__: list[str] = [
+    'MetadataSnapshot',
+    'render_metadata_json',
+    'write_metadata_json',
+]
+
+# The per-endpoint snapshot file name (DESIGN §3).
+_METADATA_FILE_NAME: str = 'metadata.json'
+
+
+@dataclass(frozen=True, slots=True)
+class MetadataSnapshot:
+    """One endpoint's last-successful-run facts, ready to render.
+
+    Every field is a plain value the orchestrator has already committed
+    elsewhere (ledger, cursor store, write report) — this carrier holds the
+    projection, never the state types themselves.
+
+    Attributes:
+        provider: The provider directory name (e.g. ``'motive'``).
+        endpoint: The endpoint name (e.g. ``'vehicles'``).
+        sync_mode: The endpoint's sync-mode label — ``'snapshot'``,
+            ``'watermark'``, or ``'feed'``.
+        generated_at: The projection instant, timezone-aware UTC.
+        records_fetched: Records fetched across the run.
+        rows_written: Rows written to disk this run.
+        duplicates_dropped: Exact-duplicate rows removed at write time.
+        files_written: Parquet files written this run.
+        deleted_partitions: The date partitions pruned this run.
+        window_start: The run's resolved window start, or ``None`` when the
+            run had no window (a snapshot run).
+        window_end: The run's resolved window end, or ``None`` when the run
+            had no window.
+        cursor_kind: The stored cursor's kind (``'date_watermark'`` or
+            ``'feed_token'``), or ``None`` when no cursor is stored.
+        cursor_value: The stored cursor's serialized value, or ``None`` when
+            no cursor is stored.
+    """
+
+    provider: str
+    endpoint: str
+    sync_mode: str
+    generated_at: datetime
+    records_fetched: int
+    rows_written: int
+    duplicates_dropped: int
+    files_written: int
+    deleted_partitions: tuple[date, ...]
+    window_start: datetime | None
+    window_end: datetime | None
+    cursor_kind: str | None
+    cursor_value: str | None
+
+
+def _optional_iso8601(moment: datetime | None) -> str | None:
+    """Render an optional UTC datetime as ISO-8601, passing ``None`` through.
+
+    Args:
+        moment: A timezone-aware UTC datetime, or ``None``.
+
+    Returns:
+        The ISO-8601 ``Z`` string, or ``None``.
+
+    Raises:
+        ValueError: ``moment`` is naive or non-UTC (from ``to_iso8601``).
+    """
+    return None if moment is None else to_iso8601(moment)
+
+
+def render_metadata_json(snapshot: MetadataSnapshot) -> str:
+    """Render a snapshot as the ``metadata.json`` document text.
+
+    A pure, deterministic render: datetimes via the timing codec's ISO-8601
+    form, dates via ``isoformat``, two-space indentation, one trailing
+    newline. ``schema_version`` names the document shape so a human diffing
+    files across package versions can tell a shape change from a data change.
+
+    Args:
+        snapshot: The committed run facts to render.
+
+    Returns:
+        The complete JSON document text, trailing newline included.
+
+    Raises:
+        ValueError: A snapshot datetime is naive or non-UTC (from the codec).
+    """
+    cursor = (
+        None
+        if snapshot.cursor_kind is None
+        else {'kind': snapshot.cursor_kind, 'value': snapshot.cursor_value}
+    )
+    document = {
+        'schema_version': 1,
+        'provider': snapshot.provider,
+        'endpoint': snapshot.endpoint,
+        'sync_mode': snapshot.sync_mode,
+        'generated_at': to_iso8601(snapshot.generated_at),
+        'last_run': {
+            'records_fetched': snapshot.records_fetched,
+            'rows_written': snapshot.rows_written,
+            'duplicates_dropped': snapshot.duplicates_dropped,
+            'files_written': snapshot.files_written,
+            'deleted_partitions': [
+                deleted_date.isoformat() for deleted_date in snapshot.deleted_partitions
+            ],
+            'window_start': _optional_iso8601(snapshot.window_start),
+            'window_end': _optional_iso8601(snapshot.window_end),
+        },
+        'cursor': cursor,
+    }
+    return json.dumps(document, indent=2) + '\n'
+
+
+def write_metadata_json(endpoint_directory: Path, text: str) -> None:
+    """Atomically write ``text`` as the endpoint directory's ``metadata.json``.
+
+    ``atomic_write_text``'s temp-then-rename, aimed at the projection's fixed
+    file name. The endpoint directory is deliberately not created: an absent
+    directory means no data ever landed — an upstream bug surfaced here as
+    ``OSError``, not silenced by a ``mkdir``.
+
+    Args:
+        endpoint_directory: The endpoint's existing output directory.
+        text: The complete document text (from ``render_metadata_json``).
+
+    Raises:
+        OSError: The write or rename failed — including a missing endpoint
+            directory (the temp is cleaned up first).
+
+    Side Effects:
+        Writes and renames files inside ``endpoint_directory``.
+    """
+    atomic_write_text(text, endpoint_directory / _METADATA_FILE_NAME)

fleetpull/storage/partitioned.py

@@ -0,0 +1,205 @@
+# src/fleetpull/storage/partitioned.py
+"""The date-partitioned writer family: hive ``date=YYYY-MM-DD`` partitions.
+
+``PartitionedWriter`` is the family's shared substance -- stage each piece as
+date-split shards, fold each touched date's shards into its ``part.parquet``,
+prune where the cell prunes -- and ``WatermarkPartitionedWriter`` is the
+shipped ``(DATE_PARTITIONED, WatermarkMode)`` cell (replace each covered
+partition, prune the covered-but-empty dates; DESIGN §3/§4). The feed cell is
+deliberately NOT in this family: it appends numbered part files with per-write
+durability and no window (``storage/append.py``). ``select_writer``
+(``storage/writers.py``) is the routing face that constructs these.
+"""
+
+from abc import ABC, abstractmethod
+from datetime import date
+from pathlib import Path
+
+import polars as pl
+
+from fleetpull.incremental import DateWindow
+from fleetpull.storage.files import partition_part_file
+from fleetpull.storage.pruning import prune_window_partitions, window_dates
+from fleetpull.storage.read import read_parquet_if_exists
+from fleetpull.storage.result import WriteResult
+from fleetpull.storage.staging import (
+    clear_partition_staging,
+    compact_partition,
+    stage_shard,
+)
+
+__all__: list[str] = ['PartitionedWriter', 'WatermarkPartitionedWriter']
+
+
+class PartitionedWriter(ABC):
+    """Writers whose dataset is hive ``date=YYYY-MM-DD`` partitions.
+
+    Shared substance: each ``write`` stages its piece as date-split shards
+    (``stage_shard``); ``finalize`` folds each touched date's shards into that
+    date's ``part.parquet`` (``compact_partition``) and reports. Two per-cell
+    decisions are the subclass's: whether compaction folds in the existing
+    partition (``_reads_existing`` -- fold-in cells would, replace cells do
+    not), and whether the run prunes the covered-but-empty dates (``_prunes``
+    -- a watermark refresh authoritatively replaces its window, so it prunes).
+    The ABC owns the staging and the finalize orchestration; the per-partition
+    compaction is the shared ``compact_partition`` it drives.
+
+    ``write`` carries the window tripwire: every staged partition date must lie
+    in ``window_dates(window)``. Upstream, the resume window is day-aligned at
+    resolution and the orchestrator window-filters each batch before writing,
+    so a staged date outside the window means an upstream boundary was missed
+    -- the require-inside half of the normalize-at-boundary doctrine (the
+    canonical-UTC stance), raised loudly here rather than silently replacing or
+    pruning a partition the run had no right to touch.
+    """
+
+    @property
+    @abstractmethod
+    def _reads_existing(self) -> bool:
+        """Whether compaction folds the existing ``part.parquet`` into the result."""
+        ...
+
+    @property
+    @abstractmethod
+    def _prunes(self) -> bool:
+        """Whether ``finalize`` deletes the window's covered-but-empty partitions."""
+        ...
+
+    def __init__(
+        self,
+        target_dir: Path,
+        event_time_column: str,
+        window: DateWindow,
+        *,
+        drop_duplicates: bool = True,
+    ) -> None:
+        """Bind the writer to an endpoint directory, event-time column, and window.
+
+        Clears any stale shards a prior crashed run left under the window's covered
+        dates before staging anything, so a later ``compact_partition`` folds only
+        this run's shards, not a superseded row's pre-edit version. A covered date
+        the clear empties (a crash before any ``part.parquet`` existed) has its
+        now-empty ``date=`` directory removed too, upholding the
+        no-empty-partition-directory invariant (DESIGN §3/§14).
+
+        Assumes no overlapping writer for the same endpoint and window runs
+        concurrently: the construction-time clear is destructive, so overlapping
+        runs could delete each other's live staging. Orchestration must prevent
+        overlapping runs for one endpoint.
+
+        Args:
+            target_dir: The endpoint directory holding the ``date=`` partitions.
+            event_time_column: The UTC datetime column whose date keys the
+                partitions.
+            window: The run's half-open resume window -- the prune's bound and the
+                covered-date set the construction-time staging clear sweeps.
+            drop_duplicates: Whether compaction drops exact-duplicate rows
+                (``storage.drop_exact_duplicates``, default on).
+
+        Side Effects:
+            Removes any existing ``staging/`` directory under the window's covered
+            dates, and any ``date=`` directory the clear leaves empty.
+        """
+        self._target_dir = target_dir
+        self._event_time_column = event_time_column
+        self._window = window
+        self._drop_duplicates = drop_duplicates
+        self._covered_dates: frozenset[date] = frozenset(window_dates(window))
+        self._written_dates: set[date] = set()
+        clear_partition_staging(target_dir, self._covered_dates)
+
+    def write(self, frame: pl.DataFrame) -> None:
+        """Stage one fetched piece as date-split shards, inside the window only.
+
+        Args:
+            frame: One fetched piece (e.g. one vehicle's rows over the window).
+
+        Raises:
+            ValueError: A staged partition date lies outside the window's
+                covered dates -- an upstream window filter missed rows (the
+                interior tripwire; see the class docstring). The run fails
+                loudly; the orphaned out-of-window shards are inert (``.shard``
+                files are invisible to hive ``*.parquet`` reads) until a later
+                run whose window covers that date sweeps them at construction.
+
+        Side Effects:
+            Writes ``.shard`` files under the touched dates' staging directories.
+        """
+        touched_dates = stage_shard(self._target_dir, frame, self._event_time_column)
+        out_of_window = touched_dates - self._covered_dates
+        if out_of_window:
+            raise ValueError(
+                f'staged partition dates outside the resume window: '
+                f'{sorted(out_of_window)} not in {sorted(self._covered_dates)} '
+                f'-- an upstream window filter missed these rows'
+            )
+        self._written_dates.update(touched_dates)
+
+    def finalize(self) -> WriteResult:
+        """Compact each partition, clear staging, prune if the cell prunes, and report.
+
+        Returns:
+            The write report -- rows and duplicates summed across the compacted
+            partitions, ``files_written`` the partition count, and
+            ``deleted_partitions`` the pruned dates (empty when the cell does not
+            prune).
+
+        Side Effects:
+            Writes each touched ``part.parquet`` and clears its staging; deletes
+            the pruned partition directories.
+        """
+        rows_written = 0
+        duplicates_dropped = 0
+        for partition_date in sorted(self._written_dates):
+            existing = (
+                read_parquet_if_exists(
+                    partition_part_file(self._target_dir, partition_date)
+                )
+                if self._reads_existing
+                else None
+            )
+            result = compact_partition(
+                self._target_dir,
+                partition_date,
+                existing,
+                drop_duplicates=self._drop_duplicates,
+            )
+            rows_written += result.rows_written
+            duplicates_dropped += result.duplicates_dropped
+        clear_partition_staging(self._target_dir, self._written_dates)
+        deleted_partitions = (
+            tuple(
+                prune_window_partitions(
+                    self._target_dir, self._window, self._written_dates
+                )
+            )
+            if self._prunes
+            else ()
+        )
+        return WriteResult(
+            rows_written=rows_written,
+            duplicates_dropped=duplicates_dropped,
+            files_written=len(self._written_dates),
+            deleted_partitions=deleted_partitions,
+        )
+
+
+class WatermarkPartitionedWriter(PartitionedWriter):
+    """``watermark`` + ``date_partitioned``: replace each covered partition, prune.
+
+    The floored window refetches each covered date in full, so compaction replaces
+    that date's ``part.parquet`` outright -- no existing read -- and the run prunes
+    the covered dates that returned empty (a provider that deleted or edited
+    records). Late, corrected, and deleted records all land through replacement,
+    never a row-level merge (DESIGN §3/§4).
+    """
+
+    @property
+    def _reads_existing(self) -> bool:
+        """Replace, never fold in the prior partition."""
+        return False
+
+    @property
+    def _prunes(self) -> bool:
+        """The window is authoritatively replaced, so empty-refetch dates prune."""
+        return True

fleetpull/storage/pruning.py

@@ -0,0 +1,169 @@
+# src/fleetpull/storage/pruning.py
+"""Date-partition pruning: drop the partitions a refresh window covers but did
+not write.
+
+The delete half of the date-partitioned write path's two-step (write the fetched
+partitions, then delete the covered-but-unwritten ones). A watermark refresh
+authoritatively replaces its window ``[start, end)``: every ``date=`` partition
+the window covers must, after the run, hold exactly this run's data for that date.
+A covered date the fetch did not write is therefore stale -- the provider deleted
+or edited every record it once held -- so its partition directory must go, the
+directory-grain analogue of the row-level delete-by-window (DESIGN §3/§4).
+
+Four stateless single-concern functions compose the prune, none aware of *when* in
+the run they run. The driver sequences the delete after the full per-vehicle
+fan-out, once ``written_dates`` is complete -- a partition holds the whole fleet's
+rows for its date, so the written set is not final until the last vehicle is
+processed -- but these functions take the finished set as a given:
+
+- ``window_dates`` -- the covered calendar dates of a window (pure date math).
+- ``existing_partition_dates`` -- which of a candidate date set exist on disk (the
+  filesystem probe, candidate-driven, never a directory scan).
+- ``delete_partition`` -- remove one partition directory (the single delete).
+- ``prune_window_partitions`` -- the composer: covered, on disk, minus written,
+  deleted.
+
+The set arithmetic is ``window_dates(window) ∩ {on disk} - {written}``. The
+``∩ window_dates`` term is the safety leash: it bounds the delete to the refresh
+window, so history *outside* the window is never touched. Deleting on
+``existed - written`` alone would erase every partition outside the window -- a
+data-loss bug -- which is why the window intersection lives inside the composer,
+enforced once here rather than trusted to every caller. The probe is
+candidate-driven (``is_dir`` only the window's own dates), so its cost is
+O(window), never O(dataset): listing the endpoint directory is the O(dataset) scan
+partitioning exists to avoid.
+"""
+
+import shutil
+from collections.abc import Collection
+from datetime import date, timedelta
+from pathlib import Path
+
+from fleetpull.incremental import DateWindow
+from fleetpull.storage.files import partition_dir
+
+__all__: list[str] = [
+    'delete_partition',
+    'existing_partition_dates',
+    'prune_window_partitions',
+    'window_dates',
+]
+
+
+def window_dates(window: DateWindow) -> list[date]:
+    """The calendar dates a half-open ``[start, end)`` window covers.
+
+    A partition ``date=d`` is covered iff some instant of that day lies in
+    ``[start, end)`` -- the dates ``start.date()`` through the window's
+    ``last_covered_date`` inclusive (the half-open edge and its one-microsecond
+    derivation are stated once, on ``DateWindow``).
+
+    Args:
+        window: The half-open ``[start, end)`` resume window.
+
+    Returns:
+        The covered dates in ascending order, ``start.date()`` first. Always at
+        least one date, since ``window`` guarantees ``start < end``.
+
+    Side Effects:
+        None -- pure function.
+    """
+    first = window.start.date()
+    last = window.last_covered_date
+    span_days = (last - first).days + 1
+    return [first + timedelta(days=offset) for offset in range(span_days)]
+
+
+def existing_partition_dates(
+    endpoint_dir: Path, candidate_dates: Collection[date]
+) -> set[date]:
+    """Which of ``candidate_dates`` have a partition directory on disk.
+
+    Candidate-driven: probes only the ``date=`` directories for the dates handed
+    in (``is_dir`` per candidate), never lists ``endpoint_dir``. The cost is
+    therefore O(candidates), not O(dataset) -- the point of anchoring the prune to
+    the refresh window rather than scanning the tree (DESIGN §3).
+
+    Args:
+        endpoint_dir: The endpoint directory holding the ``date=`` partitions.
+        candidate_dates: The dates to probe (the window's covered dates).
+
+    Returns:
+        The subset of ``candidate_dates`` whose partition directory exists. Empty
+        when none exist (e.g. a first run, or an endpoint directory not yet
+        created).
+
+    Side Effects:
+        None -- reads directory existence only; creates and writes nothing.
+    """
+    return {
+        candidate_date
+        for candidate_date in candidate_dates
+        if partition_dir(endpoint_dir, candidate_date).is_dir()
+    }
+
+
+def delete_partition(endpoint_dir: Path, partition_date: date) -> None:
+    """Remove one date-partition directory and everything under it.
+
+    Deletes the whole ``date=YYYY-MM-DD`` directory (the part file and any temp
+    siblings), not just the part file, so no empty partition directory is left
+    behind -- a present-but-empty ``date=`` directory would read as a date with
+    data when scanned, the invariant ``split_by_date`` upholds by never creating
+    empty partitions (DESIGN §3).
+
+    Strict: the directory must exist. A missing directory is a caller bug -- the
+    composer deletes only dates ``existing_partition_dates`` just confirmed present
+    -- and a silent no-op on a *delete* would hide that logic error, so the
+    underlying ``rmtree`` is allowed to raise.
+
+    Args:
+        endpoint_dir: The endpoint directory holding the ``date=`` partitions.
+        partition_date: The calendar date whose partition directory to remove.
+
+    Side Effects:
+        Recursively deletes the partition directory from the filesystem.
+
+    Raises:
+        FileNotFoundError: The partition directory does not exist.
+        OSError: The directory could not be removed (permissions, etc.).
+    """
+    shutil.rmtree(partition_dir(endpoint_dir, partition_date))
+
+
+def prune_window_partitions(
+    endpoint_dir: Path, window: DateWindow, written_dates: Collection[date]
+) -> list[date]:
+    """Delete the partitions ``window`` covers but this run did not write.
+
+    Computes the stale set itself -- ``window_dates(window)`` intersected with the
+    partitions present on disk, minus ``written_dates`` -- and deletes each. The
+    intersection with the window is the safety leash that keeps the delete inside
+    the refresh window (DESIGN §3); it lives here, not in the caller, so the bound
+    is enforced in one tested place. Usually returns empty: in steady state every
+    covered date gets data, so nothing is stale. That rarity is exactly why the
+    path must be airtight -- a directory delete that almost never fires is where a
+    latent bug hides.
+
+    Args:
+        endpoint_dir: The endpoint directory holding the ``date=`` partitions.
+        window: The half-open ``[start, end)`` window this run refreshed.
+        written_dates: The dates this run wrote a partition for (the keys of the
+            per-date split). Order and duplicates do not matter.
+
+    Returns:
+        The dates whose partitions were deleted, ascending. Empty when no covered
+        partition was stale.
+
+    Side Effects:
+        Deletes stale partition directories from the filesystem.
+
+    Raises:
+        OSError: A stale partition directory could not be removed.
+    """
+    covered = window_dates(window)
+    present = existing_partition_dates(endpoint_dir, covered)
+    stale = sorted(present - set(written_dates))
+    for stale_date in stale:
+        delete_partition(endpoint_dir, stale_date)
+    return stale

fleetpull/storage/read.py

@@ -0,0 +1,35 @@
+# src/fleetpull/storage/read.py
+"""Existence-tolerant parquet reads: the read sibling of the atomic write.
+
+A writer that combines this run's records with what is already on disk reads the
+prior file through here; a first run -- no file yet -- comes back as ``None``
+rather than an error, so the writer's combine handles the empty case the same way
+every time. The built cells all replace or append rather than combine (the
+snapshot writer rewrites the single file; the partitioned watermark cell replaces
+its partitions; the append-log feed cell only ever adds new part files), so no
+writer reads today; the one combining cell -- the single-file watermark cell --
+remains unbuilt.
+"""
+
+from pathlib import Path
+
+import polars as pl
+
+__all__: list[str] = ['read_parquet_if_exists']
+
+
+def read_parquet_if_exists(path: Path) -> pl.DataFrame | None:
+    """Read a parquet file, or ``None`` if it does not exist.
+
+    Args:
+        path: The parquet file to read.
+
+    Returns:
+        The frame, or ``None`` when ``path`` does not exist (a first run).
+
+    Side Effects:
+        Reads ``path`` if present.
+    """
+    if path.exists():
+        return pl.read_parquet(path)
+    return None

fleetpull/storage/result.py

@@ -0,0 +1,34 @@
+# src/fleetpull/storage/result.py
+"""The storage write report.
+
+``WriteResult`` is what a ``DatasetWriter.finalize`` returns -- the run ledger
+reads it (via the orchestrator, not storage): the per-run write stats a
+consumer inspects to know what a sync produced.
+"""
+
+from dataclasses import dataclass
+from datetime import date
+
+__all__: list[str] = ['WriteResult']
+
+
+@dataclass(frozen=True, slots=True)
+class WriteResult:
+    """What one endpoint's write produced this run.
+
+    Attributes:
+        rows_written: Rows written to disk this run. For ``single`` (a full
+            rewrite) this is the whole dataset; for ``date_partitioned`` it is the
+            rows across the touched partitions, not the dataset total.
+        duplicates_dropped: Exact-duplicate rows removed at write time.
+        files_written: Parquet files written -- ``1`` for ``single``, the count of
+            touched partitions for ``date_partitioned``.
+        deleted_partitions: The date partitions deleted this run -- the
+            covered-but-empty dates a date-partitioned watermark refresh prunes.
+            Empty for every other cell.
+    """
+
+    rows_written: int
+    duplicates_dropped: int
+    files_written: int
+    deleted_partitions: tuple[date, ...] = ()