fleetpull 0.1.0__py3-none-any.whl

fleetpull/storage/append.py

@@ -0,0 +1,187 @@
+# src/fleetpull/storage/append.py
+"""The append-log writer: the feed cell's accumulate-only write path.
+
+``FeedAppendWriter`` is the ``(APPEND_LOG, FeedMode)`` cell (DESIGN §3/§4).
+Unlike every other writer, each ``write`` is durable the moment it returns:
+the piece's rows are split by event date and each date's rows land as the
+next-numbered ``part-NNNNN.parquet`` in that date's partition, through the
+atomic temp-then-rename write. That per-write durability is what the feed
+drive's per-page crash order stands on — a page's parquet must be on disk
+before its token commits (§14's I1), so the feed cell cannot defer its bytes
+to ``finalize`` the way the staged and single-file cells do; ``finalize``
+only reports.
+
+The cell is append-only in the strictest sense (§14's I3): it writes new
+part files and never reads, rewrites, or deletes an existing one. Part
+numbering is max-existing + 1, scanned per write — sound because parquet
+merge per endpoint is single-writer (DESIGN §5) — and the chosen target is
+still existence-checked before the rename, so a violated single-writer
+assumption clobbers nothing and fails loudly instead.
+
+Deliberately NO write-time exact dedup, unlike every other cell: the dataset
+contract is stored-as-emitted (DESIGN §4) — re-emitted versions and a crash
+window's refetched page land as new rows, and the consumer reconciles
+calculated feeds by ``(id, max version)`` and active feeds by ``id``.
+Deduping a crash-window duplicate would require reading and rewriting
+already-landed part files, which is exactly what I3 forbids; within-page
+duplicates are the provider's emission, kept verbatim.
+"""
+
+import re
+from pathlib import Path
+from typing import Final
+
+import polars as pl
+
+from fleetpull.exceptions import ProviderResponseError
+from fleetpull.storage.atomic import atomic_write_parquet
+from fleetpull.storage.files import append_part_file, partition_dir
+from fleetpull.storage.result import WriteResult
+from fleetpull.storage.splitting import split_by_date
+
+__all__: list[str] = ['FeedAppendWriter']
+
+# The strict shape of an append-log part file name. The scan parses numbers
+# out of exactly this shape; anything else matching 'part-*.parquet' in an
+# append-log partition is a foreign file and fails loudly (the corruption
+# stance) rather than silently skewing the numbering.
+_APPEND_PART_PATTERN: Final[re.Pattern[str]] = re.compile(r'part-(\d+)\.parquet\Z')
+
+
+def _next_part_number(partition_directory: Path) -> int:
+    """The next append part number for one partition: max existing + 1.
+
+    Scans only the given partition directory (never the endpoint tree), so
+    the cost is O(parts in this partition). A missing directory — the
+    partition's first-ever part — starts at 1.
+
+    Args:
+        partition_directory: The ``date=YYYY-MM-DD`` directory to scan.
+
+    Returns:
+        The next part number (>= 1).
+
+    Raises:
+        ValueError: A ``part-*.parquet`` file in the partition does not match
+            the strict ``part-NNNNN.parquet`` shape — a foreign file in an
+            append-log partition is a dataset-layout violation, surfaced
+            loudly rather than silently skewing the numbering.
+
+    Side Effects:
+        None -- reads directory listings only.
+    """
+    if not partition_directory.is_dir():
+        return 1
+    highest = 0
+    for part_path in partition_directory.glob('part-*.parquet'):
+        match = _APPEND_PART_PATTERN.fullmatch(part_path.name)
+        if match is None:
+            raise ValueError(
+                f'foreign part file in append-log partition: {part_path} '
+                f'does not match part-NNNNN.parquet'
+            )
+        highest = max(highest, int(match.group(1)))
+    return highest + 1
+
+
+class FeedAppendWriter:
+    """``feed`` + ``append_log``: append numbered part files, touch nothing else.
+
+    Each ``write`` lands its piece durably (one new part file per event date
+    present), so the feed drive can commit that piece's token immediately
+    after; ``finalize`` reports the accumulated counts and writes nothing.
+    ``duplicates_dropped`` is always ``0`` — the cell performs no write-time
+    dedup by design (stored-as-emitted; the module docstring carries the
+    rationale).
+    """
+
+    def __init__(self, target_dir: Path, event_time_column: str) -> None:
+        """Bind the writer to an endpoint directory and event-time column.
+
+        Args:
+            target_dir: The endpoint directory holding the ``date=`` partitions.
+            event_time_column: The UTC datetime column whose date routes each
+                row into its partition.
+
+        Side Effects:
+            None -- nothing is swept or cleared; prior parts are never touched.
+        """
+        self._target_dir = target_dir
+        self._event_time_column = event_time_column
+        self._rows_written = 0
+        self._files_written = 0
+
+    def write(self, frame: pl.DataFrame) -> None:
+        """Append one piece durably: a new numbered part per event date present.
+
+        Splits ``frame`` by the UTC date of the event-time column and writes
+        each date's rows as that partition's next ``part-NNNNN.parquet``,
+        atomically and DURABLY (the durable-rename recipe: the token commit
+        is fsynced, so the page it covers must be too -- a power loss never
+        persists a token past unwritten data). An empty frame writes nothing (an at-head feed page). The
+        write is durable when this returns — the caller may commit the
+        piece's feed token immediately after (§14's per-page crash order).
+
+        Args:
+            frame: One fetched piece (one feed page's validated rows).
+
+        Raises:
+            ProviderResponseError: A row's event-time value is null — a
+                feed record without its partition key is a provider
+                contract violation surfaced loudly BEFORE any part lands
+                (never a mid-write stall behind an untyped error).
+            ValueError: A foreign ``part-*.parquet`` file skews a partition's
+                numbering (from the scan).
+            RuntimeError: The scanned next part number already exists on disk
+                — the single-writer assumption is violated; refusing keeps the
+                append-only invariant (I3) intact instead of clobbering a
+                landed part.
+
+        Side Effects:
+            Writes one new part file per event date present, creating
+            partition directories as needed. Never modifies an existing file.
+        """
+        null_dated = frame[self._event_time_column].null_count() if frame.height else 0
+        if null_dated:
+            raise ProviderResponseError(
+                detail=(
+                    f'{null_dated} feed record(s) carry a null '
+                    f'{self._event_time_column!r} -- the append-log partition '
+                    f'key; refusing the page whole before any part lands'
+                )
+            )
+        for partition_date, sub_frame in split_by_date(frame, self._event_time_column):
+            part_number = _next_part_number(
+                partition_dir(self._target_dir, partition_date)
+            )
+            target = append_part_file(self._target_dir, partition_date, part_number)
+            if target.exists():
+                raise RuntimeError(
+                    f'append-log part collision: {target} already exists -- '
+                    f'two writers on one endpoint violate the single-writer '
+                    f'invariant (DESIGN section 5)'
+                )
+            atomic_write_parquet(sub_frame, target, durable=True)
+            self._rows_written += sub_frame.height
+            self._files_written += 1
+
+    def finalize(self) -> WriteResult:
+        """Report the accumulated append counts; write nothing.
+
+        Every piece already landed durably in ``write`` (the per-page crash
+        order requires it), so there is nothing left to persist.
+
+        Returns:
+            The write report — rows and part files appended this run,
+            ``duplicates_dropped`` always ``0`` (no write-time dedup, by
+            design), ``deleted_partitions`` always empty (nothing is ever
+            deleted).
+
+        Side Effects:
+            None.
+        """
+        return WriteResult(
+            rows_written=self._rows_written,
+            duplicates_dropped=0,
+            files_written=self._files_written,
+        )

fleetpull/storage/atomic.py

@@ -0,0 +1,161 @@
+# src/fleetpull/storage/atomic.py
+"""The atomic file writes: the storage layer's durability primitives.
+
+Every layout's every write goes through here. A file write is not atomic -- a
+crash mid-write corrupts the file -- so the content is written to a temp sibling
+and renamed onto the target, which POSIX guarantees is atomic on one filesystem.
+The prior target is untouched until the rename; a crash leaves either the old
+file or the new, never a partial one (DESIGN §5 crash-safety). The temp is
+always cleaned up: on success (already renamed away) or on failure.
+``atomic_write_parquet`` is the parquet write every dataset writer uses;
+``atomic_write_text`` is the same temp-then-rename skeleton for the
+``metadata.json`` projection's document text.
+"""
+
+import os
+from pathlib import Path
+
+import polars as pl
+
+from fleetpull.polars_typing import ParquetCompression
+from fleetpull.storage.files import temp_sibling_path
+
+__all__: list[str] = ['atomic_write_parquet', 'atomic_write_text']
+
+
+def _fsync_path(path: Path) -> None:
+    """Flush one path's kernel buffers to stable storage.
+
+    The open/fsync/close block the durable-rename recipe applies to the
+    temp file before the rename and to each directory of the durable
+    chain after it.
+
+    Args:
+        path: The file or directory to fsync.
+
+    Raises:
+        OSError: The open or fsync failed.
+
+    Side Effects:
+        Issues an ``fsync`` system call.
+    """
+    descriptor = os.open(path, os.O_RDONLY)
+    try:
+        os.fsync(descriptor)
+    finally:
+        os.close(descriptor)
+
+
+def _durable_directory_chain(directory: Path) -> list[Path]:
+    """The directories a durable write must fsync after its rename.
+
+    Always ``directory`` itself -- the rename lands the file's entry
+    there. When the write is about to CREATE missing ancestors (a feed
+    page opening a new ``date=`` partition, or the first-ever write of an
+    endpoint), each new directory's own entry lives one level up, so the
+    chain extends through every missing ancestor to the first
+    pre-existing one: a new-directory chain fsynced only at its deepest
+    link is NOT durable -- power loss can drop the newly created
+    directory (and the file inside it) from the un-fsynced parent while
+    a later fsynced commit survives, persisting a cursor past lost data.
+
+    Must be computed BEFORE the ``mkdir`` that creates the chain, while
+    the missing set is still observable.
+
+    Args:
+        directory: The file's parent directory, existing or about to be
+            created.
+
+    Returns:
+        ``directory`` first, then each missing ancestor's parent up to
+        and including the first pre-existing directory. Just
+        ``[directory]`` when it already exists.
+    """
+    chain = [directory]
+    current = directory
+    while not current.exists():
+        chain.append(current.parent)
+        current = current.parent
+    return chain
+
+
+def atomic_write_parquet(
+    frame: pl.DataFrame,
+    target: Path,
+    compression: ParquetCompression = 'snappy',
+    *,
+    durable: bool = False,
+) -> None:
+    """Write ``frame`` to ``target`` atomically via temp-then-rename.
+
+    Ensures ``target``'s parent directory exists, writes the frame to a temp
+    sibling, then atomically renames it onto ``target``. Never leaves a partial
+    ``target``.
+
+    Args:
+        frame: The DataFrame to persist.
+        target: The final parquet path.
+        compression: Parquet compression codec. ``'snappy'`` by default (fast,
+            BigQuery-friendly); a later config surface parameterizes it.
+        durable: When True, fsync the temp file before the rename and the
+            durable directory chain after it -- the parent, every ancestor
+            this write newly created, and the first pre-existing ancestor
+            (whose entry for the new chain must also reach stable
+            storage). The feed append cell requires it: its token commit
+            is fsynced (SQLite), so a power loss must never persist a
+            token whose page -- or whose newly created partition
+            directory -- the page cache still held. The self-healing
+            writers (replace-partition under lookback refetch, snapshot
+            rewrite) stay non-durable -- a lost write there is refetched
+            by design.
+
+    Side Effects:
+        Creates ``target``'s parent directory; writes and renames files;
+        when ``durable``, fsyncs the file and the directory chain.
+
+    Raises:
+        OSError: If the write, rename, or fsync fails (the temp is cleaned
+            up first).
+    """
+    durable_chain = _durable_directory_chain(target.parent) if durable else []
+    target.parent.mkdir(parents=True, exist_ok=True)
+    temp: Path = temp_sibling_path(target)
+    try:
+        frame.write_parquet(temp, compression=compression)
+        if durable:
+            _fsync_path(temp)
+        temp.replace(target)
+        for directory in durable_chain:
+            _fsync_path(directory)
+    finally:
+        temp.unlink(missing_ok=True)
+
+
+def atomic_write_text(text: str, target: Path) -> None:
+    """Write ``text`` to ``target`` atomically via temp-then-rename.
+
+    The parquet write's temp-then-rename skeleton applied to document text:
+    temp-sibling in the target's own directory, so the rename is
+    same-filesystem and atomic; a crash leaves the prior file or the new one,
+    never a partial. The target's parent directory is deliberately NOT
+    created -- the one caller (the ``metadata.json`` projection) requires an
+    absent endpoint directory to surface as ``OSError``, never be papered
+    over with a ``mkdir``.
+
+    Args:
+        text: The complete document text to persist, written UTF-8.
+        target: The final file path; its parent directory must exist.
+
+    Raises:
+        OSError: The write or rename failed -- including a missing parent
+            directory (the temp is cleaned up first).
+
+    Side Effects:
+        Writes and renames files inside ``target``'s directory.
+    """
+    temp: Path = temp_sibling_path(target)
+    try:
+        temp.write_text(text, encoding='utf-8')
+        temp.replace(target)
+    finally:
+        temp.unlink(missing_ok=True)

fleetpull/storage/files.py

@@ -0,0 +1,176 @@
+# src/fleetpull/storage/files.py
+"""Storage file-path construction: the parquet-format-specific paths under an
+endpoint directory.
+
+Pure path arithmetic, no filesystem access. The single-layout data file, the
+date-partitioned layout's partition / part / staging paths, and the temp sibling
+used for atomic writes live here -- storage-specific, unlike the shared
+endpoint-directory construction in ``paths``. The temp sibling sits in the
+target's own directory so the rename that follows is same-filesystem and
+therefore atomic.
+"""
+
+from datetime import date
+from pathlib import Path
+from uuid import uuid4
+
+from fleetpull.paths import date_partition_segment
+
+__all__: list[str] = [
+    'append_part_file',
+    'data_file',
+    'partition_dir',
+    'partition_part_file',
+    'partition_staging_dir',
+    'partition_staging_shard',
+    'temp_sibling_path',
+]
+
+# The single-layout data file name (DESIGN §3).
+_SINGLE_FILE_NAME: str = 'data.parquet'
+
+# The date-partitioned layout's per-partition part file name (DESIGN §3).
+_PART_FILE_NAME: str = 'part.parquet'
+
+# The per-partition staging directory and shard suffix (DESIGN §3).
+_STAGING_DIR_NAME: str = 'staging'
+
+# The append-log layout's numbered part-file width (DESIGN §3): five digits
+# zero-padded, so directory listings sort chronologically at a glance. Purely
+# cosmetic — the append scan parses the number, never sorts the text — so a
+# partition beyond 99,999 parts widens naturally without breaking anything.
+_APPEND_PART_NUMBER_WIDTH: int = 5
+
+
+def data_file(endpoint_dir: Path) -> Path:
+    """The ``single``-layout data file under an endpoint directory.
+
+    Args:
+        endpoint_dir: The endpoint directory (from ``endpoint_directory``).
+
+    Returns:
+        ``{endpoint_dir}/data.parquet``.
+    """
+    return endpoint_dir / _SINGLE_FILE_NAME
+
+
+def partition_dir(endpoint_dir: Path, partition_date: date) -> Path:
+    """The date-partition directory for one date under an endpoint directory.
+
+    The single place the hive ``date=YYYY-MM-DD`` directory path is built;
+    ``partition_part_file`` and the prune step both go through it, so the
+    structural fact lives once.
+
+    Args:
+        endpoint_dir: The endpoint directory (from ``endpoint_directory``).
+        partition_date: The partition's calendar date.
+
+    Returns:
+        ``{endpoint_dir}/date=YYYY-MM-DD``.
+
+    Side Effects:
+        None -- pure path arithmetic; no filesystem access.
+    """
+    return endpoint_dir / date_partition_segment(partition_date)
+
+
+def partition_part_file(endpoint_dir: Path, partition_date: date) -> Path:
+    """The date-partitioned part file for one date under an endpoint directory.
+
+    Args:
+        endpoint_dir: The endpoint directory (from ``endpoint_directory``).
+        partition_date: The partition's calendar date.
+
+    Returns:
+        ``{endpoint_dir}/date=YYYY-MM-DD/part.parquet``.
+
+    Side Effects:
+        None -- pure path arithmetic; no filesystem access.
+    """
+    return partition_dir(endpoint_dir, partition_date) / _PART_FILE_NAME
+
+
+def append_part_file(
+    endpoint_dir: Path, partition_date: date, part_number: int
+) -> Path:
+    """The append-log layout's numbered part file for one date partition.
+
+    The path half of the append-only feed cell (DESIGN §3/§4): each feed page
+    lands as the next-numbered ``part-NNNNN.parquet`` in the partitions its
+    records' event dates route to; the numbering scan lives with the append
+    writer (``storage/append.py``), keeping this module pure path arithmetic.
+
+    Args:
+        endpoint_dir: The endpoint directory (from ``endpoint_directory``).
+        partition_date: The partition's calendar date.
+        part_number: The part's ordinal (>= 1; the writer's scan supplies
+            max-existing + 1).
+
+    Returns:
+        ``{endpoint_dir}/date=YYYY-MM-DD/part-NNNNN.parquet``.
+
+    Side Effects:
+        None -- pure path arithmetic; no filesystem access.
+    """
+    return partition_dir(endpoint_dir, partition_date) / (
+        f'part-{part_number:0{_APPEND_PART_NUMBER_WIDTH}d}.parquet'
+    )
+
+
+def partition_staging_dir(endpoint_dir: Path, partition_date: date) -> Path:
+    """The staging directory inside one date partition.
+
+    Holds the per-piece ``.shard`` files a fanned-out write lands before
+    compaction folds them into ``part.parquet``. Inside the partition directory
+    so staging is co-located with the partition it feeds; the ``.shard`` extension
+    on the shards (not ``.parquet``) keeps them out of a hive ``*.parquet`` read.
+
+    Args:
+        endpoint_dir: The endpoint directory (from ``endpoint_directory``).
+        partition_date: The partition's calendar date.
+
+    Returns:
+        ``{endpoint_dir}/date=YYYY-MM-DD/staging``.
+
+    Side Effects:
+        None -- pure path arithmetic; no filesystem access.
+    """
+    return partition_dir(endpoint_dir, partition_date) / _STAGING_DIR_NAME
+
+
+def partition_staging_shard(endpoint_dir: Path, partition_date: date) -> Path:
+    """A unique ``.shard`` path under one date partition's staging directory.
+
+    Each call returns a fresh uuid-named shard, so the fanned-out writes to one
+    date never collide. The ``.shard`` extension keeps the file out of a hive
+    ``*.parquet`` read; compaction reads it back by explicit path.
+
+    Args:
+        endpoint_dir: The endpoint directory (from ``endpoint_directory``).
+        partition_date: The partition's calendar date.
+
+    Returns:
+        ``{endpoint_dir}/date=YYYY-MM-DD/staging/shard-{uuid}.shard``.
+
+    Side Effects:
+        None -- pure path construction (the uuid makes each call unique).
+    """
+    return partition_staging_dir(endpoint_dir, partition_date) / (
+        f'shard-{uuid4().hex}.shard'
+    )
+
+
+def temp_sibling_path(target: Path) -> Path:
+    """A unique temporary path beside ``target`` for an atomic write.
+
+    Placed in ``target``'s own directory so the follow-up rename stays on one
+    filesystem (POSIX guarantees same-filesystem rename atomicity). The unique
+    suffix avoids colliding with a stale temp from an earlier interrupted write.
+
+    Args:
+        target: The final file the temp will be renamed onto.
+
+    Returns:
+        A hidden, uniquely-named sibling temp path.
+    """
+    return target.with_name(f'.{target.name}.{uuid4().hex}.tmp')

fleetpull/storage/frames.py

@@ -0,0 +1,98 @@
+# src/fleetpull/storage/frames.py
+"""Frame operations the write path composes: the exact-duplicate dedup and the
+half-open window-membership predicate.
+
+Pure DataFrame helpers. ``drop_exact_duplicates`` is the write-time exact
+dedup (DESIGN §6), and ``dedup_counting`` is the flag-and-count composition
+the replace-partition and single-file writers run on
+their finalized frames, clearing byte-identical rows as a cheap safety net.
+The feed append cell deliberately does NOT (DESIGN §4's stored-as-emitted
+contract and §14's append-only invariant): crash-window and re-emission
+duplicates are the log's honest content, reconciled by the consumer. ``in_window`` is the half-open ``[start, end)``
+membership predicate that defines the window boundary in exactly one place: its
+consumer today is the orchestrator's batch shaping, which keeps a fetch's
+in-window rows before they reach a writer; the future single-file window-clearing
+cells will apply it in both polarities (delete the existing window's rows, keep
+the fresh fetch's in-window rows) from the same rule.
+"""
+
+import polars as pl
+
+from fleetpull.incremental import DateWindow
+
+__all__: list[str] = ['dedup_counting', 'drop_exact_duplicates', 'in_window']
+
+
+def dedup_counting(frame: pl.DataFrame, *, enabled: bool) -> tuple[pl.DataFrame, int]:
+    """Apply the write-time exact dedup when enabled, counting what it dropped.
+
+    The dedup-and-count step every deduping write path runs before
+    persisting -- the finalized single file and each compacted partition --
+    stated once so the flag handling and the dropped-row arithmetic never
+    drift between writers.
+
+    Args:
+        frame: The frame about to be written.
+        enabled: The ``storage.drop_exact_duplicates`` switch; ``False``
+            passes ``frame`` through byte-for-byte.
+
+    Returns:
+        The frame to write and the count of exact-duplicate rows removed
+        (``0`` when disabled or none were found).
+    """
+    if not enabled:
+        return frame, 0
+    written = drop_exact_duplicates(frame)
+    return written, frame.height - written.height
+
+
+def drop_exact_duplicates(frame: pl.DataFrame) -> pl.DataFrame:
+    """Drop byte-identical duplicate rows, preserving first-occurrence order.
+
+    Exactness over all columns: two rows collapse only if every value matches.
+    Same-key-different-payload rows are deliberately kept -- collapsing those is
+    semantic dedup, out of scope (DESIGN §6).
+
+    Args:
+        frame: The frame to dedup.
+
+    Returns:
+        The frame with exact-duplicate rows removed, order preserved.
+    """
+    return frame.unique(maintain_order=True)
+
+
+def in_window(event_time_column: str, window: DateWindow) -> pl.Expr:
+    """The half-open ``[start, end)`` window-membership predicate for a column.
+
+    Returns the boolean Polars expression true for rows whose
+    ``event_time_column`` falls in ``window`` -- ``>= window.start`` and
+    ``< window.end``, the half-open boundary made literal. It is a *predicate*,
+    not a filter, so the boundary is defined in exactly one place and "removal"
+    stays the caller's concern (DESIGN §4). Its consumer today is the
+    orchestrator's batch shaping, which keeps a fetch's in-window rows with
+    ``frame.filter(in_window(col, w))``; the future single-file window-clearing
+    cells will add the other polarity, ``frame.filter(~in_window(col, w))``, to
+    delete a window's rows from the existing on-disk frame.
+
+    The full ``[start, end)`` predicate, not merely ``>= start``: in steady state
+    ``end`` is ``now`` and ``< end`` binds nothing, but a historical backfill
+    chunk has a real ``end``, and ``< end`` is what stops an event on a chunk
+    boundary from being claimed by both the chunk ending at it and the one
+    starting at it. ``window`` already guarantees ``start < end`` at construction,
+    so the predicate never sees an inverted range.
+
+    Args:
+        event_time_column: Name of the UTC datetime column to test.
+        window: The half-open ``[start, end)`` resume window.
+
+    Returns:
+        A boolean ``pl.Expr`` true for in-window rows; apply it (or its negation)
+        with ``DataFrame.filter``.
+
+    Side Effects:
+        None -- builds an expression; evaluates nothing.
+    """
+    return (pl.col(event_time_column) >= window.start) & (
+        pl.col(event_time_column) < window.end
+    )