dstrack 0.1.0__py3-none-any.whl

dstrack/paths.py

@@ -0,0 +1,88 @@
+"""Resolution of the local store's directory, [STORE_DIRNAME][dstrack.paths.STORE_DIRNAME].
+
+Mirrors how git resolves `.git/`: absent an explicit override, it is found by
+walking up from the current directory until a `STORE_DIRNAME/` directory turns up.
+"""
+
+import os
+from pathlib import Path
+from typing import Final
+
+from dstrack.errors import StoreNotFoundError
+from dstrack.utils import get_invocation_path
+
+STORE_DIRNAME: Final = ".dstrack"
+ROOT_PATH_ENV_VAR: Final = "DSTRACK_ROOT_PATH"
+
+
+def _find_store_root(start: Path) -> Path:
+    """Walk upwards from `start` looking for a `STORE_DIRNAME/` directory.
+
+    Stops, like git, at the filesystem root or as soon as it would cross
+    onto a different filesystem/mount than `start`, rather than wandering
+    indefinitely upwards into unrelated system directories.
+
+    Args:
+        start: Directory to start the search from.
+
+    Returns:
+        The path to the `STORE_DIRNAME/` directory itself, the first one found
+        at or above `start`.
+
+    Raises:
+        StoreNotFoundError: If no directory at or above `start`, up to the
+            nearest filesystem boundary, contains a `STORE_DIRNAME/` directory.
+    """
+    current = start.resolve()
+    boundary_dev = current.stat().st_dev
+
+    while True:
+        candidate = current / STORE_DIRNAME
+        if candidate.is_dir():
+            return candidate
+
+        parent = current.parent
+        if parent == current or parent.stat().st_dev != boundary_dev:
+            break
+        current = parent
+
+    raise StoreNotFoundError(
+        f"Could not find a `{STORE_DIRNAME}/` directory in `{start}` or any "
+        "of its parent directories up to the filesystem boundary. Run "
+        "`dstrack init` to create one, or point at one explicitly via "
+        f"`--root` or `{ROOT_PATH_ENV_VAR}`."
+    )
+
+
+def resolve_store_root(root: Path | str | None = None) -> Path:
+    """Resolve the path to the local store, [STORE_DIRNAME][dstrack.paths.STORE_DIRNAME].
+
+    Checked in order, the first one available wins:
+        1. `root`, e.g. forwarded from a CLI `--root` option: the directory
+           `STORE_DIRNAME/` lives (or will be created) in.
+        2. The `DSTRACK_ROOT_PATH` environment variable: same meaning as
+           `root`.
+        3. Walking upwards from the current working directory until a
+           `STORE_DIRNAME/` directory is found, the same way git resolves `.git/`.
+
+    Args:
+        root: Explicit parent directory of `STORE_DIRNAME/`, typically supplied
+            by a CLI `--root` option. Taken as-is, without checking that
+            `STORE_DIRNAME/` actually exists inside it.
+
+    Returns:
+        The path to the `STORE_DIRNAME/` directory itself, as an absolute path.
+
+    Raises:
+        StoreNotFoundError: `root` is `None`, `DSTRACK_ROOT_PATH` is
+            unset, and no `STORE_DIRNAME/` directory is found by walking up from
+            the current directory.
+    """
+    if root is not None:
+        return Path(root).expanduser().resolve() / STORE_DIRNAME
+
+    env_root = os.environ.get(ROOT_PATH_ENV_VAR)
+    if env_root:
+        return Path(env_root).expanduser().resolve() / STORE_DIRNAME
+
+    return _find_store_root(get_invocation_path())

dstrack/readers/__init__.py

@@ -0,0 +1,69 @@
+"""Tabular data readers for dstrack.
+
+Built-in readers
+----------------
+[CsvReader][dstrack.readers._csv.CsvReader] - reads ``.csv`` files; no extra dependencies.
+
+Extending
+---------
+Implement [TabularReader][dstrack.readers._protocol.TabularReader] on any class to
+create a custom reader. That alone is enough to build a snapshot from Python:
+
+Examples:
+    ```python
+    >>> from dstrack.readers import ColumnInfo, TabularReader
+    >>> class MyParquetReader:
+    ...     def columns(self):
+    ...         return [ColumnInfo("x", "int64")]
+    ...     def iter_batches(self, batch_size=1000):
+    ...         return iter([[]])
+    ...
+    >>> isinstance(MyParquetReader(), TabularReader)
+    True
+
+    ```
+
+To make a reader reachable *by name* as well - from the CLI, or by extension
+inference - it must also satisfy
+[ReaderFactory][dstrack.readers._protocol.ReaderFactory] (a ``from_path`` classmethod), and
+be registered. There are two ways in, for two different situations:
+
+- **Shipping a reader in a package** others install: declare an entry point in
+  the ``dstrack.readers`` group, and set ``EXTENSIONS`` on the class. It is then
+  picked up automatically, and ``dstrack track data.parquet`` just works with
+  nothing extra typed:
+
+    ```toml
+    [project.entry-points."dstrack.readers"]
+    parquet = "dstrack_parquet:ParquetReader"
+    ```
+
+- **A reader in your own project**, not installed as a plugin: call
+  [register_reader][dstrack.readers._registry.register_reader] from Python, or name it on
+  the command line as ``--reader "mypackage.readers:ExcelReader"``.
+"""
+
+from ._csv import CsvReader
+from ._protocol import Cell, ColumnInfo, ReaderFactory, TabularReader
+from ._registry import (
+    ENTRY_POINT_GROUP,
+    available_readers,
+    known_extensions,
+    register_reader,
+)
+from ._resolve import load_reader_class, resolve_reader, resolve_reader_class
+
+__all__ = [
+    "ENTRY_POINT_GROUP",
+    "Cell",
+    "ColumnInfo",
+    "CsvReader",
+    "ReaderFactory",
+    "TabularReader",
+    "available_readers",
+    "known_extensions",
+    "load_reader_class",
+    "register_reader",
+    "resolve_reader",
+    "resolve_reader_class",
+]

dstrack/readers/_csv.py

@@ -0,0 +1,361 @@
+import csv
+import logging
+import os
+from collections import Counter
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any, TextIO
+
+from ._protocol import Cell, ColumnInfo
+
+logger = logging.getLogger(__name__)
+
+
+_NULL_PATTERNS: frozenset[str] = frozenset(
+    {"", "null", "NULL", "Null", "NaN", "nan", "NA", "N/A", "n/a", "None", "none"}
+)
+_BOOL_TRUE: frozenset[str] = frozenset({"true", "True", "TRUE"})
+_BOOL_ALL: frozenset[str] = frozenset(
+    {"true", "True", "TRUE", "false", "False", "FALSE"}
+)
+
+
+def _deduplicate_fieldnames(fieldnames: list[str]) -> list[str]:
+    """Deduplicates strings in list by appending a counter.
+
+    If the list contains repeated values, an int counter is added
+    to the end of the value.
+
+    Names that appear exactly once are treated as reserved: generated suffixes
+    will never claim them.  For example, ``["a", "a", "a_1"]`` becomes
+    ``["a", "a_2", "a_1"]`` rather than ``["a", "a_1", "a_1_1"]``.
+
+    Args:
+        fieldnames: list of strings to be deduplicated
+
+    Returns:
+        Deduplicated values
+    """
+    counts = Counter(fieldnames)
+    reserved = {name for name, count in counts.items() if count == 1}
+    seen: dict[str, int] = {}
+    used: set[str] = set()
+    result = []
+    for name in fieldnames:
+        candidate = name
+        suffix = seen.get(name, 1)
+        while candidate in used or (candidate != name and candidate in reserved):
+            candidate = f"{name}_{suffix}"
+            suffix += 1
+        seen[name] = suffix
+        used.add(candidate)
+        result.append(candidate)
+    return result
+
+
+def _looks_like_datetime(value: str) -> bool:
+    """Check for a leading ISO-8601 date (YYYY-MM-DD) without importing re.
+
+    Args:
+        value: Raw string value from a CSV cell.
+
+    Returns:
+        ``True`` if the value starts with a ``YYYY-MM-DD`` pattern.
+    """
+    return (
+        len(value) >= 10
+        and value[4] == "-"
+        and value[7] == "-"
+        and value[:4].isdigit()
+        and value[5:7].isdigit()
+        and value[8:10].isdigit()
+    )
+
+
+def _infer_dtype(samples: list[str]) -> str:
+    """Return the narrowest dtype that fits all non-null sample values.
+
+    Probes in order: ``int64``, ``float64``, ``bool``, ``datetime64``,
+    falling back to ``string``.
+
+    Args:
+        samples: Raw string values from a single column, including empty
+            strings for missing values.
+
+    Returns:
+        One of ``"int64"``, ``"float64"``, ``"bool"``, ``"datetime64"``,
+        or ``"string"``.
+    """
+    non_null = [v for v in samples if v not in _NULL_PATTERNS]
+    if not non_null:
+        return "string"
+    if all(v.lstrip("-").isdigit() for v in non_null):
+        return "int64"
+    try:
+        for v in non_null:
+            float(v)
+        return "float64"
+    except ValueError:
+        pass
+    if all(v in _BOOL_ALL for v in non_null):
+        return "bool"
+    if all(_looks_like_datetime(v) for v in non_null):
+        return "datetime64"
+    return "string"
+
+
+def _coerce(raw: str | None, dtype: str) -> Cell:
+    """Cast a raw CSV string to the column's inferred Python type.
+
+    Args:
+        raw: Raw string from the CSV cell, or ``None`` if the field is absent.
+        dtype: Target dtype string (e.g. ``"int64"``, ``"float64"``).
+
+    Returns:
+        The coerced value, or ``None`` for empty or missing inputs.
+    """
+    if raw is None or raw in _NULL_PATTERNS:
+        return None
+    if dtype == "int64":
+        return int(raw)
+    if dtype == "float64":
+        return float(raw)
+    if dtype == "bool":
+        if raw not in _BOOL_ALL:
+            raise ValueError(f"Invalid boolean value for bool column: {raw!r}")
+        return raw in _BOOL_TRUE
+    # string, datetime64, bytes - keep as str
+    return raw
+
+
+class CsvReader:
+    """Reads a CSV file using the standard-library ``csv`` module.
+
+    Satisfies [TabularReader][dstrack.readers._protocol.TabularReader] without
+    inheriting from it.  Column dtypes are inferred from the first
+    ``sample_rows`` data rows; every subsequent call to
+    [columns()][dstrack.readers._csv.CsvReader.columns] returns the cached
+    result.
+
+    The file's modification time and size are recorded when schema inference
+    runs.  [iter_batches()][dstrack.readers._csv.CsvReader.iter_batches] checks
+    them again before reading and raises [RuntimeError][] if the file has
+    changed, preventing silent schema/data mismatches.
+
+    Note:
+        Change detection relies on ``mtime_ns`` and file size reported by the
+        OS.  On filesystems with coarse modification-time resolution (FAT32,
+        some network or CI mounts), two writes that happen within the same
+        clock tick will share the same ``mtime_ns``, so a modification that
+        also preserves the file size may go undetected.  If you need
+        guaranteed detection in such environments, ensure at least one clock
+        tick (≥ 10 ms on most systems) elapses between calling
+        [columns()][dstrack.readers._csv.CsvReader.columns] and overwriting the
+        file.
+
+    Args:
+        path: Path to the CSV file.
+        sample_rows: Number of rows to read for dtype inference.
+            The values are read and data types for each column are inferred
+            from them.
+        encoding: File encoding passed to [open][].  Defaults to
+            ``"utf-8"``; use ``"cp1252"`` or ``"latin-1"`` for Excel exports.
+        rename_duplicates: When ``True``, duplicate header names are made unique
+            by appending a counter suffix (e.g. ``col``, ``col_1``, ``col_2``).
+            Headers that already appear exactly once in the file are treated as
+            reserved: generated suffixes will never overwrite them (e.g.
+            ``["a", "a", "a_1"]`` → ``["a", "a_2", "a_1"]``).
+            When ``False`` (default), a [ValueError][] is raised instead.
+        column_dtypes: Optional mapping of column name to dtype string that
+            overrides the inferred dtype for those columns.  Only the listed
+            columns are affected; all others are still inferred automatically.
+            ``"bytes"`` is not a valid override (see ADR-0002); passing it
+            raises [ValueError][].
+        **csv_kwargs: Forwarded verbatim to [DictReader][csv.DictReader]
+            (e.g. ``delimiter=";"``, ``quotechar="'"``).
+    """
+
+    EXTENSIONS = (".csv",)
+
+    @classmethod
+    def from_path(cls, path: str | Path) -> "CsvReader":
+        """Build a reader for ``path`` with default options.
+
+        Satisfies [ReaderFactory][dstrack.readers._protocol.ReaderFactory], which is how
+        the registry and ``--reader`` construct a reader they only know by name.
+        Options other than the path are not reachable this way; construct the
+        reader directly to set them.
+
+        Args:
+            path: Path to the CSV file.
+
+        Returns:
+            A [CsvReader][dstrack.readers._csv.CsvReader] bound to ``path``.
+        """
+        return cls(path)
+
+    def __init__(
+        self,
+        path: str | Path,
+        *,
+        sample_rows: int = 200,
+        encoding: str = "utf-8",
+        rename_duplicates: bool = False,
+        column_dtypes: dict[str, str] | None = None,
+        **csv_kwargs: Any,
+    ) -> None:
+        if column_dtypes:
+            bad = [name for name, dt in column_dtypes.items() if dt == "bytes"]
+            if bad:
+                raise ValueError(
+                    f"'bytes' dtype is not supported by CsvReader (column(s): {bad}). "
+                    "Keep the column as 'string' and decode in application code. "
+                    "See ADR-0002."
+                )
+        self._path = Path(path)
+        self._sample_rows = sample_rows
+        self._encoding = encoding
+        self._rename_duplicates = rename_duplicates
+        self._column_dtypes = column_dtypes or {}
+        self._csv_kwargs = csv_kwargs
+        self._columns: list[ColumnInfo] | None = None
+        self._file_stat: tuple[int, int] | None = None
+
+    @property
+    def path(self) -> Path:
+        """Path to the CSV file this reader is bound to."""
+        return self._path
+
+    def columns(self) -> list[ColumnInfo]:
+        """Return column descriptors, inferring dtypes on the first call.
+
+        Returns:
+            Ordered list of [ColumnInfo][dstrack.readers._protocol.ColumnInfo]
+            objects, one per CSV field.
+        """
+        if self._columns is None:
+            self._columns = self._detect_columns()
+        return list(self._columns)
+
+    def _stat_signature(self, fh: TextIO) -> tuple[int, int]:
+        """Return the change signature ``(mtime_ns, size)`` for an open file.
+
+        Args:
+            fh: An open handle to the CSV file.
+
+        Returns:
+            The file's ``mtime_ns`` and size, used to detect whether the file
+            was modified between schema inference and iteration.
+        """
+        stat = os.fstat(fh.fileno())
+        return (stat.st_mtime_ns, stat.st_size)
+
+    def _raise_if_file_changed(self, fh: TextIO) -> None:
+        """Raise if the file changed since schema inference recorded it.
+
+        Compares the current signature of ``fh`` against the one captured in
+        [_detect_columns()][dstrack.readers._csv.CsvReader._detect_columns].
+        Does nothing if no signature has been recorded yet.
+
+        Args:
+            fh: An open handle to the CSV file.
+
+        Raises:
+            RuntimeError: If the file's signature differs from the recorded one.
+        """
+        if self._file_stat is not None and self._stat_signature(fh) != self._file_stat:
+            raise RuntimeError(
+                f"File changed between schema inference and iteration: {self._path}"
+            )
+
+    def _detect_columns(self) -> list[ColumnInfo]:
+        """Read up to ``sample_rows`` rows and infer a ColumnInfo per field.
+
+        Records ``(mtime_ns, size)`` of the open file handle in
+        ``_file_stat`` for later comparison by
+        [iter_batches()][dstrack.readers._csv.CsvReader.iter_batches].
+
+        Returns:
+            Ordered list of [ColumnInfo][dstrack.readers._protocol.ColumnInfo]
+            objects, one per CSV field.
+        """
+        logger.debug(
+            f"Inferring column dtypes for {self._path} from up to {self._sample_rows} sample rows",
+        )
+        samples: dict[str, list[str]] = {}
+        with self._path.open(newline="", encoding=self._encoding) as fh:
+            self._file_stat = self._stat_signature(fh)
+            reader = csv.DictReader(fh, **self._csv_kwargs)
+            fieldnames = reader.fieldnames
+            if not fieldnames:
+                return []
+            if len(set(fieldnames)) != len(fieldnames):
+                if not self._rename_duplicates:
+                    raise ValueError(
+                        "Duplicate CSV headers detected. Pass rename_duplicates=True"
+                        " to automatically rename them (e.g. 'col', 'col_1', 'col_2')."
+                    )
+                fieldnames = _deduplicate_fieldnames(list(fieldnames))
+                reader.fieldnames = fieldnames
+            for name in fieldnames:
+                samples[name] = []
+            for i, row in enumerate(reader):
+                if i >= self._sample_rows:
+                    break
+                for name in fieldnames:
+                    raw = row.get(name)
+                    samples[name].append("" if raw is None else raw)
+
+        columns = [
+            ColumnInfo(
+                name=name,
+                dtype=self._column_dtypes.get(name) or _infer_dtype(vals),
+                nullable=any(v in _NULL_PATTERNS for v in vals),
+            )
+            for name, vals in samples.items()
+        ]
+        logger.debug(f"Inferred {len(columns)} columns for {self._path}")
+        return columns
+
+    def iter_batches(self, batch_size: int = 1000) -> Iterator[list[list[Cell]]]:
+        """Yield batches of coerced rows.
+
+        Opens the file once per call.  Before reading, compares the file's
+        current ``mtime_ns`` and size against the values recorded during
+        schema inference and raises if they differ.
+
+        Args:
+            batch_size: Maximum number of rows per batch.
+
+        Yields:
+            A list of rows; each row is a list of
+                [Cell][dstrack.readers._protocol.Cell] values aligned with
+                [columns()][dstrack.readers._csv.CsvReader.columns].
+
+        Raises:
+            RuntimeError: If the file was modified since
+                [columns()][dstrack.readers._csv.CsvReader.columns] was last
+                called.
+        """
+        cols = self.columns()
+        names = [c.name for c in cols]
+        dtypes = [c.dtype for c in cols]
+        batch: list[list[Cell]] = []
+
+        with self._path.open(newline="", encoding=self._encoding) as fh:
+            self._raise_if_file_changed(fh)
+            reader = csv.DictReader(fh, **self._csv_kwargs)
+            _ = reader.fieldnames  # consume header row
+            reader.fieldnames = names
+            for row in reader:
+                batch.append(
+                    [
+                        _coerce(row.get(name), dtype)
+                        for name, dtype in zip(names, dtypes, strict=True)
+                    ]
+                )
+                if len(batch) >= batch_size:
+                    yield batch
+                    batch = []
+        if batch:
+            yield batch

dstrack/readers/_protocol.py

@@ -0,0 +1,120 @@
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Protocol, TypeAlias, runtime_checkable
+
+# A single cell value after dtype coercion; None represents a missing value.
+Cell: TypeAlias = int | float | str | bool | None
+
+
+@dataclass(frozen=True)
+class ColumnInfo:
+    """Metadata for one column of a tabular dataset.
+
+    Attributes:
+        name: Column name as it appears in the source.
+        dtype: Storage type using snapshot-schema vocabulary: ``int64``,
+            ``float64``, ``string``, ``bool``, ``datetime64``, or ``bytes``.
+        nullable: ``True`` if any value in this column may be ``None``.
+    """
+
+    name: str
+    dtype: str
+    nullable: bool = True
+
+
+@runtime_checkable
+class TabularReader(Protocol):
+    """Structural protocol for tabular data sources.
+
+    Any class that exposes
+    [columns()][dstrack.readers._protocol.TabularReader.columns] and
+    [iter_batches()][dstrack.readers._protocol.TabularReader.iter_batches]
+    satisfies this protocol, no inheritance required.  Third-party readers for
+    Parquet, SQL, HuggingFace datasets, etc. only need to implement these two
+    methods.
+
+    Examples:
+        >>> class MyParquetReader:
+        ...     def columns(self):
+        ...         return [ColumnInfo("x", "int64")]
+        ...
+        ...     def iter_batches(self, batch_size=1000):
+        ...         return iter([[]])
+        >>> isinstance(MyParquetReader(), TabularReader)
+        True
+    """
+
+    def columns(self) -> list[ColumnInfo]:
+        """Return column descriptors.
+
+        May open or inspect the source on the first call; subsequent calls
+        should return a cached result.
+
+        Returns:
+            Ordered list of [ColumnInfo][dstrack.readers._protocol.ColumnInfo]
+            objects, one per column.
+        """
+        ...
+
+    def iter_batches(self, batch_size: int = 1000) -> Iterator[list[list[Cell]]]:
+        """Yield non-empty batches of rows.
+
+        Each row is a list of coerced values aligned with
+        [columns()][dstrack.readers._protocol.TabularReader.columns].
+        Missing values are represented as ``None``.
+
+        Args:
+            batch_size: Maximum number of rows per batch.
+
+        Yields:
+            A list of rows, each row being a list of
+            [Cell][dstrack.readers._protocol.Cell] values.
+        """
+        ...
+
+
+@runtime_checkable
+class ReaderFactory(Protocol):
+    """Construction contract for readers reached *by name* rather than by instance.
+
+    [TabularReader][dstrack.readers._protocol.TabularReader] describes how a
+    reader is *read*, and says nothing about how one is *built*: code that
+    already holds an instance never needs to know. But the registry and the
+    ``"package.module:ClassName"`` spec only ever yield a class, so they need a
+    uniform way to turn that class into an instance given a source path.
+    [from_path()][dstrack.readers._protocol.ReaderFactory.from_path] is that way,
+    and it is checked against this protocol before the class is ever called.
+
+    This is deliberately a second, separate protocol: a reader used only from
+    Python (constructed by the caller, handed straight to
+    [SnapshotBuilder][dstrack.snapshot._builder.SnapshotBuilder]) still needs
+    nothing beyond [TabularReader][dstrack.readers._protocol.TabularReader].
+    Only readers that are registered, or named on the command line, must also
+    satisfy this one.
+
+    Examples:
+        ```python
+        >>> from pathlib import Path
+        >>> from dstrack.readers import CsvReader, ReaderFactory
+        >>> isinstance(CsvReader, ReaderFactory)  # the class, not an instance
+        True
+
+        ```
+    """
+
+    def from_path(self, path: Path) -> TabularReader:
+        """Build a reader for ``path`` using default options.
+
+        Implemented as a ``classmethod`` on the reader class; the protocol is
+        therefore checked against the class object itself
+        (``isinstance(MyReader, ReaderFactory)``).
+
+        Args:
+            path: Path to the dataset source the reader will read.
+
+        Returns:
+            An instance satisfying
+            [TabularReader][dstrack.readers._protocol.TabularReader].
+        """
+        ...