dstrack 0.1.0__py3-none-any.whl

dstrack/snapshot/__init__.py

@@ -0,0 +1,46 @@
+"""Snapshot computation for dstrack.
+
+Classes
+-------
+[SnapshotBuilder][dstrack.snapshot._builder.SnapshotBuilder]
+    Builds a complete snapshot from a single reader, combining the two below.
+
+[MetadataBuilder][dstrack.snapshot._metadata.MetadataBuilder]
+    Builds identity and structural snapshot fields from a reader's schema.
+
+[StatsComputer][dstrack.snapshot._stats.StatsComputer]
+    Computes per-column and dataset-level statistics in a single data pass.
+
+Result types
+------------
+[SnapshotMetadata][dstrack.snapshot._metadata.SnapshotMetadata],
+[DatasetStats][dstrack.snapshot._stats.DatasetStats]
+"""
+
+from ._builder import SnapshotBuilder, build_snapshot_dict
+from ._metadata import MetadataBuilder, SnapshotMetadata
+from ._stats import (
+    DatasetStats,
+    DatetimeColumnStats,
+    HistogramStats,
+    NumericColumnStats,
+    OtherColumnStats,
+    PercentileStats,
+    StatsComputer,
+    StringColumnStats,
+)
+
+__all__ = [
+    "DatasetStats",
+    "DatetimeColumnStats",
+    "HistogramStats",
+    "MetadataBuilder",
+    "NumericColumnStats",
+    "OtherColumnStats",
+    "PercentileStats",
+    "SnapshotBuilder",
+    "SnapshotMetadata",
+    "StatsComputer",
+    "StringColumnStats",
+    "build_snapshot_dict",
+]

dstrack/snapshot/_builder.py

@@ -0,0 +1,114 @@
+import dataclasses
+from pathlib import Path, PurePath
+from typing import Any
+
+from dstrack.readers import TabularReader
+from dstrack.snapshot._metadata import MetadataBuilder, SnapshotMetadata
+from dstrack.snapshot._stats import DatasetStats, StatsComputer
+
+
+class SnapshotBuilder:
+    """Builds a complete dataset snapshot from a single reader.
+
+    Combines [MetadataBuilder][dstrack.snapshot._metadata.MetadataBuilder]
+    (identity and schema) and
+    [StatsComputer][dstrack.snapshot._stats.StatsComputer] (a data pass over
+    the rows) into one JSON-ready snapshot dict.  Import it to build snapshots
+    from Python without going through the CLI:
+
+    Examples:
+        >>> from dstrack.readers import CsvReader
+        >>> from dstrack.snapshot import SnapshotBuilder
+        >>> reader = CsvReader("data.csv")  # doctest: +SKIP
+        >>> snapshot = SnapshotBuilder().build(  # doctest: +SKIP
+        ...     reader,
+        ...     dataset_name="customers",
+        ...     dataset_path="data.csv",
+        ...     created_by="alice",
+        ... )
+
+    Args:
+        metadata_builder: Builder used for identity and schema fields.
+            Defaults to a fresh
+            [MetadataBuilder][dstrack.snapshot._metadata.MetadataBuilder].
+        stats_computer: Computer used for the per-column and dataset-level
+            statistics.  Defaults to a fresh
+            [StatsComputer][dstrack.snapshot._stats.StatsComputer].  Pass a
+            pre-configured ``StatsComputer(max_rows=...)`` here to change the
+            in-memory row limit the statistics pass enforces.
+    """
+
+    def __init__(
+        self,
+        *,
+        metadata_builder: MetadataBuilder | None = None,
+        stats_computer: StatsComputer | None = None,
+    ) -> None:
+        self._metadata_builder = metadata_builder or MetadataBuilder()
+        self._stats_computer = stats_computer or StatsComputer()
+
+    def build(
+        self,
+        reader: TabularReader,
+        *,
+        dataset_name: str,
+        dataset_path: str | PurePath,
+        created_by: str,
+        source_type: str = "file",
+        source: str | Path | None = None,
+        source_hash: str | None = None,
+    ) -> dict[str, Any]:
+        """Build a JSON-serializable snapshot from a reader.
+
+        The reader is consumed once for schema inference and once for the
+        statistics data pass.
+
+        Args:
+            reader: Any [TabularReader][dstrack.readers._protocol.TabularReader].
+            dataset_name: Human-readable dataset name stored in the snapshot.
+            dataset_path: Source path or URI recorded in the snapshot as a
+                forward-slash string.  Never opened; pass ``source`` when the
+                data lives elsewhere.
+            created_by: User or process identifier.
+            source_type: Origin kind (``"file"``, ``"directory"``, etc.).
+            source: Location the data actually lives at, used only to compute
+                ``source_hash``.  Defaults to ``dataset_path``.
+            source_hash: Pre-computed source hash.  When ``None`` and the
+                source resolves to a regular file, a SHA-256 of the file bytes
+                is computed automatically by the metadata builder.
+
+        Returns:
+            A ``dict`` merging every metadata and statistics field, ready to be
+            serialized with [json.dumps][json.dumps].
+        """
+        metadata = self._metadata_builder.build(
+            reader,
+            dataset_name=dataset_name,
+            dataset_path=dataset_path,
+            source_type=source_type,
+            created_by=created_by,
+            source=source,
+            source_hash=source_hash,
+        )
+        stats = self._stats_computer.compute(reader)
+        return build_snapshot_dict(metadata, stats)
+
+
+def build_snapshot_dict(
+    metadata: SnapshotMetadata, stats: DatasetStats
+) -> dict[str, Any]:
+    """Merge metadata and statistics into one JSON-serializable snapshot dict.
+
+    The two inputs contribute disjoint sets of keys, so the merge never drops
+    a field.
+
+    Args:
+        metadata: Identity and schema fields from
+            [MetadataBuilder][dstrack.snapshot._metadata.MetadataBuilder].
+        stats: Volume, per-column, and quality fields from
+            [StatsComputer][dstrack.snapshot._stats.StatsComputer].
+
+    Returns:
+        A ``dict`` combining both, with nested dataclasses expanded to dicts.
+    """
+    return {**dataclasses.asdict(metadata), **dataclasses.asdict(stats)}

dstrack/snapshot/_metadata.py

@@ -0,0 +1,154 @@
+import hashlib
+import uuid
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path, PurePath
+from typing import Any
+
+from dstrack.readers import ColumnInfo, TabularReader
+from dstrack.snapshot._version import FORMAT_VERSION
+
+
+@dataclass
+class SnapshotMetadata:
+    """Identity and structural fields for a snapshot.
+
+    Covers every required top-level field that does not require a data pass:
+    versioning identifiers, authorship, schema shape, and the source hash.
+    """
+
+    format_version: str
+    snapshot_id: str
+    created_at: str
+    created_by: str
+    dataset_name: str
+    dataset_path: str
+    source_type: str
+    source_hash: str
+    num_columns: int
+    columns: list[dict[str, Any]]
+    schema_hash: str
+
+
+class MetadataBuilder:
+    """Builds identity and structural metadata without reading data rows.
+
+    Computes snapshot_id, created_at, created_by, dataset_name,
+    dataset_path, source_type, source_hash, num_columns, columns, and
+    schema_hash from the reader's column descriptors alone.
+
+    Note:
+        schema_hash is order-independent: reordering a dataset's columns
+        without changing their names or dtypes produces the same hash.
+    """
+
+    def build(
+        self,
+        reader: TabularReader,
+        *,
+        dataset_name: str,
+        dataset_path: str | PurePath,
+        source_type: str,
+        created_by: str,
+        source: str | Path | None = None,
+        source_hash: str | None = None,
+    ) -> SnapshotMetadata:
+        """Build metadata for a snapshot.
+
+        ``dataset_path`` is what gets *recorded*; ``source`` is what gets
+        *read*.  They differ whenever the recorded path is relative to a path
+        root that is not the current working directory, as it is for snapshots
+        written by the CLI.
+
+        Args:
+            reader: Any [TabularReader][dstrack.readers._protocol.TabularReader];
+                only [columns()][dstrack.readers._protocol.TabularReader.columns]
+                is called.
+            dataset_name: Human-readable dataset name stored in the snapshot.
+            dataset_path: Source path or URI at snapshot time, recorded in the
+                snapshot as a forward-slash string.  Never opened.
+            source_type: Origin kind (``"file"``, ``"directory"``, etc.).
+            created_by: User or process identifier.
+            source: Location the data actually lives at, used only to compute
+                ``source_hash``.  Defaults to ``dataset_path``, which is
+                correct when the recorded path is one the process can open.
+            source_hash: Pre-computed source hash.  When ``None`` and the
+                source resolves to a regular file, a SHA-256 of the file bytes
+                is computed automatically.
+
+        Returns:
+            A populated
+                [SnapshotMetadata][dstrack.snapshot._metadata.SnapshotMetadata]
+                instance.
+        """
+        cols = reader.columns()
+        # Recorded with forward slashes so a store written on Windows still
+        # matches the same dataset when read on POSIX, and vice versa.
+        path_str = (
+            dataset_path.as_posix()
+            if isinstance(dataset_path, PurePath)
+            else str(dataset_path)
+        )
+
+        if source_hash is None:
+            p = Path(source if source is not None else dataset_path)
+            source_hash = _hash_file(p) if p.is_file() else ""
+
+        return SnapshotMetadata(
+            format_version=FORMAT_VERSION,
+            snapshot_id=str(uuid.uuid4()),
+            created_at=datetime.now(UTC).isoformat(),
+            created_by=created_by,
+            dataset_name=dataset_name,
+            dataset_path=path_str,
+            source_type=source_type,
+            source_hash=source_hash,
+            num_columns=len(cols),
+            columns=_columns_to_dicts(cols),
+            schema_hash=_schema_hash(cols),
+        )
+
+
+def _hash_file(path: Path) -> str:
+    """Compute the SHA-256 hex digest of a file's contents.
+
+    Args:
+        path: Path to the file to hash.
+
+    Returns:
+        The hex-encoded SHA-256 digest of the file's bytes.
+    """
+    h = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def _columns_to_dicts(cols: list[ColumnInfo]) -> list[dict[str, Any]]:
+    """Convert column descriptors into plain dicts for serialization.
+
+    Args:
+        cols: Column descriptors as returned by a reader.
+
+    Returns:
+        One dict per column with ``name``, ``dtype``, and ``nullable`` keys.
+    """
+    return [{"name": c.name, "dtype": c.dtype, "nullable": c.nullable} for c in cols]
+
+
+def _schema_hash(cols: list[ColumnInfo]) -> str:
+    """Compute a deterministic hash of a schema's column names and dtypes.
+
+    Args:
+        cols: Column descriptors, in any order.
+
+    Returns:
+        A hex-encoded SHA-256 digest that changes if any column's name or
+        dtype changes. Order-independent: reordering ``cols`` does not
+        change the result. Ignores ``nullable``.
+    """
+    h = hashlib.sha256()
+    for name, dtype in sorted((c.name, c.dtype) for c in cols):
+        h.update(f"{name}\x00{dtype}\n".encode())
+    return h.hexdigest()