dstrack 0.1.0__py3-none-any.whl

dstrack/readers/_registry.py

@@ -0,0 +1,241 @@
+"""Registry of reader classes, keyed by short name and by file extension.
+
+Registration is the single extension point for readers. Three kinds of user
+reach it by three different routes, and none of them should have to do the other
+two's work:
+
+- **From Python**, a caller who already has a reader instance hands it straight
+  to [SnapshotBuilder][dstrack.snapshot._builder.SnapshotBuilder]; the registry
+  is never consulted.
+- **From an installed plugin package**, a reader self-registers through the
+  ``dstrack.readers`` entry-point group, so ``dstrack track data.parquet`` infers
+  it from the extension with nothing to type.
+- **From an ad-hoc class** in the user's own project, which is not installed as a
+  plugin and so is named explicitly as ``"package.module:ClassName"`` (see
+  [dstrack.readers._resolve][]).
+
+A registered class must satisfy both
+[TabularReader][dstrack.readers._protocol.TabularReader] (how it reads) and
+[ReaderFactory][dstrack.readers._protocol.ReaderFactory] (how it is built from a
+path);
+[check_reader_class()][dstrack.readers._registry.check_reader_class] enforces
+that *before* the class is ever instantiated.
+"""
+
+import inspect
+import logging
+from collections.abc import Sequence
+from importlib.metadata import entry_points
+from pathlib import Path
+from typing import Final
+
+from ._csv import CsvReader
+from ._protocol import ReaderFactory, TabularReader
+
+_log = logging.getLogger(__name__)
+
+# Entry-point group an installed package declares to add a reader, e.g.
+#   [project.entry-points."dstrack.readers"]
+#   parquet = "dstrack_parquet:ParquetReader"
+ENTRY_POINT_GROUP: Final[str] = "dstrack.readers"
+
+# Short name (as typed for --reader) -> reader class.
+_BY_NAME: dict[str, type[TabularReader]] = {}
+# Lowercase file extension, leading dot included -> reader class.
+_BY_EXTENSION: dict[str, type[TabularReader]] = {}
+
+_plugins_loaded = False
+
+
+def check_reader_class(obj: object, *, origin: str) -> type[TabularReader]:
+    """Validate that ``obj`` is a class usable as a reader, without calling it.
+
+    Both protocols are checked here, on the class, so that a class which merely
+    *looks* like a reader is rejected before its constructor runs on a user path.
+
+    Args:
+        obj: The candidate, typically freshly imported or freshly registered.
+        origin: How the caller referred to it (a spec, a name, an entry point),
+            used to make the error message actionable.
+
+    Returns:
+        ``obj`` itself, narrowed to a reader class.
+
+    Raises:
+        TypeError: If ``obj`` is not a class, or does not satisfy
+            [TabularReader][dstrack.readers._protocol.TabularReader] and
+            [ReaderFactory][dstrack.readers._protocol.ReaderFactory].
+    """
+    if not isinstance(obj, type):
+        raise TypeError(
+            f"{origin} resolved to {obj!r}, which is not a class. "
+            "A reader must be a class, not an instance or a function."
+        )
+    qualname = f"{obj.__module__}.{obj.__qualname__}"
+    if not issubclass(obj, TabularReader):
+        raise TypeError(
+            f"{qualname} (from {origin}) does not satisfy the TabularReader "
+            "protocol: it must define columns() and iter_batches()."
+        )
+    # Checked through an object-typed name: mypy assumes any class object
+    # already satisfies ReaderFactory, so narrowing from `object` is what keeps
+    # this branch live for the type checker as well as at runtime.
+    candidate: object = obj
+    if not isinstance(candidate, ReaderFactory):
+        raise TypeError(
+            f"{qualname} (from {origin}) does not satisfy the ReaderFactory "
+            "protocol: it must define a from_path(path) classmethod, which is "
+            "how dstrack builds a reader it knows only by name."
+        )
+    # Check that from_path is a class method
+    if not isinstance(inspect.getattr_static(obj, "from_path"), classmethod):
+        raise TypeError(
+            f"{qualname} (from {origin}) defines from_path, but not as a "
+            "classmethod: dstrack calls it on the class (MyReader.from_path(path)), "
+            "so it must be decorated with @classmethod."
+        )
+    return obj
+
+
+def build_reader(reader_cls: type[TabularReader], path: Path) -> TabularReader:
+    """Instantiate a validated reader class for ``path``.
+
+    Args:
+        reader_cls: A class already validated through
+            [check_reader_class()][dstrack.readers._registry.check_reader_class].
+        path: Path to the dataset source.
+
+    Returns:
+        The reader instance.
+
+    Raises:
+        TypeError: If ``reader_cls`` was never validated and lacks ``from_path``,
+            or if its ``from_path`` returns something that is not a
+            [TabularReader][dstrack.readers._protocol.TabularReader].
+    """
+    factory: object = reader_cls
+    if not isinstance(factory, ReaderFactory):
+        raise TypeError(f"{reader_cls!r} is not an instance of {ReaderFactory}")
+    reader: object = factory.from_path(path)
+    if not isinstance(reader, TabularReader):
+        qualname = f"{reader_cls.__module__}.{reader_cls.__qualname__}"
+        raise TypeError(
+            f"{qualname}.from_path returned {reader!r}, which is not a "
+            "TabularReader: from_path must build an object that follows this protocol."
+        )
+    return reader
+
+
+def register_reader(
+    reader_cls: type[TabularReader],
+    *,
+    name: str,
+    extensions: Sequence[str] | None = None,
+) -> None:
+    """Register a reader under a short name and the extensions it handles.
+
+    Args:
+        reader_cls: The reader class. Must satisfy both
+            [TabularReader][dstrack.readers._protocol.TabularReader] and
+            [ReaderFactory][dstrack.readers._protocol.ReaderFactory].
+        name: Short name, as typed for ``--reader`` (e.g. ``"csv"``).
+        extensions: Extensions this reader claims, leading dot included. When
+            omitted, the class's ``EXTENSIONS`` attribute is used, so a plugin
+            can declare its extensions once on the class and register through a
+            bare entry point.
+
+    Raises:
+        TypeError: If ``reader_cls`` does not satisfy both reader protocols.
+        ValueError: If ``name`` or any extension is already taken. Registration
+            never silently displaces an existing reader: two packages fighting
+            over ``.parquet`` is a conflict the user has to see, not one to
+            resolve by import order.
+        ValueError: If any extension is malformed. An extension is matched
+            against [Path.suffix][pathlib.PurePath.suffix], so it must be a
+            single leading-dot suffix such as ``".csv"``: a value like ``"csv"``
+            or ``".tar.gz"`` could never match and is rejected rather than
+            silently registered as dead weight.
+    """
+    check_reader_class(reader_cls, origin=f"reader {name!r}")
+
+    if extensions is None:
+        extensions = getattr(reader_cls, "EXTENSIONS", ())
+
+    if name in _BY_NAME:
+        raise ValueError(
+            f"Reader name {name!r} is already registered to "
+            f"{_BY_NAME[name].__module__}.{_BY_NAME[name].__qualname__}."
+        )
+
+    claimed = [ext.lower() for ext in extensions]
+    for ext in claimed:
+        # An extension is only ever compared against Path.suffix, which is a
+        # single leading-dot component (".csv").
+        if not ext.startswith(".") or ext == "." or Path(f"_{ext}").suffix != ext:
+            raise ValueError(
+                f"Extension {ext!r} (for reader {name!r}) is malformed: an "
+                "extension must be a single leading-dot suffix such as '.csv', "
+                "so that it can match Path.suffix."
+            )
+    for ext in claimed:
+        if ext in _BY_EXTENSION:
+            owner = _BY_EXTENSION[ext]
+            raise ValueError(
+                f"Extension {ext!r} is already registered to "
+                f"{owner.__module__}.{owner.__qualname__}."
+            )
+
+    _BY_NAME[name] = reader_cls
+    for ext in claimed:
+        _BY_EXTENSION[ext] = reader_cls
+
+
+def _load_plugins() -> None:
+    """Register readers advertised by installed packages, once per process.
+
+    A plugin that fails to import or conflicts with an existing registration is
+    logged and skipped: one broken third-party package must not take down
+    `dstrack track` for a dataset that does not even use it.
+    """
+    global _plugins_loaded
+    if _plugins_loaded:
+        return
+    # Set before loading: a plugin that raises must not be retried on every
+    # subsequent lookup.
+    _plugins_loaded = True
+
+    for entry_point in entry_points(group=ENTRY_POINT_GROUP):
+        try:
+            register_reader(entry_point.load(), name=entry_point.name)
+        except Exception as e:
+            _log.warning(
+                f"Ignoring reader plugin {entry_point.name!r} "
+                f"({entry_point.value}): {e}"
+            )
+
+
+def reader_class_for_name(name: str) -> type[TabularReader] | None:
+    """Look up a reader by its short name, or ``None`` if nothing claims it."""
+    _load_plugins()
+    return _BY_NAME.get(name)
+
+
+def reader_class_for_extension(extension: str) -> type[TabularReader] | None:
+    """Look up a reader by file extension, or ``None`` if nothing claims it."""
+    _load_plugins()
+    return _BY_EXTENSION.get(extension.lower())
+
+
+def available_readers() -> dict[str, type[TabularReader]]:
+    """Return all registered readers by short name, built-in and plugin alike."""
+    _load_plugins()
+    return dict(_BY_NAME)
+
+
+def known_extensions() -> dict[str, type[TabularReader]]:
+    """Return all registered extensions and the reader class that claims each."""
+    _load_plugins()
+    return dict(_BY_EXTENSION)
+
+
+register_reader(CsvReader, name="csv")

dstrack/readers/_resolve.py

@@ -0,0 +1,140 @@
+"""Selection and loading of a reader for a given source.
+
+A [TabularReader][dstrack.readers._protocol.TabularReader] is chosen one of three
+ways, in order of how much the user has to type:
+
+1. Implicitly, from the source file's extension (``data.csv`` ->
+   [CsvReader][dstrack.readers._csv.CsvReader]).
+   Plugin readers registered through the ``dstrack.readers`` entry-point group
+   participate here too, so an installed reader needs nothing on the command line.
+2. By short name (``"csv"``), for when the extension is absent or misleading.
+3. By ``"package.module:ClassName"`` spec, which imports the class directly. This
+   is the escape hatch for a reader that lives in the user's own project and is
+   not installed as a plugin.
+
+Security: a spec is arbitrary import-by-name, i.e. code execution. It is only
+ever accepted from the invoking user (a ``--reader`` argument they typed), and
+is never persisted into a snapshot nor read back out of one. See ADR-0003.
+"""
+
+import importlib
+from pathlib import Path
+
+from ._protocol import TabularReader
+from ._registry import (
+    available_readers,
+    build_reader,
+    check_reader_class,
+    known_extensions,
+    reader_class_for_extension,
+    reader_class_for_name,
+)
+
+
+def load_reader_class(spec: str) -> type[TabularReader]:
+    """Import a reader class from a ``"package.module:ClassName"`` spec.
+
+    The class is validated against both reader protocols before being returned,
+    so a mistyped spec that happens to name some other importable object fails
+    here rather than part-way through a snapshot.
+
+    Args:
+        spec: A ``"<module>:<class>"`` string. The module part is imported and
+            the class part is looked up on it.
+
+    Returns:
+        The referenced class (not an instance).
+
+    Raises:
+        ValueError: If ``spec`` is not of the form ``"module:ClassName"``, the
+            module cannot be imported, or the class is not found on it.
+        TypeError: If the referenced object is not a usable reader class.
+    """
+    module_name, sep, class_name = spec.partition(":")
+    if not sep or not module_name or not class_name:
+        raise ValueError(
+            f"Invalid reader spec {spec!r}. "
+            "Expected the form 'package.module:ClassName'."
+        )
+    try:
+        module = importlib.import_module(module_name)
+    except ImportError as e:
+        raise ValueError(
+            f"Could not import module {module_name!r} from reader spec {spec!r}."
+        ) from e
+    try:
+        obj = getattr(module, class_name)
+    except AttributeError as e:
+        raise ValueError(
+            f"Module {module_name!r} has no attribute {class_name!r} "
+            f"(from reader spec {spec!r})."
+        ) from e
+
+    return check_reader_class(obj, origin=f"reader spec {spec!r}")
+
+
+def resolve_reader_class(
+    path: str | Path, *, reader: str | None = None
+) -> type[TabularReader]:
+    """Choose the reader class for ``path``, without instantiating it.
+
+    Args:
+        path: Path to the dataset source.
+        reader: Either a registered short name (``"csv"``) or a
+            ``"package.module:ClassName"`` spec, told apart by the ``":"``.
+            When omitted, the reader is inferred from ``path``'s extension.
+
+    Returns:
+        A validated reader class.
+
+    Raises:
+        ValueError: If ``reader`` names nothing known, a spec is malformed, or
+            no reader is registered for ``path``'s extension.
+        TypeError: If the resolved class is not a usable reader.
+    """
+    if reader is not None:
+        if ":" in reader:
+            return load_reader_class(reader)
+        reader_cls = reader_class_for_name(reader)
+        if reader_cls is None:
+            known = ", ".join(sorted(available_readers())) or "(none)"
+            raise ValueError(
+                f"Unknown reader {reader!r}. Available readers: {known}. "
+                "Pass 'package.module:ClassName' to use a reader that is not "
+                "installed as a plugin."
+            )
+        return reader_cls
+
+    path = Path(path)
+    extension = path.suffix.lower()
+    reader_cls = reader_class_for_extension(extension)
+    if reader_cls is None:
+        known = ", ".join(sorted(known_extensions())) or "(none)"
+        raise ValueError(
+            f"No reader registered for extension {extension!r} (path: {path}). "
+            f"Known extensions: {known}. Name a reader explicitly, either a "
+            "registered one or 'package.module:ClassName'."
+        )
+    return reader_cls
+
+
+def resolve_reader(path: str | Path, *, reader: str | None = None) -> TabularReader:
+    """Build a reader for ``path``, explicitly named or inferred from its extension.
+
+    Args:
+        path: Path to the dataset source the reader will read.
+        reader: Optional short name or ``"package.module:ClassName"`` spec. When
+            omitted, the reader is inferred from ``path``'s file extension.
+
+    Returns:
+        An instantiated reader satisfying
+        [TabularReader][dstrack.readers._protocol.TabularReader].
+
+    Raises:
+        ValueError: If ``reader`` names nothing known, a spec is malformed, or
+            no reader is registered for ``path``'s extension.
+        TypeError: If the resolved class is not a usable reader.
+    """
+    path = Path(path)
+    reader_cls = resolve_reader_class(path, reader=reader)
+    return build_reader(reader_cls, path)

dstrack/schemas/snapshot_v1.json

@@ -0,0 +1,299 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/leoyala/dstrack/schemas/snapshot_v1.json",
+  "title": "DatasetSnapshot",
+  "description": "A dstrack dataset snapshot. Captures semantic information about a tabular dataset at a point in time. See ADR-0001.",
+  "type": "object",
+  "required": [
+    "format_version",
+    "snapshot_id",
+    "created_at",
+    "created_by",
+    "dataset_name",
+    "dataset_path",
+    "source_type",
+    "source_hash",
+    "num_columns",
+    "columns",
+    "schema_hash",
+    "num_rows",
+    "column_stats",
+    "duplicate_row_fraction",
+    "near_duplicate_estimate",
+    "constant_columns",
+    "high_null_columns",
+    "row_minhash",
+    "row_hyperloglog"
+  ],
+  "additionalProperties": false,
+  "properties": {
+    "format_version": {
+      "type": "string",
+      "const": "1",
+      "description": "Schema version. Readers must reject snapshots with an unrecognised version."
+    },
+    "snapshot_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "Unique identifier for this snapshot."
+    },
+    "created_at": {
+      "type": "string",
+      "format": "date-time",
+      "description": "ISO-8601 timestamp of when the snapshot was taken."
+    },
+    "created_by": {
+      "type": "string",
+      "description": "User or process that created the snapshot."
+    },
+    "dataset_name": {
+      "type": "string",
+      "description": "Human-readable name for the dataset."
+    },
+    "dataset_path": {
+      "type": "string",
+      "description": "Source path or URI at snapshot time."
+    },
+    "source_type": {
+      "type": "string",
+      "examples": ["file", "directory", "database", "huggingface", "s3"],
+      "description": "Origin kind of the dataset source."
+    },
+    "source_hash": {
+      "type": "string",
+      "description": "Cryptographic hash of the raw source bytes, or a Merkle tree hash for directories."
+    },
+    "description": {
+      "type": "string",
+      "description": "Free-text description of what this dataset represents."
+    },
+    "tags": {
+      "type": "object",
+      "additionalProperties": { "type": "string" },
+      "description": "Arbitrary key-value labels for filtering and organisation."
+    },
+    "num_columns": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Total number of columns/features."
+    },
+    "columns": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/ColumnDescriptor" },
+      "description": "Ordered list of column descriptors."
+    },
+    "schema_hash": {
+      "type": "string",
+      "description": "Hash of (name, dtype) tuples in column order. Changes whenever the schema changes."
+    },
+    "num_rows": {
+      "type": "integer",
+      "minimum": 0,
+      "description": "Total number of records."
+    },
+    "column_stats": {
+      "type": "object",
+      "additionalProperties": { "$ref": "#/$defs/ColumnStats" },
+      "description": "Per-column statistics keyed by column name. The concrete shape depends on the column dtype; users must branch on the corresponding entry in `columns`."
+    },
+    "duplicate_row_fraction": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "description": "Estimated fraction of exact-duplicate rows."
+    },
+    "near_duplicate_estimate": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "description": "Approximate fraction of near-duplicate rows via MinHash/LSH."
+    },
+    "constant_columns": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Names of columns with zero variance / a single unique value."
+    },
+    "high_null_columns": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "Names of columns where null_fraction > 0.5."
+    },
+    "row_minhash": {
+      "type": "string",
+      "contentEncoding": "base64",
+      "description": "Base64-encoded MinHash signature of the row set. Enables Jaccard similarity between snapshots."
+    },
+    "row_hyperloglog": {
+      "type": "string",
+      "contentEncoding": "base64",
+      "description": "Base64-encoded HyperLogLog sketch for cardinality estimation of unique rows."
+    },
+    "parent_snapshot_id": {
+      "type": "string",
+      "format": "uuid",
+      "description": "ID of the preceding snapshot of the same dataset, if one exists."
+    },
+    "pipeline_stage": {
+      "type": "string",
+      "enum": ["raw", "cleaned", "featurized", "augmented", "sampled", "final"],
+      "description": "Stage in the data pipeline at which this snapshot was taken."
+    },
+    "transform_description": {
+      "type": "string",
+      "description": "Human-readable description of the changes applied since the parent snapshot."
+    }
+  },
+  "$defs": {
+    "ColumnDescriptor": {
+      "type": "object",
+      "required": ["name", "dtype", "nullable"],
+      "additionalProperties": false,
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Column identifier."
+        },
+        "dtype": {
+          "type": "string",
+          "examples": [
+            "int8", "int16", "int32", "int64",
+            "float16", "float32", "float64",
+            "string", "bool", "datetime64", "bytes"
+          ],
+          "description": "Storage type of the column."
+        },
+        "nullable": {
+          "type": "boolean",
+          "description": "Whether null values are structurally allowed."
+        }
+      }
+    },
+    "ColumnStats": {
+      "oneOf": [
+        { "$ref": "#/$defs/NumericColumnStats" },
+        { "$ref": "#/$defs/StringColumnStats" },
+        { "$ref": "#/$defs/DatetimeColumnStats" }
+      ]
+    },
+    "NumericColumnStats": {
+      "description": "Statistics for int* and float* columns.",
+      "type": "object",
+      "required": [
+        "null_count", "null_fraction",
+        "mean", "std", "min", "max",
+        "percentiles", "histogram", "num_unique"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "null_count": { "type": "integer", "minimum": 0 },
+        "null_fraction": { "type": "number", "minimum": 0, "maximum": 1 },
+        "mean": { "type": "number" },
+        "std": { "type": "number", "minimum": 0 },
+        "min": { "type": "number" },
+        "max": { "type": "number" },
+        "percentiles": {
+          "type": "object",
+          "required": ["p5", "p25", "p50", "p75", "p95", "p99"],
+          "additionalProperties": false,
+          "properties": {
+            "p5":  { "type": "number" },
+            "p25": { "type": "number" },
+            "p50": { "type": "number" },
+            "p75": { "type": "number" },
+            "p95": { "type": "number" },
+            "p99": { "type": "number" }
+          }
+        },
+        "histogram": {
+          "type": "object",
+          "required": ["bin_edges", "counts"],
+          "additionalProperties": false,
+          "properties": {
+            "bin_edges": {
+              "type": "array",
+              "items": { "type": "number" },
+              "description": "Length is num_bins + 1."
+            },
+            "counts": {
+              "type": "array",
+              "items": { "type": "integer", "minimum": 0 },
+              "description": "Length is num_bins."
+            }
+          }
+        },
+        "num_unique": {
+          "type": "integer",
+          "minimum": 0,
+          "description": "Exact count if cheap to compute, otherwise a sketch estimate."
+        }
+      }
+    },
+    "StringColumnStats": {
+      "description": "Statistics for string columns.",
+      "type": "object",
+      "required": [
+        "null_count", "null_fraction",
+        "num_unique", "top_values", "top_values_coverage",
+        "avg_char_length", "min_char_length", "max_char_length",
+        "avg_token_count", "min_token_count", "max_token_count"
+      ],
+      "additionalProperties": false,
+      "properties": {
+        "null_count": { "type": "integer", "minimum": 0 },
+        "null_fraction": { "type": "number", "minimum": 0, "maximum": 1 },
+        "num_unique": { "type": "integer", "minimum": 0, "description": "Cardinality." },
+        "top_values": {
+          "type": "object",
+          "additionalProperties": { "type": "integer", "minimum": 0 },
+          "description": "Top-K {value: count} pairs (default K=50)."
+        },
+        "top_values_coverage": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Fraction of rows covered by the top_values entries."
+        },
+        "avg_char_length": { "type": "number", "minimum": 0 },
+        "min_char_length": { "type": "integer", "minimum": 0 },
+        "max_char_length": { "type": "integer", "minimum": 0 },
+        "avg_token_count": {
+          "type": "number",
+          "minimum": 0,
+          "description": "Whitespace-split token count, averaged over non-null rows."
+        },
+        "min_token_count": { "type": "integer", "minimum": 0 },
+        "max_token_count": { "type": "integer", "minimum": 0 },
+        "detected_languages": {
+          "type": "object",
+          "additionalProperties": { "type": "number", "minimum": 0, "maximum": 1 },
+          "description": "Top detected BCP-47 language codes mapped to their row fraction."
+        }
+      }
+    },
+    "DatetimeColumnStats": {
+      "description": "Statistics for datetime64 columns.",
+      "type": "object",
+      "required": ["null_count", "null_fraction", "min", "max", "range_days"],
+      "additionalProperties": false,
+      "properties": {
+        "null_count": { "type": "integer", "minimum": 0 },
+        "null_fraction": { "type": "number", "minimum": 0, "maximum": 1 },
+        "min": {
+          "type": "string",
+          "format": "date-time",
+          "description": "ISO-8601 lower boundary."
+        },
+        "max": {
+          "type": "string",
+          "format": "date-time",
+          "description": "ISO-8601 upper boundary."
+        },
+        "range_days": {
+          "type": "number",
+          "minimum": 0,
+          "description": "(max - min) expressed in days."
+        }
+      }
+    }
+  }
+}