dsdiff 0.2.0__tar.gz

dsdiff-0.2.0/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: ci
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Sync dependencies
+        run: uv sync --all-extras --dev
+      - name: Lint
+        run: uv run ruff check .
+      - name: Format check
+        run: uv run ruff format --check .
+      - name: Test
+        run: uv run pytest --cov --cov-report=term-missing

dsdiff-0.2.0/.github/workflows/publish.yml

@@ -0,0 +1,21 @@
+name: publish
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  pypi:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+      - name: Build
+        run: uv build
+      - name: Publish to PyPI
+        run: uv publish

dsdiff-0.2.0/.gitignore

@@ -0,0 +1,26 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Virtual environments
+.venv/
+venv/
+
+# uv
+uv.lock
+
+# Test and coverage
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.ruff_cache/
+
+# Editor / OS
+.vscode/
+.idea/
+.DS_Store

dsdiff-0.2.0/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-toml
+      - id: check-merge-conflict

dsdiff-0.2.0/CHANGELOG.md

@@ -0,0 +1,30 @@
+# Changelog
+
+All notable changes to this project are documented here. The format is based
+on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project
+adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.0] - 2026-06-08
+
+### Added
+- `--fail-on medium` to tighten the `--check` gate beyond high-severity only.
+- Markdown output for pasting a diff into a pull request.
+- Docker image and a published container entry point.
+- Continuous integration across Python 3.10, 3.11 and 3.12.
+
+### Fixed
+- Drift is no longer missed when a column shifts entirely past the baseline
+  range; out-of-range values are clamped to the edge bins.
+
+## [0.1.0] - 2026-06-03
+
+### Added
+- `diff` command: schema changes (added, removed, retyped columns), null-rate
+  jumps, cardinality changes, and per-column distribution drift via PSI.
+- `profile` command: write a committable baseline profile with bin edges so a
+  later diff measures drift against the same buckets.
+- CSV, Parquet and JSON Lines input through polars.
+- `--json` output and a `--check` CI gate.
+
+[0.2.0]: https://github.com/jmweb-org/dsdiff/releases/tag/v0.2.0
+[0.1.0]: https://github.com/jmweb-org/dsdiff/releases/tag/v0.1.0

dsdiff-0.2.0/Dockerfile

@@ -0,0 +1,19 @@
+# Diff two datasets from inside a container by mounting them:
+#
+#   docker build -t dsdiff .
+#   docker run --rm -v "$PWD:/w" -w /w dsdiff diff a.parquet b.parquet
+#
+FROM python:3.12-slim
+
+LABEL org.opencontainers.image.source="https://github.com/jmweb-org/dsdiff"
+LABEL org.opencontainers.image.description="Diff two dataset files: schema changes plus column-level distribution drift."
+LABEL org.opencontainers.image.licenses="MIT"
+
+WORKDIR /app
+COPY pyproject.toml README.md LICENSE ./
+COPY src ./src
+
+RUN pip install --no-cache-dir .
+
+ENTRYPOINT ["dsdiff"]
+CMD ["--help"]

dsdiff-0.2.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 José del Río
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

dsdiff-0.2.0/PKG-INFO

@@ -0,0 +1,138 @@
+Metadata-Version: 2.4
+Name: dsdiff
+Version: 0.2.0
+Summary: Diff two dataset files: schema changes plus column-level distribution drift.
+Project-URL: Homepage, https://github.com/jmweb-org/dsdiff
+Project-URL: Repository, https://github.com/jmweb-org/dsdiff
+Project-URL: Issues, https://github.com/jmweb-org/dsdiff/issues
+Author: José del Río
+License: MIT License
+        
+        Copyright (c) 2026 José del Río
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Keywords: cli,data,dataset,diff,drift,mlops,psi,schema
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Utilities
+Requires-Python: >=3.10
+Requires-Dist: numpy>=1.24
+Requires-Dist: polars>=1.0
+Requires-Dist: rich>=13.0
+Requires-Dist: typer>=0.12
+Description-Content-Type: text/markdown
+
+# dsdiff
+
+[![CI](https://github.com/jmweb-org/dsdiff/actions/workflows/ci.yml/badge.svg)](https://github.com/jmweb-org/dsdiff/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/dsdiff.svg)](https://pypi.org/project/dsdiff/)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+A git-style diff between two dataset files: schema changes and column-level
+distribution drift, with a CI gate.
+
+When a dataset is regenerated, columns quietly get renamed, retyped, gain
+nulls, or shift distribution, and the pipeline keeps running while the model
+degrades. There is no quick "git diff for data" a reviewer can read on a pull
+request. `dsdiff` is that: point it at two files and it reports what changed,
+ranked by how much it should worry you.
+
+```console
+$ dsdiff diff yesterday.parquet today.parquet
+severity  column      change         detail
+high      income      drift          PSI 0.412
+high      signup_date column_added   new column
+medium    age          null_rate      null rate 0.0% -> 7.3%
+low       country      cardinality    distinct values 41 -> 44
+```
+
+## Install
+
+```console
+$ pip install dsdiff                 # from PyPI, once released
+$ pip install git+https://github.com/jmweb-org/dsdiff   # latest, available now
+```
+
+Reads CSV, Parquet and JSON Lines through polars. No services, no schema files
+to author.
+
+## Usage
+
+```console
+$ dsdiff diff a.csv b.csv            # human-readable table
+$ dsdiff diff a.csv b.csv --json     # machine-readable findings
+$ dsdiff diff a.csv b.csv --markdown # a table to paste into a PR
+$ dsdiff diff a.csv b.csv --check    # exit non-zero on a high-severity change
+```
+
+### Commit a baseline
+
+Profile a dataset once and compare future data against the saved profile,
+without re-reading the original file:
+
+```console
+$ dsdiff profile reference.parquet -o baseline.json
+$ dsdiff diff baseline.json new_batch.parquet --check
+```
+
+The baseline stores the bin edges, so drift on `new_batch` is measured against
+exactly the same buckets as the reference.
+
+### In CI
+
+```yaml
+- run: dsdiff diff baseline.json data/current.parquet --check
+```
+
+## What it checks
+
+- **Schema**: columns added, removed, or retyped (all high severity).
+- **Nulls**: a jump in the null rate of a shared column.
+- **Cardinality**: a categorical column gaining or losing distinct values.
+- **Distribution drift**: the population stability index (PSI) per column,
+  numeric columns binned by quantiles and categoricals by frequency.
+
+## Severity and the PSI scale
+
+PSI is the standard tabular drift measure. The conventional reading is used
+here: below 0.1 is **low** (no meaningful shift), 0.1 to 0.25 is **medium**,
+and 0.25 or above is **high**. Schema and type changes are always high. By
+default `--check` fails only on high-severity findings; pass `--fail-on medium`
+to tighten the gate.
+
+## Exit codes
+
+| Code | Meaning |
+| --- | --- |
+| 0 | Ran; no blocking finding (or `--check` not set) |
+| 1 | `--check` found a finding at or above the fail threshold |
+| 2 | A file was missing or in an unsupported format |
+
+## License
+
+MIT. See [LICENSE](LICENSE).

dsdiff-0.2.0/README.md

@@ -0,0 +1,90 @@
+# dsdiff
+
+[![CI](https://github.com/jmweb-org/dsdiff/actions/workflows/ci.yml/badge.svg)](https://github.com/jmweb-org/dsdiff/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/dsdiff.svg)](https://pypi.org/project/dsdiff/)
+[![Python](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org)
+[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
+A git-style diff between two dataset files: schema changes and column-level
+distribution drift, with a CI gate.
+
+When a dataset is regenerated, columns quietly get renamed, retyped, gain
+nulls, or shift distribution, and the pipeline keeps running while the model
+degrades. There is no quick "git diff for data" a reviewer can read on a pull
+request. `dsdiff` is that: point it at two files and it reports what changed,
+ranked by how much it should worry you.
+
+```console
+$ dsdiff diff yesterday.parquet today.parquet
+severity  column      change         detail
+high      income      drift          PSI 0.412
+high      signup_date column_added   new column
+medium    age          null_rate      null rate 0.0% -> 7.3%
+low       country      cardinality    distinct values 41 -> 44
+```
+
+## Install
+
+```console
+$ pip install dsdiff                 # from PyPI, once released
+$ pip install git+https://github.com/jmweb-org/dsdiff   # latest, available now
+```
+
+Reads CSV, Parquet and JSON Lines through polars. No services, no schema files
+to author.
+
+## Usage
+
+```console
+$ dsdiff diff a.csv b.csv            # human-readable table
+$ dsdiff diff a.csv b.csv --json     # machine-readable findings
+$ dsdiff diff a.csv b.csv --markdown # a table to paste into a PR
+$ dsdiff diff a.csv b.csv --check    # exit non-zero on a high-severity change
+```
+
+### Commit a baseline
+
+Profile a dataset once and compare future data against the saved profile,
+without re-reading the original file:
+
+```console
+$ dsdiff profile reference.parquet -o baseline.json
+$ dsdiff diff baseline.json new_batch.parquet --check
+```
+
+The baseline stores the bin edges, so drift on `new_batch` is measured against
+exactly the same buckets as the reference.
+
+### In CI
+
+```yaml
+- run: dsdiff diff baseline.json data/current.parquet --check
+```
+
+## What it checks
+
+- **Schema**: columns added, removed, or retyped (all high severity).
+- **Nulls**: a jump in the null rate of a shared column.
+- **Cardinality**: a categorical column gaining or losing distinct values.
+- **Distribution drift**: the population stability index (PSI) per column,
+  numeric columns binned by quantiles and categoricals by frequency.
+
+## Severity and the PSI scale
+
+PSI is the standard tabular drift measure. The conventional reading is used
+here: below 0.1 is **low** (no meaningful shift), 0.1 to 0.25 is **medium**,
+and 0.25 or above is **high**. Schema and type changes are always high. By
+default `--check` fails only on high-severity findings; pass `--fail-on medium`
+to tighten the gate.
+
+## Exit codes
+
+| Code | Meaning |
+| --- | --- |
+| 0 | Ran; no blocking finding (or `--check` not set) |
+| 1 | `--check` found a finding at or above the fail threshold |
+| 2 | A file was missing or in an unsupported format |
+
+## License
+
+MIT. See [LICENSE](LICENSE).

dsdiff-0.2.0/pyproject.toml

@@ -0,0 +1,72 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dsdiff"
+version = "0.2.0"
+description = "Diff two dataset files: schema changes plus column-level distribution drift."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { file = "LICENSE" }
+authors = [{ name = "José del Río" }]
+keywords = ["data", "dataset", "drift", "psi", "schema", "diff", "mlops", "cli"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Utilities",
+]
+dependencies = [
+    "typer>=0.12",
+    "rich>=13.0",
+    "polars>=1.0",
+    "numpy>=1.24",
+]
+
+[project.urls]
+Homepage = "https://github.com/jmweb-org/dsdiff"
+Repository = "https://github.com/jmweb-org/dsdiff"
+Issues = "https://github.com/jmweb-org/dsdiff/issues"
+
+[project.scripts]
+dsdiff = "dsdiff.cli:entrypoint"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.6",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dsdiff"]
+
+[tool.pytest.ini_options]
+addopts = "-q"
+testpaths = ["tests"]
+pythonpath = ["."]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "S", "C4", "RUF"]
+
+[tool.ruff.lint.flake8-bugbear]
+extend-immutable-calls = ["typer.Argument", "typer.Option"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["S101"]
+
+[tool.coverage.run]
+source = ["dsdiff"]
+branch = true

dsdiff-0.2.0/src/dsdiff/__init__.py

@@ -0,0 +1,19 @@
+"""dsdiff: diff two dataset files by schema and column-level drift."""
+
+from dsdiff.compare import Finding, FindingKind, compare_files, compare_profiles
+from dsdiff.dataset import DatasetProfile, profile_file
+from dsdiff.drift import Severity, psi_from_counts
+
+__version__ = "0.2.0"
+
+__all__ = [
+    "DatasetProfile",
+    "Finding",
+    "FindingKind",
+    "Severity",
+    "__version__",
+    "compare_files",
+    "compare_profiles",
+    "profile_file",
+    "psi_from_counts",
+]

dsdiff-0.2.0/src/dsdiff/__main__.py

@@ -0,0 +1,4 @@
+from dsdiff.cli import entrypoint
+
+if __name__ == "__main__":
+    entrypoint()

dsdiff-0.2.0/src/dsdiff/cli.py

@@ -0,0 +1,114 @@
+"""Command-line interface for dsdiff."""
+
+from __future__ import annotations
+
+import json
+import sys
+from enum import Enum
+from pathlib import Path
+
+import typer
+from rich.console import Console
+
+from dsdiff import __version__
+from dsdiff.compare import compare_files, has_blocking
+from dsdiff.dataset import profile_file
+from dsdiff.drift import Severity
+from dsdiff.render import findings_to_json, render_markdown, render_terminal
+
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    help="Diff two dataset files: schema changes plus column-level drift.",
+)
+_out = Console()
+_err = Console(stderr=True)
+
+EXIT_OK = 0
+EXIT_BLOCKING = 1
+EXIT_BAD_INPUT = 2
+
+
+class FailOn(str, Enum):
+    high = "high"
+    medium = "medium"
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        _out.print(f"dsdiff {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    _version: bool = typer.Option(
+        False,
+        "--version",
+        callback=_version_callback,
+        is_eager=True,
+        help="Show the version and exit.",
+    ),
+) -> None:
+    """dsdiff command-line interface."""
+
+
+@app.command("diff")
+def diff(
+    baseline: Path = typer.Argument(..., help="Baseline dataset, or a saved profile JSON."),
+    candidate: Path = typer.Argument(..., help="Dataset to compare against the baseline."),
+    as_json: bool = typer.Option(False, "--json", help="Emit findings as JSON."),
+    markdown: bool = typer.Option(False, "--markdown", help="Emit a Markdown table."),
+    check: bool = typer.Option(False, "--check", help="Exit non-zero on blocking findings."),
+    fail_on: FailOn = typer.Option(
+        FailOn.high, "--fail-on", help="Severity that --check treats as blocking."
+    ),
+) -> None:
+    """Compare two datasets and report schema and distribution changes."""
+
+    try:
+        findings = compare_files(baseline, candidate)
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        _err.print(f"dsdiff: {exc}")
+        raise typer.Exit(EXIT_BAD_INPUT) from exc
+
+    if as_json:
+        _out.print_json(json.dumps(findings_to_json(findings)))
+    elif markdown:
+        _out.print(render_markdown(findings))
+    else:
+        _out.print(render_terminal(findings))
+
+    threshold = Severity.HIGH if fail_on is FailOn.high else Severity.MEDIUM
+    if check and has_blocking(findings, threshold):
+        raise typer.Exit(EXIT_BLOCKING)
+
+
+@app.command("profile")
+def profile(
+    dataset: Path = typer.Argument(..., help="Dataset to profile."),
+    output: Path | None = typer.Option(
+        None, "-o", "--output", help="Write the profile JSON here (default: stdout)."
+    ),
+) -> None:
+    """Write a committable baseline profile of a dataset."""
+
+    try:
+        prof = profile_file(dataset)
+    except (OSError, ValueError) as exc:
+        _err.print(f"dsdiff: {exc}")
+        raise typer.Exit(EXIT_BAD_INPUT) from exc
+    payload = json.dumps(prof.to_dict(), indent=2)
+    if output is None:
+        _out.print_json(payload)
+    else:
+        Path(output).write_text(payload + "\n", encoding="utf-8")
+        _err.print(f"dsdiff: wrote {output}")
+
+
+def entrypoint() -> None:
+    try:
+        app()
+    except KeyboardInterrupt:  # pragma: no cover - interactive only
+        print("dsdiff: interrupted", file=sys.stderr)
+        raise SystemExit(130) from None