dvc-utils 0.3.0__tar.gz → 0.3.2__tar.gz

dvc_utils-0.3.2/.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+name: Verify README examples, release to PyPI
+on:
+  push:
+    branches: [ "main" ]
+    tags: [ "v**" ]
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+jobs:
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: Install parquet2json
+        run: |
+          if ! command -v parquet2json &> /dev/null; then
+            # Use cargo-binstall for faster installation if available
+            if ! command -v cargo-binstall &> /dev/null; then
+              curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
+            fi
+            cargo binstall -y parquet2json || cargo install parquet2json
+          else
+            echo "parquet2json already installed: $(parquet2json --version)"
+          fi
+      - name: Install dependencies
+        run: uv sync --extra ci --extra test
+      - name: Run pytest
+        run: |
+          source .venv/bin/activate
+          pytest
+      - name: '`dvc pull` test/data'
+        working-directory: test/data
+        run: |
+          source ../../.venv/bin/activate
+          dvc pull -r s3 -R -A
+      - name: Set up parquet-helpers
+        uses: actions/checkout@v4
+        with:
+          repository: ryan-williams/parquet-helpers
+          path: pqt
+      - name: Verify README examples
+        env:
+          # Evaluate README examples from within the `test/data` submodule
+          BMDF_WORKDIR: test/data
+        run: |
+          source .venv/bin/activate
+          export PATH="$PWD/pqt:$PATH"
+          . pqt/.pqt-rc
+          export SHELL
+          mdcmd
+          git diff --exit-code
+  release:
+    name: Release to PyPI
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - name: Build package
+        run: uv build
+      - name: Publish to PyPI
+        run: uv publish --username __token__ --password ${{ secrets.PYPI_TOKEN }}
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create ${{ github.ref_name }} \
+            --title "${{ github.ref_name }}" \
+            --generate-notes

dvc_utils-0.3.2/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/data"]
+	path = test/data
+	url = https://github.com/ryan-williams/dvc-helpers

dvc_utils-0.3.2/CLAUDE.md

@@ -0,0 +1,130 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+`dvc-utils` is a Python CLI tool for diffing DVC-tracked files between commits or against the worktree. It can optionally pipe file contents through other commands before diffing (e.g., `parquet2json`, `gunzip`, etc.), enabling semantic diffs of binary or compressed formats.
+
+Core functionality: Compare DVC-tracked files at different Git commits by resolving their MD5 hashes from `.dvc` files, looking up cached content in the DVC cache, and piping through optional preprocessing commands before diffing.
+
+## Build & Development Commands
+
+This project uses `uv` for dependency management:
+
+```bash
+# Initialize project (first time setup)
+uv init
+
+# Install dependencies
+uv sync
+
+# Install with test dependencies
+uv sync --extra test
+
+# Run tests
+pytest
+
+# Run tests (with activated venv)
+source .venv/bin/activate
+pytest
+
+# Build the package
+uv build
+
+# Install in development mode
+pip install -e .
+```
+
+## Testing
+
+Tests are in `tests/` directory. The test suite uses a DVC-tracked test data repository at `test/data` (submodule of [ryan-williams/dvc-helpers@test]).
+
+Key test commands:
+```bash
+# Run all tests
+pytest
+
+# Run specific test file
+pytest tests/test_diff_exit_codes.py
+
+# Run tests with verbose output (already configured in pytest.ini)
+pytest -v
+```
+
+The test data directory requires DVC pull:
+```bash
+cd test/data
+dvc pull -r s3 -R -A
+```
+
+## Architecture
+
+### Core Module Structure
+
+- **`src/dvc_utils/cli.py`**: Minimal Click group definition (entry point)
+- **`src/dvc_utils/main.py`**: Main entry point that invokes the CLI
+- **`src/dvc_utils/diff.py`**: Core `dvc-diff` command implementation
+  - Parses refspecs (e.g., `HEAD^..HEAD` or single commit)
+  - Resolves DVC-tracked file paths to cache paths via MD5 lookups
+  - Handles both individual files and DVC-tracked directories
+  - Uses `dffs.join_pipelines` to run preprocessing commands on both sides of diff
+- **`src/dvc_utils/path.py`**: DVC path resolution utilities
+  - `dvc_paths()`: Normalizes path/dvc_path pairs
+  - `dvc_md5()`: Extracts MD5 from `.dvc` file at specific Git ref
+  - `dvc_cache_path()`: Resolves MD5 to actual cache file path
+  - `dvc_cache_dir()`: Finds DVC cache directory (respects `DVC_UTILS_CACHE_DIR` env var)
+- **`src/dvc_utils/sync.py`**: Incomplete `pull-x` command (not currently functional)
+
+### Key Dependencies
+
+- **`dffs`**: Provides `join_pipelines()` for executing parallel command pipelines and diffing their outputs
+- **`utz`**: Utility library with subprocess wrappers (`utz.process`), error printing (`err`), and file hashing
+- **`click`**: CLI framework
+- **`pyyaml`**: For parsing `.dvc` YAML files
+
+### How DVC Diffing Works
+
+1. Parse the path argument to get both the data path and `.dvc` file path
+2. For each side of the diff (before/after commits):
+   - Use `git show <ref>:<path>.dvc` to get the DVC file content at that ref
+   - Parse the YAML to extract the MD5 hash
+   - Construct cache path: `<cache_dir>/files/md5/<first_2_chars>/<remaining_chars>`
+3. If preprocessing commands are specified (e.g., `dvc-diff wc -l foo.dvc`):
+   - Use `dffs.join_pipelines()` to run commands on both cache files and diff the outputs
+4. Otherwise, run `diff` directly on the two cache files
+
+### DVC Directory Handling
+
+For DVC-tracked directories (`.dvc` files with multiple entries), the tool:
+- Parses the directory's `.dvc` file JSON structure
+- Compares MD5 hashes for each file in the directory
+- Outputs changes as: `filename: <old_md5> -> <new_md5>`
+
+### Exit Code Behavior
+
+The tool propagates exit codes correctly:
+- `0`: No differences found
+- `1`: Differences found (standard `diff` behavior)
+- `>1`: Error in preprocessing pipeline or diff execution
+
+## Entry Points
+
+Defined in `pyproject.toml`:
+- `dvc-utils`: Main entry point (currently just invokes CLI group)
+- `dvc-diff`: Direct entry to diff command
+
+## Environment Variables
+
+- `DVC_UTILS_CACHE_DIR`: Override DVC cache directory location (relative to git root)
+- `SHELL`: Used as default shell for executing preprocessing commands
+- `BMDF_WORKDIR`: Used in CI for running README example verification from subdirectory
+
+## CI/CD Notes
+
+- Uses GitHub Actions (`.github/workflows/ci.yml`)
+- Tests run on every push/PR to main
+- Releases to PyPI on version tags (`v**`)
+- README examples are verified using `bmdf` (bash-markdown-fence) in the `test/data` directory
+- Requires `parquet2json` (Rust tool) for Parquet-related examples
+- AWS credentials needed for DVC remote access in tests

{dvc_utils-0.3.0/src/dvc_utils.egg-info → dvc_utils-0.3.2}/PKG-INFO

@@ -1,22 +1,23 @@
 Metadata-Version: 2.4
 Name: dvc-utils
-Version: 0.3.0
+Version: 0.3.2
 Summary: CLI for diffing DVC-tracked files at two commits (or one commit vs. current worktree), optionally passing both through another command first
-Author-email: Ryan Williams <ryan@runsascoded.com>
-License: MIT
 Project-URL: Homepage, https://github.com/runsascoded/dvc-utils
 Project-URL: Author URL, https://github.com/ryan-williams
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
+Author-email: Ryan Williams <ryan@runsascoded.com>
+License: MIT
 License-File: LICENSE
+Requires-Python: >=3.10
 Requires-Dist: click
-Requires-Dist: dffs>=0.0.5
+Requires-Dist: dffs>=0.0.7
 Requires-Dist: pyyaml
 Requires-Dist: utz>=0.20.0
 Provides-Extra: ci
-Requires-Dist: bmdf==0.5.2; extra == "ci"
-Requires-Dist: dvc-s3; extra == "ci"
-Dynamic: license-file
+Requires-Dist: bmdf==0.5.2; extra == 'ci'
+Requires-Dist: dvc-s3; extra == 'ci'
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == 'test'
+Description-Content-Type: text/markdown
 
 # dvc-utils
 Diff [DVC] files, optionally piping through other commands first.
@@ -61,6 +62,10 @@ dvc-diff
 #   optional) at HEAD (last committed value) vs. the current worktree content.
 #
 # Options:
+#   -b, --both / -B, --no-both    Merge stderr into stdout in pipeline commands
+#                                 (like shell `2>&1`), so stderr is included in
+#                                 diff output. Default: stderr shown only on
+#                                 command failures.
 #   -c, --color / -C, --no-color  Force or prevent colorized output
 #   -r, --refspec TEXT            <commit 1>..<commit 2> (compare two commits)
 #                                 or <commit> (compare <commit> to the worktree)

{dvc_utils-0.3.0 → dvc_utils-0.3.2}/README.md

@@ -41,6 +41,10 @@ dvc-diff
 #   optional) at HEAD (last committed value) vs. the current worktree content.
 #
 # Options:
+#   -b, --both / -B, --no-both    Merge stderr into stdout in pipeline commands
+#                                 (like shell `2>&1`), so stderr is included in
+#                                 diff output. Default: stderr shown only on
+#                                 command failures.
 #   -c, --color / -C, --no-color  Force or prevent colorized output
 #   -r, --refspec TEXT            <commit 1>..<commit 2> (compare two commits)
 #                                 or <commit> (compare <commit> to the worktree)

{dvc_utils-0.3.0 → dvc_utils-0.3.2}/pyproject.toml

@@ -1,20 +1,20 @@
 [build-system]
-requires = ["setuptools>=75"]
-build-backend = "setuptools.build_meta"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 
 [project]
 name = "dvc-utils"
-version = "0.3.0"
+version = "0.3.2"
 description = "CLI for diffing DVC-tracked files at two commits (or one commit vs. current worktree), optionally passing both through another command first"
 readme = "README.md"
 license = {text = "MIT"}
 authors = [
     {name = "Ryan Williams", email = "ryan@runsascoded.com"}
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "click",
-    "dffs>=0.0.5",
+    "dffs>=0.0.7",
     "pyyaml",
     "utz>=0.20.0",
 ]
@@ -24,6 +24,9 @@ ci = [
     "bmdf==0.5.2",
     "dvc-s3",
 ]
+test = [
+    "pytest>=7.0.0",
+]
 
 [project.urls]
 Homepage = "https://github.com/runsascoded/dvc-utils"
@@ -33,8 +36,10 @@ Homepage = "https://github.com/runsascoded/dvc-utils"
 dvc-utils = "dvc_utils.main:main"
 dvc-diff = "dvc_utils.diff:dvc_diff"
 
-[tool.setuptools]
-package-dir = {"" = "src"}
+[dependency-groups]
+dev = [
+    "pytest>=7.0.0",
+]
 
-[tool.setuptools.packages.find]
-where = ["src"]
+[tool.hatch.build.targets.wheel]
+packages = ["src/dvc_utils"]

dvc_utils-0.3.2/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = -v --tb=short

{dvc_utils-0.3.0 → dvc_utils-0.3.2}/src/dvc_utils/diff.py

@@ -18,6 +18,7 @@ from dvc_utils.path import dvc_paths, dvc_cache_path
     short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first',
     no_args_is_help=True,
 )
+@option('-b/-B', '--both/--no-both', default=False, help='Merge stderr into stdout in pipeline commands (like shell `2>&1`), so stderr is included in diff output. Default: stderr shown only on command failures.')
 @option('-c/-C', '--color/--no-color', default=None, help='Force or prevent colorized output')
 @option('-r', '--refspec', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
 @option('-R', '--ref', help='Shorthand for `-r <ref>^..<ref>`, i.e. inspect a specific commit (vs. its parent)')
@@ -29,6 +30,7 @@ from dvc_utils.path import dvc_paths, dvc_cache_path
 @option('-x', '--exec-cmd', 'exec_cmds', multiple=True, help='Command(s) to execute before diffing; alternate syntax to passing commands as positional arguments')
 @argument('args', metavar='[exec_cmd...] <path>', nargs=-1)
 def dvc_diff(
+    both: bool,
     color: bool | None,
     refspec: str | None,
     ref: str | None,
@@ -111,14 +113,16 @@ def dvc_diff(
                 cmds1 = [ shlex.split(cmd) for cmd in cmds1 ]
                 cmds2 = [ shlex.split(cmd) for cmd in cmds2 ]
 
-            join_pipelines(
+            returncode = join_pipelines(
                 base_cmd=['diff', *diff_args],
                 cmds1=cmds1,
                 cmds2=cmds2,
                 verbose=verbose,
                 shell=shell,
                 executable=shell_executable,
+                both=both,
             )
+            exit(returncode)
         else:
             res = process.run('diff', *diff_args, path1 or '/dev/null', path2 or '/dev/null', log=log, check=False)
             exit(res.returncode)

{dvc_utils-0.3.0 → dvc_utils-0.3.2}/src/dvc_utils/path.py

@@ -60,7 +60,8 @@ def dvc_md5(
         relpath = basename(dvc_path)
         if relpath.endswith(".dvc"):
             relpath = relpath[:-len(".dvc")]
-        while cur_dir and cur_dir != '.':
+        prev_dir = None
+        while cur_dir and cur_dir != '.' and cur_dir != prev_dir:
             dir_cache_path = dvc_cache_path(ref=git_ref, dvc_path=f"{cur_dir}.dvc", log=log)
             if dir_cache_path:
                 with open(dir_cache_path, 'r') as f:
@@ -71,6 +72,7 @@ def dvc_md5(
                 else:
                     raise RuntimeError(f"{relpath=} not found in DVC-tracked dir {cur_dir}")
             relpath = join(basename(cur_dir), relpath)
+            prev_dir = cur_dir
             cur_dir = dirname(cur_dir)
         return None
     dvc_obj = yaml.safe_load(dvc_spec)

dvc_utils-0.3.2/tests/test_diff_exit_codes.py

@@ -0,0 +1,83 @@
+"""Tests for dvc-diff exit code handling."""
+import pytest
+import subprocess
+from pathlib import Path
+import tempfile
+
+
+class TestDiffExitCodes:
+    """Test that dvc-diff properly propagates exit codes from pipeline commands."""
+
+    def test_successful_pipeline_returns_zero(self, tmp_path):
+        """Test that successful identical pipeline returns 0."""
+        # Create test files
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\nbar\n")
+        file2.write_text("foo\nbar\n")
+
+        # Run diff-x (not dvc-diff, but tests the same join_pipelines code)
+        result = subprocess.run(
+            ["diff-x", "cat", str(file1), str(file2)],
+            capture_output=True,
+        )
+        assert result.returncode == 0
+
+    def test_diff_found_returns_one(self, tmp_path):
+        """Test that differences found returns 1."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\n")
+        file2.write_text("bar\n")
+
+        result = subprocess.run(
+            ["diff-x", "cat", str(file1), str(file2)],
+            capture_output=True,
+        )
+        assert result.returncode == 1
+
+    def test_pipeline_error_propagates(self, tmp_path):
+        """Test that pipeline command errors propagate to exit code."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\n")
+        file2.write_text("bar\n")
+
+        # Use a command that will fail
+        result = subprocess.run(
+            ["diff-x", "cat /nonexistent/file/that/does/not/exist ||", str(file1), str(file2)],
+            capture_output=True,
+            shell=False,
+        )
+        # Should return non-zero due to cat failing
+        assert result.returncode != 0
+
+    def test_false_command_propagates_error(self, tmp_path):
+        """Test that 'false' command in pipeline propagates error."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\n")
+        file2.write_text("bar\n")
+
+        # Use 'false' which always returns 1
+        result = subprocess.run(
+            ["diff-x", "cat", "false", str(file1), str(file2)],
+            capture_output=True,
+        )
+        # Should return non-zero due to false in pipeline
+        assert result.returncode != 0
+
+    def test_multi_stage_pipeline_error(self, tmp_path):
+        """Test that errors in multi-stage pipelines are detected."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\nbar\n")
+        file2.write_text("bar\nfoo\n")
+
+        # Pipeline: sort (succeeds) | false (fails)
+        result = subprocess.run(
+            ["diff-x", "-x", "sort", "-x", "false", str(file1), str(file2)],
+            capture_output=True,
+        )
+        # Should return non-zero due to false in pipeline
+        assert result.returncode != 0