dvc-utils 0.2.0__tar.gz → 0.3.1__tar.gz

dvc_utils-0.3.1/.github/workflows/ci.yml

@@ -0,0 +1,74 @@
+name: Verify README examples, release to PyPI
+on:
+  push:
+    branches: [ "main" ]
+    tags: [ "v**" ]
+  pull_request:
+    branches: [ "main" ]
+  workflow_dispatch:
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+jobs:
+  test:
+    name: Verify README examples
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+      - uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: true
+      - name: Set up Python 3.11
+        run: uv python install 3.11
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo install parquet2json
+      - name: Install dependencies
+        run: uv sync --extra ci --extra test
+      - name: Run pytest
+        run: |
+          source .venv/bin/activate
+          pytest
+      - name: '`dvc pull` test/data'
+        working-directory: test/data
+        run: |
+          source ../../.venv/bin/activate
+          dvc pull -r s3 -R -A
+      - name: Set up parquet-helpers
+        uses: actions/checkout@v4
+        with:
+          repository: ryan-williams/parquet-helpers
+          path: pqt
+      - name: Verify README examples
+        env:
+          # Evaluate README examples from within the `test/data` submodule
+          BMDF_WORKDIR: test/data
+        run: |
+          source .venv/bin/activate
+          export PATH="$PWD/pqt:$PATH"
+          . pqt/.pqt-rc
+          export SHELL
+          mdcmd
+          git diff --exit-code
+  release:
+    name: Release to PyPI
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v5
+      - name: Build package
+        run: uv build
+      - name: Publish to PyPI
+        run: uv publish --username __token__ --password ${{ secrets.PYPI_TOKEN }}
+      - name: Create GitHub Release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create ${{ github.ref_name }} \
+            --title "${{ github.ref_name }}" \
+            --generate-notes

dvc_utils-0.3.1/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "test/data"]
+	path = test/data
+	url = https://github.com/ryan-williams/dvc-helpers

{dvc-utils-0.2.0/src/dvc_utils.egg-info → dvc_utils-0.3.1}/PKG-INFO

@@ -1,13 +1,23 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dvc-utils
-Version: 0.2.0
-Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
-Home-page: https://github.com/runsascoded/dvc-utils
-Author: Ryan Williams
-Author-email: ryan@runsascoded.com
+Version: 0.3.1
+Summary: CLI for diffing DVC-tracked files at two commits (or one commit vs. current worktree), optionally passing both through another command first
+Project-URL: Homepage, https://github.com/runsascoded/dvc-utils
+Project-URL: Author URL, https://github.com/ryan-williams
+Author-email: Ryan Williams <ryan@runsascoded.com>
 License: MIT
-Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Python: >=3.9
+Requires-Dist: click
+Requires-Dist: dffs>=0.0.6
+Requires-Dist: pyyaml
+Requires-Dist: utz>=0.20.0
+Provides-Extra: ci
+Requires-Dist: bmdf==0.5.2; extra == 'ci'
+Requires-Dist: dvc-s3; extra == 'ci'
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == 'test'
+Description-Content-Type: text/markdown
 
 # dvc-utils
 Diff [DVC] files, optionally piping through other commands first.
@@ -32,25 +42,11 @@ pip install dvc-utils
 ```
 
 ## Usage <a id="usage"></a>
-<!-- `bmdf -- dvc-utils --help` -->
-```bash
-dvc-utils --help
-# Usage: dvc-utils [OPTIONS] COMMAND [ARGS]...
-#
-# Options:
-#   --help  Show this message and exit.
-#
-# Commands:
-#   diff  Diff a DVC-tracked file at two commits (or one commit vs. current
-#         worktree), optionally passing both through another command first
-```
+Currently one command is exposed, `dvc-diff`:
 
-The single subcommand, `dvc-utils diff`, is also exposed directly as `dvc-dff`:
-
-### `dvc-diff` <a id="dvc-diff"></a>
-<!-- `bmdf -- dvc-diff --help` -->
+<!-- `bmdf -- dvc-diff` -->
 ```bash
-dvc-diff --help
+dvc-diff
 # Usage: dvc-diff [OPTIONS] [exec_cmd...] <path>
 #
 #   Diff a file at two commits (or one commit vs. current worktree), optionally
@@ -87,9 +83,8 @@ dvc-diff --help
 ```
 
 ## Examples <a id="examples"></a>
-These examples are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data`
-
-([`test/data`] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files used for testing [`git-diff-dvc.sh`])
+- Examples below are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data` (see [.github/workflows/ci.yml](.github/workflows/ci.yml)).
+- [test/data] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files (used in that repo for testing [`git-diff-dvc.sh`]).
 
 [`8ec2060`] added a DVC-tracked text file, `test.txt`:
 
@@ -159,9 +154,9 @@ dvc-diff -R f92c1d2 pqa test.parquet
 
 [`f29e52a`] updated `test.parquet`:
 
-<!-- `bmdf -- dvc-diff -R f29e52a pqa test.parquet` -->
+<!-- `bmdf -E PQT_TXT_OPTS=-n2 -- dvc-diff -R f29e52a pqa test.parquet` -->
 ```bash
-dvc-diff -R f29e52a pqa test.parquet
+PQT_TXT_OPTS=-n2 dvc-diff -R f29e52a pqa test.parquet
 # 1,3c1,3
 # < MD5: 4379600b26647a50dfcd0daa824e8219
 # < 1635 bytes
@@ -174,17 +169,16 @@ dvc-diff -R f29e52a pqa test.parquet
 # <   OPTIONAL INT64 num;
 # ---
 # >   OPTIONAL INT32 num;
-# 26a27,38
-# > }
-# > {
-# >   "num": 666,
-# >   "str": "fff"
-# > }
-# > {
+# 19,20c19,20
+# <   "num": 444,
+# <   "str": "ddd"
+# ---
 # >   "num": 777,
 # >   "str": "ggg"
-# > }
-# > {
+# 23,24c23,24
+# <   "num": 555,
+# <   "str": "eee"
+# ---
 # >   "num": 888,
 # >   "str": "hhh"
 ```
@@ -458,6 +452,7 @@ This helped me see that the data update in question (`c0..c1`) dropped some fiel
 
 [`mdcmd`]: https://github.com/runsascoded/bash-markdown-fence?tab=readme-ov-file#bmdf
 [`test/data`]: test/data
+[test/data]: test/data
 [ryan-williams/dvc-helpers@test]: https://github.com/ryan-williams/dvc-helpers/tree/test
 [`git-diff-dvc.sh`]: https://github.com/ryan-williams/dvc-helpers/blob/main/git-diff-dvc.sh
 

{dvc-utils-0.2.0 → dvc_utils-0.3.1}/README.md

@@ -21,25 +21,11 @@ pip install dvc-utils
 ```
 
 ## Usage <a id="usage"></a>
-<!-- `bmdf -- dvc-utils --help` -->
-```bash
-dvc-utils --help
-# Usage: dvc-utils [OPTIONS] COMMAND [ARGS]...
-#
-# Options:
-#   --help  Show this message and exit.
-#
-# Commands:
-#   diff  Diff a DVC-tracked file at two commits (or one commit vs. current
-#         worktree), optionally passing both through another command first
-```
-
-The single subcommand, `dvc-utils diff`, is also exposed directly as `dvc-dff`:
+Currently one command is exposed, `dvc-diff`:
 
-### `dvc-diff` <a id="dvc-diff"></a>
-<!-- `bmdf -- dvc-diff --help` -->
+<!-- `bmdf -- dvc-diff` -->
 ```bash
-dvc-diff --help
+dvc-diff
 # Usage: dvc-diff [OPTIONS] [exec_cmd...] <path>
 #
 #   Diff a file at two commits (or one commit vs. current worktree), optionally
@@ -76,9 +62,8 @@ dvc-diff --help
 ```
 
 ## Examples <a id="examples"></a>
-These examples are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data`
-
-([`test/data`] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files used for testing [`git-diff-dvc.sh`])
+- Examples below are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data` (see [.github/workflows/ci.yml](.github/workflows/ci.yml)).
+- [test/data] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files (used in that repo for testing [`git-diff-dvc.sh`]).
 
 [`8ec2060`] added a DVC-tracked text file, `test.txt`:
 
@@ -148,9 +133,9 @@ dvc-diff -R f92c1d2 pqa test.parquet
 
 [`f29e52a`] updated `test.parquet`:
 
-<!-- `bmdf -- dvc-diff -R f29e52a pqa test.parquet` -->
+<!-- `bmdf -E PQT_TXT_OPTS=-n2 -- dvc-diff -R f29e52a pqa test.parquet` -->
 ```bash
-dvc-diff -R f29e52a pqa test.parquet
+PQT_TXT_OPTS=-n2 dvc-diff -R f29e52a pqa test.parquet
 # 1,3c1,3
 # < MD5: 4379600b26647a50dfcd0daa824e8219
 # < 1635 bytes
@@ -163,17 +148,16 @@ dvc-diff -R f29e52a pqa test.parquet
 # <   OPTIONAL INT64 num;
 # ---
 # >   OPTIONAL INT32 num;
-# 26a27,38
-# > }
-# > {
-# >   "num": 666,
-# >   "str": "fff"
-# > }
-# > {
+# 19,20c19,20
+# <   "num": 444,
+# <   "str": "ddd"
+# ---
 # >   "num": 777,
 # >   "str": "ggg"
-# > }
-# > {
+# 23,24c23,24
+# <   "num": 555,
+# <   "str": "eee"
+# ---
 # >   "num": 888,
 # >   "str": "hhh"
 ```
@@ -447,6 +431,7 @@ This helped me see that the data update in question (`c0..c1`) dropped some fiel
 
 [`mdcmd`]: https://github.com/runsascoded/bash-markdown-fence?tab=readme-ov-file#bmdf
 [`test/data`]: test/data
+[test/data]: test/data
 [ryan-williams/dvc-helpers@test]: https://github.com/ryan-williams/dvc-helpers/tree/test
 [`git-diff-dvc.sh`]: https://github.com/ryan-williams/dvc-helpers/blob/main/git-diff-dvc.sh
 

dvc_utils-0.3.1/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dvc-utils"
+version = "0.3.1"
+description = "CLI for diffing DVC-tracked files at two commits (or one commit vs. current worktree), optionally passing both through another command first"
+readme = "README.md"
+license = {text = "MIT"}
+authors = [
+    {name = "Ryan Williams", email = "ryan@runsascoded.com"}
+]
+requires-python = ">=3.9"
+dependencies = [
+    "click",
+    "dffs>=0.0.6",
+    "pyyaml",
+    "utz>=0.20.0",
+]
+
+[project.optional-dependencies]
+ci = [
+    "bmdf==0.5.2",
+    "dvc-s3",
+]
+test = [
+    "pytest>=7.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/runsascoded/dvc-utils"
+"Author URL" = "https://github.com/ryan-williams"
+
+[project.scripts]
+dvc-utils = "dvc_utils.main:main"
+dvc-diff = "dvc_utils.diff:dvc_diff"
+
+[dependency-groups]
+dev = [
+    "pytest>=7.0.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/dvc_utils"]

dvc_utils-0.3.1/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = -v --tb=short

{dvc-utils-0.2.0 → dvc_utils-0.3.1}/src/dvc_utils/__init__.py

@@ -1,2 +1,2 @@
-from . import cli, path
+from . import cli, diff, path
 from .path import dvc_cache_dir, dvc_md5, dvc_paths, dvc_path

dvc_utils-0.3.1/src/dvc_utils/cli.py

@@ -0,0 +1,17 @@
+import json
+import shlex
+from os import listdir
+from os.path import isdir, join
+from typing import Tuple
+
+import click
+from click import option, argument, group
+from dffs import join_pipelines
+from utz import process, err, hash_file
+
+from dvc_utils.path import dvc_paths, dvc_cache_path
+
+
+@group()
+def cli():
+    pass

dvc-utils-0.2.0/src/dvc_utils/cli.py → dvc_utils-0.3.1/src/dvc_utils/diff.py

@@ -6,18 +6,18 @@ from typing import Tuple
 
 import click
 from click import option, argument, group
-from qmdx import join_pipelines
+from dffs import join_pipelines
 from utz import process, err, hash_file
 
+from dvc_utils.cli import cli
 from dvc_utils.path import dvc_paths, dvc_cache_path
 
 
-@group()
-def cli():
-    pass
-
-
-@cli.command('diff', short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first')
+@cli.command(
+    'diff',
+    short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first',
+    no_args_is_help=True,
+)
 @option('-c/-C', '--color/--no-color', default=None, help='Force or prevent colorized output')
 @option('-r', '--refspec', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
 @option('-R', '--ref', help='Shorthand for `-r <ref>^..<ref>`, i.e. inspect a specific commit (vs. its parent)')
@@ -28,7 +28,7 @@ def cli():
 @option('-w', '--ignore-whitespace', is_flag=True, help="Ignore whitespace differences (pass `-w` to `diff`)")
 @option('-x', '--exec-cmd', 'exec_cmds', multiple=True, help='Command(s) to execute before diffing; alternate syntax to passing commands as positional arguments')
 @argument('args', metavar='[exec_cmd...] <path>', nargs=-1)
-def dvc_utils_diff(
+def dvc_diff(
     color: bool | None,
     refspec: str | None,
     ref: str | None,
@@ -111,7 +111,7 @@ def dvc_utils_diff(
                 cmds1 = [ shlex.split(cmd) for cmd in cmds1 ]
                 cmds2 = [ shlex.split(cmd) for cmd in cmds2 ]
 
-            join_pipelines(
+            returncode = join_pipelines(
                 base_cmd=['diff', *diff_args],
                 cmds1=cmds1,
                 cmds2=cmds2,
@@ -119,10 +119,7 @@ def dvc_utils_diff(
                 shell=shell,
                 executable=shell_executable,
             )
+            exit(returncode)
         else:
             res = process.run('diff', *diff_args, path1 or '/dev/null', path2 or '/dev/null', log=log, check=False)
             exit(res.returncode)
-
-
-if __name__ == '__main__':
-    cli()

dvc_utils-0.3.1/src/dvc_utils/main.py

@@ -0,0 +1,9 @@
+from dvc_utils.cli import cli
+from dvc_utils import diff
+
+def main():
+    cli()
+
+
+if __name__ == '__main__':
+    cli()

{dvc-utils-0.2.0 → dvc_utils-0.3.1}/src/dvc_utils/path.py

@@ -49,8 +49,13 @@ def dvc_md5(
     dir_path = get_dir_path()
     dir_path = '' if dir_path == '.' else f'{dir_path}{sep}'
     dvc_path = f"{dir_path}{dvc_path}"
-    dvc_spec = process.output('git', 'show', f'{git_ref}:{dvc_path}', log=err if log else None, err_ok=True, stderr=DEVNULL)
-    if dvc_spec is None:
+    dvc_spec = process.output(
+        'git', 'show', f'{git_ref}:{dvc_path}',
+        err_ok=True,
+        log=err if log else None,
+        stderr=None if log else DEVNULL,
+    )
+    if not dvc_spec:
         cur_dir = dirname(dvc_path)
         relpath = basename(dvc_path)
         if relpath.endswith(".dvc"):

dvc_utils-0.3.1/src/dvc_utils/sync.py

@@ -0,0 +1,13 @@
+from click import option
+
+from dvc_utils.cli import cli
+from git import Repo
+
+
+@cli.command('pull-x', short_help='Sync DVC cache files from an S3 remote')
+@option('-n', '--dry-run', is_flag=True, help='Print files that would be synced, don\'t actually perform sync')
+@option('-p', '--path', 'paths', multiple=True, help='Path globs to sync')
+@option('-r', '--ref', 'refs', multiple=True, help='Git refs to sync DVC files from')
+def pull_x(dry_run, paths, refs):
+    repo = Repo()
+

dvc_utils-0.3.1/tests/test_diff_exit_codes.py

@@ -0,0 +1,83 @@
+"""Tests for dvc-diff exit code handling."""
+import pytest
+import subprocess
+from pathlib import Path
+import tempfile
+
+
+class TestDiffExitCodes:
+    """Test that dvc-diff properly propagates exit codes from pipeline commands."""
+
+    def test_successful_pipeline_returns_zero(self, tmp_path):
+        """Test that successful identical pipeline returns 0."""
+        # Create test files
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\nbar\n")
+        file2.write_text("foo\nbar\n")
+
+        # Run diff-x (not dvc-diff, but tests the same join_pipelines code)
+        result = subprocess.run(
+            ["diff-x", "cat", str(file1), str(file2)],
+            capture_output=True,
+        )
+        assert result.returncode == 0
+
+    def test_diff_found_returns_one(self, tmp_path):
+        """Test that differences found returns 1."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\n")
+        file2.write_text("bar\n")
+
+        result = subprocess.run(
+            ["diff-x", "cat", str(file1), str(file2)],
+            capture_output=True,
+        )
+        assert result.returncode == 1
+
+    def test_pipeline_error_propagates(self, tmp_path):
+        """Test that pipeline command errors propagate to exit code."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\n")
+        file2.write_text("bar\n")
+
+        # Use a command that will fail
+        result = subprocess.run(
+            ["diff-x", "cat /nonexistent/file/that/does/not/exist ||", str(file1), str(file2)],
+            capture_output=True,
+            shell=False,
+        )
+        # Should return non-zero due to cat failing
+        assert result.returncode != 0
+
+    def test_false_command_propagates_error(self, tmp_path):
+        """Test that 'false' command in pipeline propagates error."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\n")
+        file2.write_text("bar\n")
+
+        # Use 'false' which always returns 1
+        result = subprocess.run(
+            ["diff-x", "cat", "false", str(file1), str(file2)],
+            capture_output=True,
+        )
+        # Should return non-zero due to false in pipeline
+        assert result.returncode != 0
+
+    def test_multi_stage_pipeline_error(self, tmp_path):
+        """Test that errors in multi-stage pipelines are detected."""
+        file1 = tmp_path / "test1.txt"
+        file2 = tmp_path / "test2.txt"
+        file1.write_text("foo\nbar\n")
+        file2.write_text("bar\nfoo\n")
+
+        # Pipeline: sort (succeeds) | false (fails)
+        result = subprocess.run(
+            ["diff-x", "-x", "sort", "-x", "false", str(file1), str(file2)],
+            capture_output=True,
+        )
+        # Should return non-zero due to false in pipeline
+        assert result.returncode != 0