dvc-utils 0.0.4__tar.gz → 0.0.6__tar.gz

{dvc-utils-0.0.4 → dvc-utils-0.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dvc-utils
-Version: 0.0.4
+Version: 0.0.6
 Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
 Home-page: https://github.com/runsascoded/dvc-utils
 Author: Ryan Williams
@@ -15,7 +15,7 @@ CLI for diffing [DVC] files, optionally passing both through another command fir
 <!-- toc -->
 - [Installation](#installation)
 - [Usage](#usage)
-    - [`dvc-utils diff`](#dvc-utils-diff)
+    - [`dvc-diff`](#dvc-diff)
 - [Examples](#examples)
     - [Parquet file](#parquet-diff)
         - [Schema diff](#parquet-schema-diff)
@@ -42,11 +42,13 @@ dvc-utils --help
 #         worktree), optionally passing both through another command first
 ```
 
-### `dvc-utils diff` <a id="dvc-utils-diff"></a>
-<!-- `bmdf -- dvc-utils diff --help` -->
+The single subcommand, `dvc-utils diff`, is also exposed directly as `dvc-dff`:
+
+### `dvc-diff` <a id="dvc-diff"></a>
+<!-- `bmdf -- dvc-diff --help` -->
 ```bash
-dvc-utils diff --help
-# Usage: dvc-utils diff [OPTIONS] [cmd...] <path>
+dvc-diff --help
+# Usage: dvc-diff [OPTIONS] [exec_cmd...] <path>
 #
 #   Diff a file at two commits (or one commit vs. current worktree), optionally
 #   passing both through `cmd` first
@@ -69,6 +71,8 @@ dvc-utils diff --help
 #                            to `diff`)
 #   -v, --verbose            Log intermediate commands to stderr
 #   -w, --ignore-whitespace  Ignore whitespace differences (pass `-w` to `diff`)
+#   -x, --exec-cmd TEXT      Command(s) to execute before diffing; alternate
+#                            syntax to passing commands as positional arguments
 #   --help                   Show this message and exit.
 ```
 
@@ -91,7 +95,7 @@ parquet_schema() {
     parquet2json "$1" schema
 }
 export -f parquet_schema
-dvc-utils diff -r $commit^..$commit parquet_schema $path
+dvc-diff -r $commit^..$commit parquet_schema $path
 ```
 <details><summary>Output</summary>
 
@@ -149,7 +153,7 @@ pretty_print_first_row() {
     parquet2json "$1" cat -l 1 | jq .
 }
 export -f pretty_print_first_row
-dvc-utils diff -r $commit^..$commit pretty_print_first_row $path
+dvc-diff -r $commit^..$commit pretty_print_first_row $path
 ```
 
 <details><summary>Output</summary>
@@ -208,7 +212,7 @@ parquet_row_count() {
     parquet2json "$1" rowcount
 }
 export -f parquet_row_count
-dvc-utils diff -r $commit^..$commit parquet_row_count $path
+dvc-diff -r $commit^..$commit parquet_row_count $path
 ```
 
 This time we get no output; [the given `$commit`][commit] didn't change the row count in the DVC-tracked Parquet file [`$path`][commit path].

{dvc-utils-0.0.4 → dvc-utils-0.0.6}/README.md

@@ -4,7 +4,7 @@ CLI for diffing [DVC] files, optionally passing both through another command fir
 <!-- toc -->
 - [Installation](#installation)
 - [Usage](#usage)
-    - [`dvc-utils diff`](#dvc-utils-diff)
+    - [`dvc-diff`](#dvc-diff)
 - [Examples](#examples)
     - [Parquet file](#parquet-diff)
         - [Schema diff](#parquet-schema-diff)
@@ -31,11 +31,13 @@ dvc-utils --help
 #         worktree), optionally passing both through another command first
 ```
 
-### `dvc-utils diff` <a id="dvc-utils-diff"></a>
-<!-- `bmdf -- dvc-utils diff --help` -->
+The single subcommand, `dvc-utils diff`, is also exposed directly as `dvc-dff`:
+
+### `dvc-diff` <a id="dvc-diff"></a>
+<!-- `bmdf -- dvc-diff --help` -->
 ```bash
-dvc-utils diff --help
-# Usage: dvc-utils diff [OPTIONS] [cmd...] <path>
+dvc-diff --help
+# Usage: dvc-diff [OPTIONS] [exec_cmd...] <path>
 #
 #   Diff a file at two commits (or one commit vs. current worktree), optionally
 #   passing both through `cmd` first
@@ -58,6 +60,8 @@ dvc-utils diff --help
 #                            to `diff`)
 #   -v, --verbose            Log intermediate commands to stderr
 #   -w, --ignore-whitespace  Ignore whitespace differences (pass `-w` to `diff`)
+#   -x, --exec-cmd TEXT      Command(s) to execute before diffing; alternate
+#                            syntax to passing commands as positional arguments
 #   --help                   Show this message and exit.
 ```
 
@@ -80,7 +84,7 @@ parquet_schema() {
     parquet2json "$1" schema
 }
 export -f parquet_schema
-dvc-utils diff -r $commit^..$commit parquet_schema $path
+dvc-diff -r $commit^..$commit parquet_schema $path
 ```
 <details><summary>Output</summary>
 
@@ -138,7 +142,7 @@ pretty_print_first_row() {
     parquet2json "$1" cat -l 1 | jq .
 }
 export -f pretty_print_first_row
-dvc-utils diff -r $commit^..$commit pretty_print_first_row $path
+dvc-diff -r $commit^..$commit pretty_print_first_row $path
 ```
 
 <details><summary>Output</summary>
@@ -197,7 +201,7 @@ parquet_row_count() {
     parquet2json "$1" rowcount
 }
 export -f parquet_row_count
-dvc-utils diff -r $commit^..$commit parquet_row_count $path
+dvc-diff -r $commit^..$commit parquet_row_count $path
 ```
 
 This time we get no output; [the given `$commit`][commit] didn't change the row count in the DVC-tracked Parquet file [`$path`][commit path].

{dvc-utils-0.0.4 → dvc-utils-0.0.6}/dvc_utils/main.py

@@ -1,14 +1,11 @@
 from functools import cache
 from os import environ as env, getcwd
-
-from typing import Optional, Tuple
-
-import shlex
 from os.path import join, relpath
+import shlex
+from subprocess import Popen, PIPE
+from typing import Optional, Tuple
 
 from click import option, argument, group
-from subprocess import Popen
-
 import click
 import yaml
 from utz import process, singleton, err
@@ -52,7 +49,7 @@ def dvc_cache_dir(log: bool = False) -> str:
 def dvc_md5(git_ref: str, dvc_path: str, log: bool = False) -> str:
     dir_path = get_dir_path()
     dir_path = '' if dir_path == '.' else f'{dir_path}/'
-    dvc_spec = process.output('git', 'show', f'{git_ref}:{dir_path}{dvc_path}', log=log)
+    dvc_spec = process.output('git', 'show', f'{git_ref}:{dir_path}{dvc_path}', log=err if log else None)
     dvc_obj = yaml.safe_load(dvc_spec)
     out = singleton(dvc_obj['outs'], dedupe=False)
     md5 = out['md5']
@@ -73,15 +70,28 @@ def dvc_cache_path(ref: str, dvc_path: Optional[str] = None, log: bool = False)
 
 
 def diff_cmds(
-    cmd1: str,
-    cmd2: str,
+    cmds1: list[str],
+    cmds2: list[str],
     verbose: bool = False,
     color: bool = False,
     unified: int | None = None,
     ignore_whitespace: bool = False,
     **kwargs,
 ):
-    """Run two commands and diff their output.
+    """Run two sequences of piped commands and diff their output.
+
+    Args:
+        cmds1: First sequence of commands to pipe together
+        cmds2: Second sequence of commands to pipe together
+        verbose: Whether to print commands being executed
+        color: Whether to show colored diff output
+        unified: Number of unified context lines, or None
+        ignore_whitespace: Whether to ignore whitespace changes
+        **kwargs: Additional arguments passed to subprocess.Popen
+
+    Each command sequence will be piped together before being compared.
+    For example, if cmds1 = ['cat foo.txt', 'sort'], the function will
+    execute 'cat foo.txt | sort' before comparing with cmds2's output.
 
     Adapted from https://stackoverflow.com/a/28840955"""
     with named_pipes(n=2) as pipes:
@@ -96,11 +106,45 @@ def diff_cmds(
         ]
         diff = Popen(diff_cmd)
         processes = []
-        for path, cmd in ((pipe1, cmd1), (pipe2, cmd2)):
-            with open(path, 'wb', 0) as pipe:
-                if verbose:
-                    err(f"Running: {cmd}")
-                processes.append(Popen(cmd, stdout=pipe, close_fds=True, **kwargs))
+
+        for pipe, cmds in ((pipe1, cmds1), (pipe2, cmds2)):
+            if verbose:
+                err(f"Running pipeline: {' | '.join(cmds)}")
+
+            # Create the pipeline of processes
+            prev_process = None
+            for i, cmd in enumerate(cmds):
+                is_last = i + 1 == len(cmds)
+
+                # For the first process, take input from the original source
+                stdin = None if prev_process is None else prev_process.stdout
+
+                # For the last process, output to the named pipe
+                if is_last:
+                    with open(pipe, 'wb', 0) as pipe_fd:
+                        proc = Popen(
+                            cmd,
+                            stdin=stdin,
+                            stdout=pipe_fd,
+                            close_fds=True,
+                            **kwargs
+                        )
+                # For intermediate processes, output to a pipe
+                else:
+                    proc = Popen(
+                        cmd,
+                        stdin=stdin,
+                        stdout=PIPE,
+                        close_fds=True,
+                        **kwargs
+                    )
+
+                if prev_process is not None:
+                    prev_process.stdout.close()
+
+                processes.append(proc)
+                prev_process = proc
+
         for p in [diff] + processes:
             p.wait()
 
@@ -112,7 +156,8 @@ def diff_cmds(
 @option('-U', '--unified', type=int, help='Number of lines of context to show (passes through to `diff`)')
 @option('-v', '--verbose', is_flag=True, help="Log intermediate commands to stderr")
 @option('-w', '--ignore-whitespace', is_flag=True, help="Ignore whitespace differences (pass `-w` to `diff`)")
-@argument('args', metavar='[cmd...] <path>', nargs=-1)
+@option('-x', '--exec-cmd', 'exec_cmds', multiple=True, help='Command(s) to execute before diffing; alternate syntax to passing commands as positional arguments')
+@argument('args', metavar='[exec_cmd...] <path>', nargs=-1)
 def dvc_utils_diff(
     color: bool,
     refspec: str | None,
@@ -120,6 +165,7 @@ def dvc_utils_diff(
     unified: int | None,
     verbose: bool,
     ignore_whitespace: bool,
+    exec_cmds: Tuple[str, ...],
     args: Tuple[str, ...],
 ):
     """Diff a file at two commits (or one commit vs. current worktree), optionally passing both through `cmd` first
@@ -134,14 +180,8 @@ def dvc_utils_diff(
         raise click.UsageError('Must specify [cmd...] <path>')
 
     shell = not no_shell
-    if len(args) == 2:
-        cmd, path = args
-        cmd = shlex.split(cmd)
-    elif len(args) == 1:
-        cmd = None
-        path, = args
-    else:
-        raise click.UsageError('Maximum 2 positional args: [cmd] <path>')
+    *cmds, path = args
+    cmds = list(exec_cmds) + cmds
 
     path, dvc_path = dvc_paths(path)
 
@@ -158,17 +198,27 @@ def dvc_utils_diff(
     before_path = dvc_cache_path(before, dvc_path, log=log)
     after_path = path if after is None else dvc_cache_path(after, dvc_path, log=log)
 
-    if cmd:
-        def args(path: str):
-            arr = cmd + [path]
-            return shlex.join(arr) if shell else arr
+    if cmds:
+        cmd, *sub_cmds = cmds
+        if not shell:
+            sub_cmds = [ shlex.split(c) for c in sub_cmds ]
+            before_cmds = [
+                shlex.split(f'{cmd} {before_path}'),
+                *sub_cmds,
+            ]
+            after_cmds = [
+                shlex.split(f'{cmd} {after_path}'),
+                *sub_cmds,
+            ]
+            shell_kwargs = {}
+        else:
+            before_cmds = [ f'{cmd} {before_path}', *sub_cmds ]
+            after_cmds = [ f'{cmd} {after_path}', *sub_cmds ]
+            shell_kwargs = dict(shell=shell)
 
-        shell_kwargs = dict(shell=shell) if shell else {}
-        before_cmd = args(before_path)
-        after_cmd = args(after_path)
         diff_cmds(
-            before_cmd,
-            after_cmd,
+            before_cmds,
+            after_cmds,
             verbose=verbose,
             color=color,
             unified=unified,

{dvc-utils-0.0.4 → dvc-utils-0.0.6}/dvc_utils.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dvc-utils
-Version: 0.0.4
+Version: 0.0.6
 Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
 Home-page: https://github.com/runsascoded/dvc-utils
 Author: Ryan Williams
@@ -15,7 +15,7 @@ CLI for diffing [DVC] files, optionally passing both through another command fir
 <!-- toc -->
 - [Installation](#installation)
 - [Usage](#usage)
-    - [`dvc-utils diff`](#dvc-utils-diff)
+    - [`dvc-diff`](#dvc-diff)
 - [Examples](#examples)
     - [Parquet file](#parquet-diff)
         - [Schema diff](#parquet-schema-diff)
@@ -42,11 +42,13 @@ dvc-utils --help
 #         worktree), optionally passing both through another command first
 ```
 
-### `dvc-utils diff` <a id="dvc-utils-diff"></a>
-<!-- `bmdf -- dvc-utils diff --help` -->
+The single subcommand, `dvc-utils diff`, is also exposed directly as `dvc-dff`:
+
+### `dvc-diff` <a id="dvc-diff"></a>
+<!-- `bmdf -- dvc-diff --help` -->
 ```bash
-dvc-utils diff --help
-# Usage: dvc-utils diff [OPTIONS] [cmd...] <path>
+dvc-diff --help
+# Usage: dvc-diff [OPTIONS] [exec_cmd...] <path>
 #
 #   Diff a file at two commits (or one commit vs. current worktree), optionally
 #   passing both through `cmd` first
@@ -69,6 +71,8 @@ dvc-utils diff --help
 #                            to `diff`)
 #   -v, --verbose            Log intermediate commands to stderr
 #   -w, --ignore-whitespace  Ignore whitespace differences (pass `-w` to `diff`)
+#   -x, --exec-cmd TEXT      Command(s) to execute before diffing; alternate
+#                            syntax to passing commands as positional arguments
 #   --help                   Show this message and exit.
 ```
 
@@ -91,7 +95,7 @@ parquet_schema() {
     parquet2json "$1" schema
 }
 export -f parquet_schema
-dvc-utils diff -r $commit^..$commit parquet_schema $path
+dvc-diff -r $commit^..$commit parquet_schema $path
 ```
 <details><summary>Output</summary>
 
@@ -149,7 +153,7 @@ pretty_print_first_row() {
     parquet2json "$1" cat -l 1 | jq .
 }
 export -f pretty_print_first_row
-dvc-utils diff -r $commit^..$commit pretty_print_first_row $path
+dvc-diff -r $commit^..$commit pretty_print_first_row $path
 ```
 
 <details><summary>Output</summary>
@@ -208,7 +212,7 @@ parquet_row_count() {
     parquet2json "$1" rowcount
 }
 export -f parquet_row_count
-dvc-utils diff -r $commit^..$commit parquet_row_count $path
+dvc-diff -r $commit^..$commit parquet_row_count $path
 ```
 
 This time we get no output; [the given `$commit`][commit] didn't change the row count in the DVC-tracked Parquet file [`$path`][commit path].

{dvc-utils-0.0.4 → dvc-utils-0.0.6}/dvc_utils.egg-info/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
+dvc-diff = dvc_utils.main:dvc_utils_diff
 dvc-utils = dvc_utils.main:cli

{dvc-utils-0.0.4 → dvc-utils-0.0.6}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name='dvc-utils',
-    version="0.0.4",
+    version="0.0.6",
     description="CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
@@ -10,6 +10,7 @@ setup(
     entry_points={
         'console_scripts': [
             'dvc-utils = dvc_utils.main:cli',
+            'dvc-diff = dvc_utils.main:dvc_utils_diff',
         ],
     },
     license="MIT",