dvc-utils 0.2.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

dvc_utils/__init__.py

@@ -1,2 +1,2 @@
-from . import cli, path
+from . import cli, diff, path
 from .path import dvc_cache_dir, dvc_md5, dvc_paths, dvc_path

dvc_utils/cli.py

@@ -6,7 +6,7 @@ from typing import Tuple
 
 import click
 from click import option, argument, group
-from qmdx import join_pipelines
+from dffs import join_pipelines
 from utz import process, err, hash_file
 
 from dvc_utils.path import dvc_paths, dvc_cache_path
@@ -15,114 +15,3 @@ from dvc_utils.path import dvc_paths, dvc_cache_path
 @group()
 def cli():
     pass
-
-
-@cli.command('diff', short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first')
-@option('-c/-C', '--color/--no-color', default=None, help='Force or prevent colorized output')
-@option('-r', '--refspec', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
-@option('-R', '--ref', help='Shorthand for `-r <ref>^..<ref>`, i.e. inspect a specific commit (vs. its parent)')
-@option('-s', '--shell-executable', help=f'Shell to use for executing commands; defaults to $SHELL')
-@option('-S', '--no-shell', is_flag=True, help="Don't pass `shell=True` to Python `subprocess`es")
-@option('-U', '--unified', type=int, help='Number of lines of context to show (passes through to `diff`)')
-@option('-v', '--verbose', is_flag=True, help="Log intermediate commands to stderr")
-@option('-w', '--ignore-whitespace', is_flag=True, help="Ignore whitespace differences (pass `-w` to `diff`)")
-@option('-x', '--exec-cmd', 'exec_cmds', multiple=True, help='Command(s) to execute before diffing; alternate syntax to passing commands as positional arguments')
-@argument('args', metavar='[exec_cmd...] <path>', nargs=-1)
-def dvc_utils_diff(
-    color: bool | None,
-    refspec: str | None,
-    ref: str | None,
-    shell_executable: str | None,
-    no_shell: bool,
-    unified: int | None,
-    verbose: bool,
-    ignore_whitespace: bool,
-    exec_cmds: Tuple[str, ...],
-    args: Tuple[str, ...],
-):
-    """Diff a file at two commits (or one commit vs. current worktree), optionally passing both through `cmd` first
-
-    Examples:
-
-    dvc-utils diff -r HEAD^..HEAD wc -l foo.dvc  # Compare the number of lines (`wc -l`) in `foo` (the file referenced by `foo.dvc`) at the previous vs. current commit (`HEAD^..HEAD`).
-
-    dvc-utils diff md5sum foo  # Diff the `md5sum` of `foo` (".dvc" extension is optional) at HEAD (last committed value) vs. the current worktree content.
-    """
-    if not args:
-        raise click.UsageError('Must specify [cmd...] <path>')
-
-    shell = not no_shell
-    *cmds, path = args
-    cmds = list(exec_cmds) + cmds
-
-    path, dvc_path = dvc_paths(path)
-
-    if refspec and ref:
-        raise ValueError("Specify -r/--refspec xor -R/--ref")
-    if ref:
-        refspec = f'{ref}^..{ref}'
-    elif not refspec:
-        refspec = 'HEAD'
-
-    pcs = refspec.split('..', 1)
-    if len(pcs) == 1:
-        before = pcs[0]
-        after = None
-    elif len(pcs) == 2:
-        before, after = pcs
-    else:
-        raise ValueError(f"Invalid refspec: {refspec}")
-
-    log = err if verbose else False
-    path1 = dvc_cache_path(before, dvc_path, log=log)
-    path2 = (path if after is None else dvc_cache_path(after, dvc_path, log=log))
-
-    if isdir(path):
-        dir_json1 = dir_json2 = {}
-        if path1:
-            with open(path1, 'r') as f:
-                obj = json.load(f)
-                dir_json1 = { e["relpath"]: e["md5"] for e in obj }
-        if path2:
-            if path2 == path and after is None:
-                dir_json2 = {}
-                for file in listdir(path2):
-                    md5 = hash_file(join(path2, file), hash_name='md5')
-                    dir_json2[file] = md5
-            else:
-                with open(path2, 'r') as f:
-                    dir_json2 = { obj["relpath"]: obj["md5"] for obj in json.load(f) }
-        for relpath in sorted(set(dir_json1) | set(dir_json2)):
-            md5_1 = dir_json1.get(relpath)
-            md5_2 = dir_json2.get(relpath)
-            if md5_1 != md5_2:
-                print(f'{relpath}: {md5_1} -> {md5_2}')
-    else:
-        diff_args = [
-            *(['-w'] if ignore_whitespace else []),
-            *(['-U', str(unified)] if unified is not None else []),
-            *(['--color=always'] if color is True else ['--color=never'] if color is False else []),
-        ]
-        if cmds:
-            cmd, *sub_cmds = cmds
-            cmds1 = [ 'cat /dev/null' ] if path1 is None else [ f'{cmd} {path1 or "/dev/null"}', *sub_cmds ]
-            cmds2 = [ 'cat /dev/null' ] if path2 is None else [ f'{cmd} {path2 or "/dev/null"}', *sub_cmds ]
-            if not shell:
-                cmds1 = [ shlex.split(cmd) for cmd in cmds1 ]
-                cmds2 = [ shlex.split(cmd) for cmd in cmds2 ]
-
-            join_pipelines(
-                base_cmd=['diff', *diff_args],
-                cmds1=cmds1,
-                cmds2=cmds2,
-                verbose=verbose,
-                shell=shell,
-                executable=shell_executable,
-            )
-        else:
-            res = process.run('diff', *diff_args, path1 or '/dev/null', path2 or '/dev/null', log=log, check=False)
-            exit(res.returncode)
-
-
-if __name__ == '__main__':
-    cli()

dvc_utils/diff.py

@@ -0,0 +1,125 @@
+import json
+import shlex
+from os import listdir
+from os.path import isdir, join
+from typing import Tuple
+
+import click
+from click import option, argument, group
+from dffs import join_pipelines
+from utz import process, err, hash_file
+
+from dvc_utils.cli import cli
+from dvc_utils.path import dvc_paths, dvc_cache_path
+
+
+@cli.command(
+    'diff',
+    short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first',
+    no_args_is_help=True,
+)
+@option('-c/-C', '--color/--no-color', default=None, help='Force or prevent colorized output')
+@option('-r', '--refspec', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
+@option('-R', '--ref', help='Shorthand for `-r <ref>^..<ref>`, i.e. inspect a specific commit (vs. its parent)')
+@option('-s', '--shell-executable', help=f'Shell to use for executing commands; defaults to $SHELL')
+@option('-S', '--no-shell', is_flag=True, help="Don't pass `shell=True` to Python `subprocess`es")
+@option('-U', '--unified', type=int, help='Number of lines of context to show (passes through to `diff`)')
+@option('-v', '--verbose', is_flag=True, help="Log intermediate commands to stderr")
+@option('-w', '--ignore-whitespace', is_flag=True, help="Ignore whitespace differences (pass `-w` to `diff`)")
+@option('-x', '--exec-cmd', 'exec_cmds', multiple=True, help='Command(s) to execute before diffing; alternate syntax to passing commands as positional arguments')
+@argument('args', metavar='[exec_cmd...] <path>', nargs=-1)
+def dvc_diff(
+    color: bool | None,
+    refspec: str | None,
+    ref: str | None,
+    shell_executable: str | None,
+    no_shell: bool,
+    unified: int | None,
+    verbose: bool,
+    ignore_whitespace: bool,
+    exec_cmds: Tuple[str, ...],
+    args: Tuple[str, ...],
+):
+    """Diff a file at two commits (or one commit vs. current worktree), optionally passing both through `cmd` first
+
+    Examples:
+
+    dvc-utils diff -r HEAD^..HEAD wc -l foo.dvc  # Compare the number of lines (`wc -l`) in `foo` (the file referenced by `foo.dvc`) at the previous vs. current commit (`HEAD^..HEAD`).
+
+    dvc-utils diff md5sum foo  # Diff the `md5sum` of `foo` (".dvc" extension is optional) at HEAD (last committed value) vs. the current worktree content.
+    """
+    if not args:
+        raise click.UsageError('Must specify [cmd...] <path>')
+
+    shell = not no_shell
+    *cmds, path = args
+    cmds = list(exec_cmds) + cmds
+
+    path, dvc_path = dvc_paths(path)
+
+    if refspec and ref:
+        raise ValueError("Specify -r/--refspec xor -R/--ref")
+    if ref:
+        refspec = f'{ref}^..{ref}'
+    elif not refspec:
+        refspec = 'HEAD'
+
+    pcs = refspec.split('..', 1)
+    if len(pcs) == 1:
+        before = pcs[0]
+        after = None
+    elif len(pcs) == 2:
+        before, after = pcs
+    else:
+        raise ValueError(f"Invalid refspec: {refspec}")
+
+    log = err if verbose else False
+    path1 = dvc_cache_path(before, dvc_path, log=log)
+    path2 = (path if after is None else dvc_cache_path(after, dvc_path, log=log))
+
+    if isdir(path):
+        dir_json1 = dir_json2 = {}
+        if path1:
+            with open(path1, 'r') as f:
+                obj = json.load(f)
+                dir_json1 = { e["relpath"]: e["md5"] for e in obj }
+        if path2:
+            if path2 == path and after is None:
+                dir_json2 = {}
+                for file in listdir(path2):
+                    md5 = hash_file(join(path2, file), hash_name='md5')
+                    dir_json2[file] = md5
+            else:
+                with open(path2, 'r') as f:
+                    dir_json2 = { obj["relpath"]: obj["md5"] for obj in json.load(f) }
+        for relpath in sorted(set(dir_json1) | set(dir_json2)):
+            md5_1 = dir_json1.get(relpath)
+            md5_2 = dir_json2.get(relpath)
+            if md5_1 != md5_2:
+                print(f'{relpath}: {md5_1} -> {md5_2}')
+    else:
+        diff_args = [
+            *(['-w'] if ignore_whitespace else []),
+            *(['-U', str(unified)] if unified is not None else []),
+            *(['--color=always'] if color is True else ['--color=never'] if color is False else []),
+        ]
+        if cmds:
+            cmd, *sub_cmds = cmds
+            cmds1 = [ 'cat /dev/null' ] if path1 is None else [ f'{cmd} {path1 or "/dev/null"}', *sub_cmds ]
+            cmds2 = [ 'cat /dev/null' ] if path2 is None else [ f'{cmd} {path2 or "/dev/null"}', *sub_cmds ]
+            if not shell:
+                cmds1 = [ shlex.split(cmd) for cmd in cmds1 ]
+                cmds2 = [ shlex.split(cmd) for cmd in cmds2 ]
+
+            returncode = join_pipelines(
+                base_cmd=['diff', *diff_args],
+                cmds1=cmds1,
+                cmds2=cmds2,
+                verbose=verbose,
+                shell=shell,
+                executable=shell_executable,
+            )
+            exit(returncode)
+        else:
+            res = process.run('diff', *diff_args, path1 or '/dev/null', path2 or '/dev/null', log=log, check=False)
+            exit(res.returncode)

dvc_utils/main.py

@@ -0,0 +1,9 @@
+from dvc_utils.cli import cli
+from dvc_utils import diff
+
+def main():
+    cli()
+
+
+if __name__ == '__main__':
+    cli()

dvc_utils/path.py

@@ -49,8 +49,13 @@ def dvc_md5(
     dir_path = get_dir_path()
     dir_path = '' if dir_path == '.' else f'{dir_path}{sep}'
     dvc_path = f"{dir_path}{dvc_path}"
-    dvc_spec = process.output('git', 'show', f'{git_ref}:{dvc_path}', log=err if log else None, err_ok=True, stderr=DEVNULL)
-    if dvc_spec is None:
+    dvc_spec = process.output(
+        'git', 'show', f'{git_ref}:{dvc_path}',
+        err_ok=True,
+        log=err if log else None,
+        stderr=None if log else DEVNULL,
+    )
+    if not dvc_spec:
         cur_dir = dirname(dvc_path)
         relpath = basename(dvc_path)
         if relpath.endswith(".dvc"):

dvc_utils/sync.py

@@ -0,0 +1,13 @@
+from click import option
+
+from dvc_utils.cli import cli
+from git import Repo
+
+
+@cli.command('pull-x', short_help='Sync DVC cache files from an S3 remote')
+@option('-n', '--dry-run', is_flag=True, help='Print files that would be synced, don\'t actually perform sync')
+@option('-p', '--path', 'paths', multiple=True, help='Path globs to sync')
+@option('-r', '--ref', 'refs', multiple=True, help='Git refs to sync DVC files from')
+def pull_x(dry_run, paths, refs):
+    repo = Repo()
+

{dvc_utils-0.2.0.dist-info → dvc_utils-0.3.1.dist-info}/METADATA

@@ -1,17 +1,23 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: dvc-utils
-Version: 0.2.0
-Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
-Home-page: https://github.com/runsascoded/dvc-utils
-Author: Ryan Williams
-Author-email: ryan@runsascoded.com
+Version: 0.3.1
+Summary: CLI for diffing DVC-tracked files at two commits (or one commit vs. current worktree), optionally passing both through another command first
+Project-URL: Homepage, https://github.com/runsascoded/dvc-utils
+Project-URL: Author URL, https://github.com/ryan-williams
+Author-email: Ryan Williams <ryan@runsascoded.com>
 License: MIT
-Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Python: >=3.9
 Requires-Dist: click
+Requires-Dist: dffs>=0.0.6
 Requires-Dist: pyyaml
-Requires-Dist: qmdx>=0.0.5
-Requires-Dist: utz>=0.13.0
+Requires-Dist: utz>=0.20.0
+Provides-Extra: ci
+Requires-Dist: bmdf==0.5.2; extra == 'ci'
+Requires-Dist: dvc-s3; extra == 'ci'
+Provides-Extra: test
+Requires-Dist: pytest>=7.0.0; extra == 'test'
+Description-Content-Type: text/markdown
 
 # dvc-utils
 Diff [DVC] files, optionally piping through other commands first.
@@ -36,25 +42,11 @@ pip install dvc-utils
 ```
 
 ## Usage <a id="usage"></a>
-<!-- `bmdf -- dvc-utils --help` -->
-```bash
-dvc-utils --help
-# Usage: dvc-utils [OPTIONS] COMMAND [ARGS]...
-#
-# Options:
-#   --help  Show this message and exit.
-#
-# Commands:
-#   diff  Diff a DVC-tracked file at two commits (or one commit vs. current
-#         worktree), optionally passing both through another command first
-```
+Currently one command is exposed, `dvc-diff`:
 
-The single subcommand, `dvc-utils diff`, is also exposed directly as `dvc-dff`:
-
-### `dvc-diff` <a id="dvc-diff"></a>
-<!-- `bmdf -- dvc-diff --help` -->
+<!-- `bmdf -- dvc-diff` -->
 ```bash
-dvc-diff --help
+dvc-diff
 # Usage: dvc-diff [OPTIONS] [exec_cmd...] <path>
 #
 #   Diff a file at two commits (or one commit vs. current worktree), optionally
@@ -91,9 +83,8 @@ dvc-diff --help
 ```
 
 ## Examples <a id="examples"></a>
-These examples are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data`
-
-([`test/data`] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files used for testing [`git-diff-dvc.sh`])
+- Examples below are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data` (see [.github/workflows/ci.yml](.github/workflows/ci.yml)).
+- [test/data] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files (used in that repo for testing [`git-diff-dvc.sh`]).
 
 [`8ec2060`] added a DVC-tracked text file, `test.txt`:
 
@@ -163,9 +154,9 @@ dvc-diff -R f92c1d2 pqa test.parquet
 
 [`f29e52a`] updated `test.parquet`:
 
-<!-- `bmdf -- dvc-diff -R f29e52a pqa test.parquet` -->
+<!-- `bmdf -E PQT_TXT_OPTS=-n2 -- dvc-diff -R f29e52a pqa test.parquet` -->
 ```bash
-dvc-diff -R f29e52a pqa test.parquet
+PQT_TXT_OPTS=-n2 dvc-diff -R f29e52a pqa test.parquet
 # 1,3c1,3
 # < MD5: 4379600b26647a50dfcd0daa824e8219
 # < 1635 bytes
@@ -178,17 +169,16 @@ dvc-diff -R f29e52a pqa test.parquet
 # <   OPTIONAL INT64 num;
 # ---
 # >   OPTIONAL INT32 num;
-# 26a27,38
-# > }
-# > {
-# >   "num": 666,
-# >   "str": "fff"
-# > }
-# > {
+# 19,20c19,20
+# <   "num": 444,
+# <   "str": "ddd"
+# ---
 # >   "num": 777,
 # >   "str": "ggg"
-# > }
-# > {
+# 23,24c23,24
+# <   "num": 555,
+# <   "str": "eee"
+# ---
 # >   "num": 888,
 # >   "str": "hhh"
 ```
@@ -462,6 +452,7 @@ This helped me see that the data update in question (`c0..c1`) dropped some fiel
 
 [`mdcmd`]: https://github.com/runsascoded/bash-markdown-fence?tab=readme-ov-file#bmdf
 [`test/data`]: test/data
+[test/data]: test/data
 [ryan-williams/dvc-helpers@test]: https://github.com/ryan-williams/dvc-helpers/tree/test
 [`git-diff-dvc.sh`]: https://github.com/ryan-williams/dvc-helpers/blob/main/git-diff-dvc.sh
 

dvc_utils-0.3.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+dvc_utils/__init__.py,sha256=Q9_9py_bah16WN6RIryNMelBvG9IJW1B1Sm2PEU87Yk,92
+dvc_utils/cli.py,sha256=et2OeEWJeY0XUuStMVd8qmjtOgDbXl9esn3KYZCQFSI,318
+dvc_utils/diff.py,sha256=k4_1rjCfvtQkTwelPkrp2m6lrh8lU4IUg7aaKw3YhU8,5112
+dvc_utils/main.py,sha256=cfbwebEc9Auh9DXyah31kvd9Br_uCsUxzOXkucfiSnM,119
+dvc_utils/path.py,sha256=tdigsKTCZPhCIb9wRX4e_tj1lUxa2gm6lXERlfb8qY4,2955
+dvc_utils/sync.py,sha256=gDwo0Z4lMK--xRwsACIpJRWS5rF6g7Wou0LJRxNd6b4,481
+dvc_utils-0.3.1.dist-info/METADATA,sha256=aWI-NtzxzF66gfP7jH3oVf7n6HDtAzvuJg8biTFk7Cw,14986
+dvc_utils-0.3.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+dvc_utils-0.3.1.dist-info/entry_points.txt,sha256=VHnkBNH2bPbP1wX8-rKK_T13f3e9tnmAGBcrP0hjTr0,85
+dvc_utils-0.3.1.dist-info/licenses/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
+dvc_utils-0.3.1.dist-info/RECORD,,

{dvc_utils-0.2.0.dist-info → dvc_utils-0.3.1.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.1)
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any
-

dvc_utils-0.3.1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dvc-diff = dvc_utils.diff:dvc_diff
+dvc-utils = dvc_utils.main:main

dvc_utils-0.2.0.dist-info/RECORD

@@ -1,9 +0,0 @@
-dvc_utils/__init__.py,sha256=mP-p1Sl2JMMShM_hRhu86pFNfIq_8E_feh1CN47LWcs,86
-dvc_utils/cli.py,sha256=d27Q8K77ZxwWRtHJqn_70MhfOYF8ybsCEamG7wrAyjU,5079
-dvc_utils/path.py,sha256=eEP-r6o33BZsEz6JNllzEbU8ficW9slAFMJ2l02IRrY,2903
-dvc_utils-0.2.0.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
-dvc_utils-0.2.0.dist-info/METADATA,sha256=ppHbZlkyxL015AhfAWcx9H6UGWLNt2078FjdMIQgCok,14977
-dvc_utils-0.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
-dvc_utils-0.2.0.dist-info/entry_points.txt,sha256=luxCQr8OS-jMSyyDhB9KDQhUbP8UH6UMcy-vkfXX7Gg,88
-dvc_utils-0.2.0.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
-dvc_utils-0.2.0.dist-info/RECORD,,

dvc_utils-0.2.0.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-dvc-diff = dvc_utils.cli:dvc_utils_diff
-dvc-utils = dvc_utils.cli:cli

dvc_utils-0.2.0.dist-info/top_level.txt

@@ -1 +0,0 @@
-dvc_utils