dvc-utils 0.0.9__py3-none-any.whl → 0.2.0__py3-none-any.whl

dvc_utils/cli.py

@@ -1,12 +1,15 @@
+import json
 import shlex
-from os import environ as env
+from os import listdir
+from os.path import isdir, join
 from typing import Tuple
 
 import click
 from click import option, argument, group
-from utz import diff_cmds, process, err
+from qmdx import join_pipelines
+from utz import process, err, hash_file
 
-from dvc_utils.path import dvc_paths, dvc_path as dvc_cache_path
+from dvc_utils.path import dvc_paths, dvc_cache_path
 
 
 @group()
@@ -15,9 +18,10 @@ def cli():
 
 
 @cli.command('diff', short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first')
-@option('-c', '--color', is_flag=True, help='Colorize the output')
-@option('-r', '--refspec', default='HEAD', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
-@option('-s', '--shell-executable', help=f'Shell to use for executing commands; defaults to $SHELL ({env.get("SHELL")})')
+@option('-c/-C', '--color/--no-color', default=None, help='Force or prevent colorized output')
+@option('-r', '--refspec', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
+@option('-R', '--ref', help='Shorthand for `-r <ref>^..<ref>`, i.e. inspect a specific commit (vs. its parent)')
+@option('-s', '--shell-executable', help=f'Shell to use for executing commands; defaults to $SHELL')
 @option('-S', '--no-shell', is_flag=True, help="Don't pass `shell=True` to Python `subprocess`es")
 @option('-U', '--unified', type=int, help='Number of lines of context to show (passes through to `diff`)')
 @option('-v', '--verbose', is_flag=True, help="Log intermediate commands to stderr")
@@ -25,8 +29,9 @@ def cli():
 @option('-x', '--exec-cmd', 'exec_cmds', multiple=True, help='Command(s) to execute before diffing; alternate syntax to passing commands as positional arguments')
 @argument('args', metavar='[exec_cmd...] <path>', nargs=-1)
 def dvc_utils_diff(
-    color: bool,
+    color: bool | None,
     refspec: str | None,
+    ref: str | None,
     shell_executable: str | None,
     no_shell: bool,
     unified: int | None,
@@ -52,6 +57,13 @@ def dvc_utils_diff(
 
     path, dvc_path = dvc_paths(path)
 
+    if refspec and ref:
+        raise ValueError("Specify -r/--refspec xor -R/--ref")
+    if ref:
+        refspec = f'{ref}^..{ref}'
+    elif not refspec:
+        refspec = 'HEAD'
+
     pcs = refspec.split('..', 1)
     if len(pcs) == 1:
         before = pcs[0]
@@ -62,39 +74,54 @@ def dvc_utils_diff(
         raise ValueError(f"Invalid refspec: {refspec}")
 
     log = err if verbose else False
-    before_path = dvc_cache_path(before, dvc_path, log=log)
-    after_path = path if after is None else dvc_cache_path(after, dvc_path, log=log)
-
-    if cmds:
-        cmd, *sub_cmds = cmds
-        if not shell:
-            sub_cmds = [ shlex.split(c) for c in sub_cmds ]
-            before_cmds = [
-                shlex.split(f'{cmd} {before_path}'),
-                *sub_cmds,
-            ]
-            after_cmds = [
-                shlex.split(f'{cmd} {after_path}'),
-                *sub_cmds,
-            ]
-            shell_kwargs = {}
-        else:
-            before_cmds = [ f'{cmd} {before_path}', *sub_cmds ]
-            after_cmds = [ f'{cmd} {after_path}', *sub_cmds ]
-            shell_kwargs = dict(shell=shell)
-
-        diff_cmds(
-            before_cmds,
-            after_cmds,
-            verbose=verbose,
-            color=color,
-            unified=unified,
-            ignore_whitespace=ignore_whitespace,
-            shell_executable=shell_executable,
-            **shell_kwargs,
-        )
+    path1 = dvc_cache_path(before, dvc_path, log=log)
+    path2 = (path if after is None else dvc_cache_path(after, dvc_path, log=log))
+
+    if isdir(path):
+        dir_json1 = dir_json2 = {}
+        if path1:
+            with open(path1, 'r') as f:
+                obj = json.load(f)
+                dir_json1 = { e["relpath"]: e["md5"] for e in obj }
+        if path2:
+            if path2 == path and after is None:
+                dir_json2 = {}
+                for file in listdir(path2):
+                    md5 = hash_file(join(path2, file), hash_name='md5')
+                    dir_json2[file] = md5
+            else:
+                with open(path2, 'r') as f:
+                    dir_json2 = { obj["relpath"]: obj["md5"] for obj in json.load(f) }
+        for relpath in sorted(set(dir_json1) | set(dir_json2)):
+            md5_1 = dir_json1.get(relpath)
+            md5_2 = dir_json2.get(relpath)
+            if md5_1 != md5_2:
+                print(f'{relpath}: {md5_1} -> {md5_2}')
     else:
-        process.run('diff', before_path, after_path, log=log)
+        diff_args = [
+            *(['-w'] if ignore_whitespace else []),
+            *(['-U', str(unified)] if unified is not None else []),
+            *(['--color=always'] if color is True else ['--color=never'] if color is False else []),
+        ]
+        if cmds:
+            cmd, *sub_cmds = cmds
+            cmds1 = [ 'cat /dev/null' ] if path1 is None else [ f'{cmd} {path1 or "/dev/null"}', *sub_cmds ]
+            cmds2 = [ 'cat /dev/null' ] if path2 is None else [ f'{cmd} {path2 or "/dev/null"}', *sub_cmds ]
+            if not shell:
+                cmds1 = [ shlex.split(cmd) for cmd in cmds1 ]
+                cmds2 = [ shlex.split(cmd) for cmd in cmds2 ]
+
+            join_pipelines(
+                base_cmd=['diff', *diff_args],
+                cmds1=cmds1,
+                cmds2=cmds2,
+                verbose=verbose,
+                shell=shell,
+                executable=shell_executable,
+            )
+        else:
+            res = process.run('diff', *diff_args, path1 or '/dev/null', path2 or '/dev/null', log=log, check=False)
+            exit(res.returncode)
 
 
 if __name__ == '__main__':

dvc_utils/path.py

@@ -1,13 +1,19 @@
+from __future__ import annotations
+
+import json
 from functools import cache
 from os import environ as env, getcwd
-from os.path import join, relpath
-from typing import Optional, Tuple
+from os.path import join, relpath, dirname, basename, sep
+from subprocess import DEVNULL
+from typing import Tuple
 
 import yaml
 from utz import process, err, singleton
 
 
 def dvc_paths(path: str) -> Tuple[str, str]:
+    if path.endswith(sep):
+        path = path[:-len(sep)]
     if path.endswith('.dvc'):
         dvc_path = path
         path = dvc_path[:-len('.dvc')]
@@ -35,19 +41,47 @@ def dvc_cache_dir(log: bool = False) -> str:
         return process.line('dvc', 'cache', 'dir', log=log)
 
 
-def dvc_md5(git_ref: str, dvc_path: str, log: bool = False) -> str:
+def dvc_md5(
+    git_ref: str,
+    dvc_path: str,
+    log: bool = False,
+) -> str | None:
     dir_path = get_dir_path()
-    dir_path = '' if dir_path == '.' else f'{dir_path}/'
-    dvc_spec = process.output('git', 'show', f'{git_ref}:{dir_path}{dvc_path}', log=err if log else None)
+    dir_path = '' if dir_path == '.' else f'{dir_path}{sep}'
+    dvc_path = f"{dir_path}{dvc_path}"
+    dvc_spec = process.output('git', 'show', f'{git_ref}:{dvc_path}', log=err if log else None, err_ok=True, stderr=DEVNULL)
+    if dvc_spec is None:
+        cur_dir = dirname(dvc_path)
+        relpath = basename(dvc_path)
+        if relpath.endswith(".dvc"):
+            relpath = relpath[:-len(".dvc")]
+        while cur_dir and cur_dir != '.':
+            dir_cache_path = dvc_cache_path(ref=git_ref, dvc_path=f"{cur_dir}.dvc", log=log)
+            if dir_cache_path:
+                with open(dir_cache_path, 'r') as f:
+                    dir_entries = json.load(f)
+                md5s = [ e["md5"] for e in dir_entries if e["relpath"] == relpath ]
+                if len(md5s) == 1:
+                    return md5s[0]
+                else:
+                    raise RuntimeError(f"{relpath=} not found in DVC-tracked dir {cur_dir}")
+            relpath = join(basename(cur_dir), relpath)
+            cur_dir = dirname(cur_dir)
+        return None
     dvc_obj = yaml.safe_load(dvc_spec)
     out = singleton(dvc_obj['outs'], dedupe=False)
     md5 = out['md5']
     return md5
 
 
-def dvc_path(ref: str, dvc_path: Optional[str] = None, log: bool = False) -> str:
+def dvc_path(
+    ref: str,
+    dvc_path: str | None = None,
+    log: bool = False,
+) -> str | None:
     if dvc_path and not dvc_path.endswith('.dvc'):
         dvc_path += '.dvc'
+
     if dvc_path:
         md5 = dvc_md5(ref, dvc_path, log=log)
     elif ':' in ref:
@@ -55,6 +89,13 @@ def dvc_path(ref: str, dvc_path: Optional[str] = None, log: bool = False) -> str
         md5 = dvc_md5(git_ref, dvc_path, log=log)
     else:
         md5 = ref
-    dirname = md5[:2]
-    basename = md5[2:]
-    return join(dvc_cache_dir(log=log), 'files', 'md5', dirname, basename)
+
+    if md5 is None:
+        return None
+    else:
+        dirname = md5[:2]
+        basename = md5[2:]
+        return join(dvc_cache_dir(log=log), 'files', 'md5', dirname, basename)
+
+
+dvc_cache_path = dvc_path

{dvc_utils-0.0.9.dist-info → dvc_utils-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dvc-utils
-Version: 0.0.9
+Version: 0.2.0
 Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
 Home-page: https://github.com/runsascoded/dvc-utils
 Author: Ryan Williams
@@ -8,10 +8,16 @@ Author-email: ryan@runsascoded.com
 License: MIT
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: click
+Requires-Dist: pyyaml
+Requires-Dist: qmdx>=0.0.5
+Requires-Dist: utz>=0.13.0
 
 # dvc-utils
 Diff [DVC] files, optionally piping through other commands first.
 
+[![dvc-utils on PyPI](https://img.shields.io/pypi/v/dvc-utils?label=dvc-utils)][PyPI]
+
 <!-- toc -->
 - [Installation](#installation)
 - [Usage](#usage)
@@ -64,25 +70,146 @@ dvc-diff --help
 #   optional) at HEAD (last committed value) vs. the current worktree content.
 #
 # Options:
-#   -c, --color                  Colorize the output
-#   -r, --refspec TEXT           <commit 1>..<commit 2> (compare two commits) or
-#                                <commit> (compare <commit> to the worktree)
-#   -s, --shell-executable TEXT  Shell to use for executing commands; defaults
-#                                to $SHELL (/bin/bash)
-#   -S, --no-shell               Don't pass `shell=True` to Python
-#                                `subprocess`es
-#   -U, --unified INTEGER        Number of lines of context to show (passes
-#                                through to `diff`)
-#   -v, --verbose                Log intermediate commands to stderr
-#   -w, --ignore-whitespace      Ignore whitespace differences (pass `-w` to
-#                                `diff`)
-#   -x, --exec-cmd TEXT          Command(s) to execute before diffing; alternate
-#                                syntax to passing commands as positional
-#                                arguments
-#   --help                       Show this message and exit.
+#   -c, --color / -C, --no-color  Force or prevent colorized output
+#   -r, --refspec TEXT            <commit 1>..<commit 2> (compare two commits)
+#                                 or <commit> (compare <commit> to the worktree)
+#   -R, --ref TEXT                Shorthand for `-r <ref>^..<ref>`, i.e. inspect
+#                                 a specific commit (vs. its parent)
+#   -s, --shell-executable TEXT   Shell to use for executing commands; defaults
+#                                 to $SHELL
+#   -S, --no-shell                Don't pass `shell=True` to Python
+#                                 `subprocess`es
+#   -U, --unified INTEGER         Number of lines of context to show (passes
+#                                 through to `diff`)
+#   -v, --verbose                 Log intermediate commands to stderr
+#   -w, --ignore-whitespace       Ignore whitespace differences (pass `-w` to
+#                                 `diff`)
+#   -x, --exec-cmd TEXT           Command(s) to execute before diffing;
+#                                 alternate syntax to passing commands as
+#                                 positional arguments
+#   --help                        Show this message and exit.
 ```
 
 ## Examples <a id="examples"></a>
+These examples are verified with [`mdcmd`] and `$BMDF_WORKDIR=test/data`
+
+([`test/data`] is a clone of [ryan-williams/dvc-helpers@test], which contains simple DVC-tracked files used for testing [`git-diff-dvc.sh`])
+
+[`8ec2060`] added a DVC-tracked text file, `test.txt`:
+
+<!-- `bmdf -- dvc-diff -R 8ec2060 test.txt` -->
+```bash
+dvc-diff -R 8ec2060 test.txt
+# 0a1,10
+# > 1
+# > 2
+# > 3
+# > 4
+# > 5
+# > 6
+# > 7
+# > 8
+# > 9
+# > 10
+```
+
+[`0455b50`] appended some lines to `test.txt`:
+
+<!-- `bmdf -- dvc-diff -R 0455b50 test.txt` -->
+```bash
+dvc-diff -R 0455b50 test.txt
+# 10a11,15
+# > 11
+# > 12
+# > 13
+# > 14
+# > 15
+```
+
+[`f92c1d2`] added `test.parquet`:
+
+<!-- `bmdf -- dvc-diff -R f92c1d2 pqa test.parquet` -->
+```bash
+dvc-diff -R f92c1d2 pqa test.parquet
+# 0a1,27
+# > MD5: 4379600b26647a50dfcd0daa824e8219
+# > 1635 bytes
+# > 5 rows
+# > message schema {
+# >   OPTIONAL INT64 num;
+# >   OPTIONAL BYTE_ARRAY str (STRING);
+# > }
+# > {
+# >   "num": 111,
+# >   "str": "aaa"
+# > }
+# > {
+# >   "num": 222,
+# >   "str": "bbb"
+# > }
+# > {
+# >   "num": 333,
+# >   "str": "ccc"
+# > }
+# > {
+# >   "num": 444,
+# >   "str": "ddd"
+# > }
+# > {
+# >   "num": 555,
+# >   "str": "eee"
+# > }
+```
+
+[`f29e52a`] updated `test.parquet`:
+
+<!-- `bmdf -- dvc-diff -R f29e52a pqa test.parquet` -->
+```bash
+dvc-diff -R f29e52a pqa test.parquet
+# 1,3c1,3
+# < MD5: 4379600b26647a50dfcd0daa824e8219
+# < 1635 bytes
+# < 5 rows
+# ---
+# > MD5: be082c87786f3364ca9efec061a3cc21
+# > 1622 bytes
+# > 8 rows
+# 5c5
+# <   OPTIONAL INT64 num;
+# ---
+# >   OPTIONAL INT32 num;
+# 26a27,38
+# > }
+# > {
+# >   "num": 666,
+# >   "str": "fff"
+# > }
+# > {
+# >   "num": 777,
+# >   "str": "ggg"
+# > }
+# > {
+# >   "num": 888,
+# >   "str": "hhh"
+```
+
+[`3257258`] added a DVC-tracked directory `data/`, including `test.{txt,parquet}`), and removed the top-level `test.{txt,parquet}`.
+
+<!-- `bmdf -- dvc-diff -R 3257258 data` -->
+```bash
+dvc-diff -R 3257258 data
+# test.parquet: None -> c07bba3fae2b64207aa92f422506e4a2
+# test.txt: None -> e20b902b49a98b1a05ed62804c757f94
+```
+
+[`ae8638a`] changed values in `data/test.parquet`, and added rows to `data/test.txt`:
+
+<!-- `bmdf -- dvc-diff -R ae8638a data` -->
+```bash
+dvc-diff -R ae8638a data
+# test.parquet: c07bba3fae2b64207aa92f422506e4a2 -> f46dd86f608b1dc00993056c9fc55e6e
+# test.txt: e20b902b49a98b1a05ed62804c757f94 -> 9306ec0709cc72558045559ada26573b
+```
 
 ### Parquet <a id="parquet-diff"></a>
 See sample commands and output below for inspecting changes to [a DVC-tracked Parquet file][commit path] in [a given commit][commit].
@@ -246,7 +373,7 @@ pel "ddcr $r guc h1 spc kq kcr snc 'sdf seds' sort"
 
 <details>
 <summary>
-Aliases used in the pipeline:
+Explanation of aliases
 </summary>
 
 - [`gdno`] (`git diff --name-only`): list files changed in the given commit range and directory
@@ -254,9 +381,9 @@ Aliases used in the pipeline:
 - [`ddcr`] (`dvc-diff -cr`): colorized `diff` output, revision range `$r`
 - [`guc`] (`gunzip -c`): uncompress the `.csv.gz` files
 - [`h1`] (`head -n1`): only examine each file's header line
-- [`spc`] (`tr , $'\n'`): split the header line by commas (so each column name will be on one line, for easier `diff`ing below)
-- [`kq`] (`tr -d '"'`): kill quote characters (in this case, header-column name quoting changed, but I don't care about that)
-- [`kcr`] (`tr -d '\r'`): kill carriage returns (line endings also changed)
+- [`spc`] (`tr , $'\n'`): **sp**lit the header line by **c**ommas (so each column name will be on one line, for easier `diff`ing below)
+- [`kq`] (`tr -d '"'`): **k**ill **q**uote characters (in this case, header-column name quoting changed, but I don't care about that)
+- [`kcr`] (`tr -d '\r'`): **k**ill **c**arriage **r**eturns (line endings also changed)
 - [`snc`] (`sed -f 'snake_case.sed'`): snake-case column names
 - [`sdf`] (`sed -f`): execute the `sed` substitution commands defined in the `seds` file above
 - `sort`: sort the column names alphabetically (to identify missing or added columns, ignore rearrangements)
@@ -311,6 +438,7 @@ s3/ctbk/csvs/202003-citibike-tripdata.csv.gz.dvc:
 This helped me see that the data update in question (`c0..c1`) dropped some fields (`bikeid, birth_year`, `gender`, `tripduration`) and added others (`ride_id`, `rideable_type`), for `202001` and later.
 
 [DVC]: https://dvc.org/
+[PyPI]: https://pypi.org/project/dvc-utils/
 [`parquet2json`]: https://github.com/jupiter/parquet2json
 [hudcostreets/nj-crashes]: https://github.com/hudcostreets/nj-crashes
 [Parquet]: https://parquet.apache.org/
@@ -331,3 +459,15 @@ This helped me see that the data update in question (`c0..c1`) dropped some fiel
 [`kcr`]: https://github.com/ryan-williams/arg-helpers/blob/a8c60809f8878fa38b3c03614778fcf29132538e/.arg-rc#L118
 [`snc`]: https://github.com/ryan-williams/case-helpers/blob/c40a62a9656f0d52d68fb3a108ae6bb3eed3c7bd/.case-rc#L9
 [`sdf`]: https://github.com/ryan-williams/arg-helpers/blob/a8c60809f8878fa38b3c03614778fcf29132538e/.arg-rc#L138
+
+[`mdcmd`]: https://github.com/runsascoded/bash-markdown-fence?tab=readme-ov-file#bmdf
+[`test/data`]: test/data
+[ryan-williams/dvc-helpers@test]: https://github.com/ryan-williams/dvc-helpers/tree/test
+[`git-diff-dvc.sh`]: https://github.com/ryan-williams/dvc-helpers/blob/main/git-diff-dvc.sh
+
+[`8ec2060`]: https://github.com/ryan-williams/dvc-helpers/commit/8ec2060
+[`0455b50`]: https://github.com/ryan-williams/dvc-helpers/commit/0455b50
+[`f92c1d2`]: https://github.com/ryan-williams/dvc-helpers/commit/f92c1d2
+[`f29e52a`]: https://github.com/ryan-williams/dvc-helpers/commit/f29e52a
+[`3257258`]: https://github.com/ryan-williams/dvc-helpers/commit/3257258
+[`ae8638a`]: https://github.com/ryan-williams/dvc-helpers/commit/ae8638a

dvc_utils-0.2.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+dvc_utils/__init__.py,sha256=mP-p1Sl2JMMShM_hRhu86pFNfIq_8E_feh1CN47LWcs,86
+dvc_utils/cli.py,sha256=d27Q8K77ZxwWRtHJqn_70MhfOYF8ybsCEamG7wrAyjU,5079
+dvc_utils/path.py,sha256=eEP-r6o33BZsEz6JNllzEbU8ficW9slAFMJ2l02IRrY,2903
+dvc_utils-0.2.0.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
+dvc_utils-0.2.0.dist-info/METADATA,sha256=ppHbZlkyxL015AhfAWcx9H6UGWLNt2078FjdMIQgCok,14977
+dvc_utils-0.2.0.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+dvc_utils-0.2.0.dist-info/entry_points.txt,sha256=luxCQr8OS-jMSyyDhB9KDQhUbP8UH6UMcy-vkfXX7Gg,88
+dvc_utils-0.2.0.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
+dvc_utils-0.2.0.dist-info/RECORD,,

{dvc_utils-0.0.9.dist-info → dvc_utils-0.2.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.45.0)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dvc_utils-0.0.9.dist-info/RECORD

@@ -1,9 +0,0 @@
-dvc_utils/__init__.py,sha256=mP-p1Sl2JMMShM_hRhu86pFNfIq_8E_feh1CN47LWcs,86
-dvc_utils/cli.py,sha256=D5GGDPpMlKG-8IHkTXPIB2z-QAKHzbm5gQXqj7vCquY,3701
-dvc_utils/path.py,sha256=PoAbeaqRPDksY2hcUeF8xZ6Nr6hLIZprey3VNT4V5bc,1727
-dvc_utils-0.0.9.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
-dvc_utils-0.0.9.dist-info/METADATA,sha256=zhFkhQbnbOYwwhHJiXUj8otqRw5tfmVsXx-b5MlByvw,11480
-dvc_utils-0.0.9.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-dvc_utils-0.0.9.dist-info/entry_points.txt,sha256=luxCQr8OS-jMSyyDhB9KDQhUbP8UH6UMcy-vkfXX7Gg,88
-dvc_utils-0.0.9.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
-dvc_utils-0.0.9.dist-info/RECORD,,