dvc-utils 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

dvc_utils/main.py

@@ -1,5 +1,12 @@
+from functools import cache
+from os import environ as env, getcwd
+
+from typing import Optional, Tuple
+
 import shlex
-from os.path import join
+from os.path import join, relpath
+
+from click import option, argument, group
 from subprocess import Popen
 
 import click
@@ -9,12 +16,12 @@ from utz import process, singleton, err
 from dvc_utils.named_pipes import named_pipes
 
 
-@click.group()
+@group()
 def cli():
     pass
 
 
-def dvc_paths(path):
+def dvc_paths(path: str) -> Tuple[str, str]:
     if path.endswith('.dvc'):
         dvc_path = path
         path = dvc_path[:-len('.dvc')]
@@ -23,54 +30,70 @@ def dvc_paths(path):
     return path, dvc_path
 
 
-def dvc_md5(git_ref, dvc_path, log=False):
-    dvc_spec = process.output('git', 'show', f'{git_ref}:{dvc_path}', log=log)
+@cache
+def get_git_root() -> str:
+    return process.line('git', 'rev-parse', '--show-toplevel', log=False)
+
+
+@cache
+def get_dir_path() -> str:
+    return relpath(getcwd(), get_git_root())
+
+
+@cache
+def dvc_cache_dir(log: bool = False) -> str:
+    dvc_cache_relpath = env.get('DVC_UTILS_CACHE_DIR')
+    if dvc_cache_relpath:
+        return join(get_git_root(), dvc_cache_relpath)
+    else:
+        return process.line('dvc', 'cache', 'dir', log=log)
+
+
+def dvc_md5(git_ref: str, dvc_path: str, log: bool = False) -> str:
+    dir_path = get_dir_path()
+    dir_path = '' if dir_path == '.' else f'{dir_path}/'
+    dvc_spec = process.output('git', 'show', f'{git_ref}:{dir_path}{dvc_path}', log=log)
     dvc_obj = yaml.safe_load(dvc_spec)
     out = singleton(dvc_obj['outs'], dedupe=False)
     md5 = out['md5']
     return md5
 
 
-_dvc_cache_dir = None
-def dvc_cache_dir(log=False):
-    global _dvc_cache_dir
-    if _dvc_cache_dir is None:
-        _dvc_cache_dir = process.line('dvc', 'cache', 'dir', log=log)
-    return _dvc_cache_dir
-
-
-def dvc_cache_path(spec, dvc_path=None, log=False):
+def dvc_cache_path(ref: str, dvc_path: Optional[str] = None, log: bool = False) -> str:
     if dvc_path:
-        md5 = dvc_md5(spec, dvc_path, log=log)
-    elif ':' in spec:
-        git_ref, dvc_path = spec.split(':', 1)
+        md5 = dvc_md5(ref, dvc_path, log=log)
+    elif ':' in ref:
+        git_ref, dvc_path = ref.split(':', 1)
         md5 = dvc_md5(git_ref, dvc_path, log=log)
     else:
-        md5 = spec
+        md5 = ref
     dirname = md5[:2]
     basename = md5[2:]
     return join(dvc_cache_dir(log=log), 'files', 'md5', dirname, basename)
 
 
-def diff_cmds(cmd1, cmd2, **kwargs):
+def diff_cmds(cmd1: str, cmd2: str, verbose: bool = False, **kwargs):
     """Run two commands and diff their output.
 
     Adapted from https://stackoverflow.com/a/28840955"""
-    with named_pipes(n=2) as paths:
-        someprogram = Popen(['diff'] + paths)
+    with named_pipes(n=2) as pipes:
+        (pipe1, pipe2) = pipes
+        diff = Popen(['diff'] + pipes)
         processes = []
-        for path, cmd in zip(paths, [ cmd1, cmd2 ]):
+        for path, cmd in ((pipe1, cmd1), (pipe2, cmd2)):
             with open(path, 'wb', 0) as pipe:
+                if verbose:
+                    err(f"Running: {cmd}")
                 processes.append(Popen(cmd, stdout=pipe, close_fds=True, **kwargs))
-        for p in [someprogram] + processes:
+        for p in [diff] + processes:
             p.wait()
 
 
 @cli.command('diff', short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first')
-@click.option('-r', '--refspec', default='HEAD', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
-@click.option('-S', '--no-shell', is_flag=True, help="Don't pass `shell=True` to Python `subprocess`es")
-@click.option('-v', '--verbose', is_flag=True, help="Log intermediate commands to stderr")
-@click.argument('args', metavar='[cmd...] <path>', nargs=-1)
+@option('-r', '--refspec', default='HEAD', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
+@option('-S', '--no-shell', is_flag=True, help="Don't pass `shell=True` to Python `subprocess`es")
+@option('-v', '--verbose', is_flag=True, help="Log intermediate commands to stderr")
+@argument('args', metavar='[cmd...] <path>', nargs=-1)
 def dvc_utils_diff(refspec, no_shell, verbose, args):
     """Diff a file at two commits (or one commit vs. current worktree), optionally passing both through `cmd` first
 
@@ -84,33 +107,39 @@ def dvc_utils_diff(refspec, no_shell, verbose, args):
         raise click.UsageError('Must specify [cmd...] <path>')
 
     shell = not no_shell
-    (*cmd, path) = args
-    if path.endswith('.dvc'):
-        dvc_path = path
-        path = dvc_path[:-len('.dvc')]
+    if len(args) == 2:
+        cmd, path = args
+        cmd = shlex.split(cmd)
+    elif len(args) == 1:
+        cmd = None
+        path, = args
     else:
-        dvc_path = f'{path}.dvc'
+        raise click.UsageError('Maximum 2 positional args: [cmd] <path>')
+
+    path, dvc_path = dvc_paths(path)
 
     pcs = refspec.split('..', 1)
     if len(pcs) == 1:
         before = pcs[0]
         after = None
-    else:
+    elif len(pcs) == 2:
         before, after = pcs
+    else:
+        raise ValueError(f"Invalid refspec: {refspec}")
 
     log = err if verbose else False
     before_path = dvc_cache_path(before, dvc_path, log=log)
     after_path = path if after is None else dvc_cache_path(after, dvc_path, log=log)
 
     if cmd:
-        def args(path):
+        def args(path: str):
             arr = cmd + [path]
             return shlex.join(arr) if shell else arr
 
         shell_kwargs = dict(shell=shell) if shell else {}
         before_cmd = args(before_path)
         after_cmd = args(after_path)
-        diff_cmds(before_cmd, after_cmd, **shell_kwargs)
+        diff_cmds(before_cmd, after_cmd, verbose=verbose, **shell_kwargs)
     else:
         process.run('diff', before_path, after_path, log=log)
 

dvc_utils/named_pipes.py

@@ -5,7 +5,7 @@ from contextlib import contextmanager
 
 
 @contextmanager
-def named_pipes(n=1):
+def named_pipes(n: int = 1):
     """Yield a list of paths to named pipes that are created and destroyed
 
     From https://stackoverflow.com/a/28840955"""

dvc_utils-0.0.3.dist-info/METADATA

@@ -0,0 +1,217 @@
+Metadata-Version: 2.1
+Name: dvc-utils
+Version: 0.0.3
+Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
+Home-page: https://github.com/runsascoded/dvc-utils
+Author: Ryan Williams
+Author-email: ryan@runsascoded.com
+License: MIT
+Description-Content-Type: text/markdown
+License-File: LICENSE
+
+# dvc-utils
+CLI for diffing [DVC] files, optionally passing both through another command first
+
+<!-- toc -->
+- [Installation](#installation)
+- [Usage](#usage)
+    - [`dvc-utils diff`](#dvc-utils-diff)
+- [Examples](#examples)
+    - [Parquet file](#parquet-diff)
+        - [Schema diff](#parquet-schema-diff)
+        - [Row diff](#parquet-row-diff)
+        - [Row count diff](#parquet-row-count-diff)
+<!-- /toc -->
+
+## Installation <a id="installation"></a>
+```bash
+pip install dvc-utils
+```
+
+## Usage <a id="usage"></a>
+```bash
+dvc-utils --help
+# Usage: dvc-utils [OPTIONS] COMMAND [ARGS]...
+#
+# Options:
+#   --help  Show this message and exit.
+#
+# Commands:
+#   diff  Diff a DVC-tracked file at two commits (or one commit vs. current
+#         worktree), optionally passing both through another command first
+```
+
+### `dvc-utils diff` <a id="dvc-utils-diff"></a>
+```bash
+dvc-utils diff --help
+# Usage: dvc-utils diff [OPTIONS] [cmd...] <path>
+#
+#   Diff a file at two commits (or one commit vs. current worktree), optionally
+#   passing both through `cmd` first
+#
+#   Examples:
+#
+#   dvc-utils diff -r HEAD^..HEAD wc -l foo.dvc  # Compare the number of lines
+#   (`wc -l`) in `foo` (the file referenced by `foo.dvc`) at the previous vs.
+#   current commit (`HEAD^..HEAD`).
+#
+#   dvc-utils diff md5sum foo  # Diff the `md5sum` of `foo` (".dvc" extension is
+#   optional) at HEAD (last committed value) vs. the current worktree content.
+#
+# Options:
+#   -r, --refspec TEXT  <commit 1>..<commit 2> (compare two commits) or <commit>
+#                       (compare <commit> to the worktree)
+#   -S, --no-shell      Don't pass `shell=True` to Python `subprocess`es
+#   -v, --verbose       Log intermediate commands to stderr
+#   --help              Show this message and exit.
+```
+
+## Examples <a id="examples"></a>
+
+### Parquet file <a id="parquet-diff"></a>
+See sample commands and output below for inspecting changes to [a DVC-tracked Parquet file][commit path] in [a given commit][commit].
+
+Setup:
+```bash
+git clone https://github.com/hudcostreets/nj-crashes && cd nj-crashes # Clone + enter example repo
+commit=c8ae28e  # Example commit that changed some DVC-tracked Parquet files
+path=njdot/data/2001/NewJersey2001Accidents.pqt.dvc  # One of the changed files
+```
+
+#### Schema diff <a id="parquet-schema-diff"></a>
+Use [`parquet2json`] to observe schema changes to a Parquet file:
+```bash
+parquet_schema() {
+    parquet2json "$1" schema
+}
+export -f parquet_schema
+dvc-utils diff -r $commit^..$commit parquet_schema $path
+```
+<details><summary>Output</summary>
+
+```diff
+2d1
+<   OPTIONAL BYTE_ARRAY Year (STRING);
+8,10d6
+<   OPTIONAL BYTE_ARRAY Crash Date (STRING);
+<   OPTIONAL BYTE_ARRAY Crash Day Of Week (STRING);
+<   OPTIONAL BYTE_ARRAY Crash Time (STRING);
+14,17c10,13
+<   OPTIONAL BYTE_ARRAY Total Killed (STRING);
+<   OPTIONAL BYTE_ARRAY Total Injured (STRING);
+<   OPTIONAL BYTE_ARRAY Pedestrians Killed (STRING);
+<   OPTIONAL BYTE_ARRAY Pedestrians Injured (STRING);
+---
+>   OPTIONAL INT64 Total Killed;
+>   OPTIONAL INT64 Total Injured;
+>   OPTIONAL INT64 Pedestrians Killed;
+>   OPTIONAL INT64 Pedestrians Injured;
+20,21c16,17
+<   OPTIONAL BYTE_ARRAY Alcohol Involved (STRING);
+<   OPTIONAL BYTE_ARRAY HazMat Involved (STRING);
+---
+>   OPTIONAL BOOLEAN Alcohol Involved;
+>   OPTIONAL BOOLEAN HazMat Involved;
+23c19
+<   OPTIONAL BYTE_ARRAY Total Vehicles Involved (STRING);
+---
+>   OPTIONAL INT64 Total Vehicles Involved;
+29c25
+<   OPTIONAL BYTE_ARRAY Mile Post (STRING);
+---
+>   OPTIONAL DOUBLE Mile Post;
+47,48c43,44
+<   OPTIONAL BYTE_ARRAY Latitude (STRING);
+<   OPTIONAL BYTE_ARRAY Longitude (STRING);
+---
+>   OPTIONAL DOUBLE Latitude;
+>   OPTIONAL DOUBLE Longitude;
+51a48
+>   OPTIONAL INT64 Date (TIMESTAMP(MICROS,false));
+```
+
+Here we can see that various date/time columns were consolidated, and several stringly-typed columns were converted to ints, floats, and booleans.
+
+</details>
+
+#### Row diff <a id="parquet-row-diff"></a>
+Diff the first row of the Parquet file above (pretty-printed as JSON using [`jq`]), before and after the given commit:
+
+```bash
+pretty_print_first_row() {
+    # Print first row of Parquet file as JSON, pretty-print with jq
+    parquet2json "$1" cat -l 1 | jq .
+}
+export -f pretty_print_first_row
+dvc-utils diff -r $commit^..$commit pretty_print_first_row $path
+```
+
+<details><summary>Output</summary>
+
+```diff
+2d1
+<   "Year": "2001",
+8,10d6
+<   "Crash Date": "12/21/2001",
+<   "Crash Day Of Week": "F",
+<   "Crash Time": "1834",
+14,17c10,13
+<   "Total Killed": "0",
+<   "Total Injured": "0",
+<   "Pedestrians Killed": "0",
+<   "Pedestrians Injured": "0",
+---
+>   "Total Killed": 0,
+>   "Total Injured": 0,
+>   "Pedestrians Killed": 0,
+>   "Pedestrians Injured": 0,
+20,21c16,17
+<   "Alcohol Involved": "N",
+<   "HazMat Involved": "N",
+---
+>   "Alcohol Involved": false,
+>   "HazMat Involved": false,
+23c19
+<   "Total Vehicles Involved": "2",
+---
+>   "Total Vehicles Involved": 2,
+29c25
+<   "Mile Post": "",
+---
+>   "Mile Post": null,
+47,48c43,44
+<   "Latitude": "",
+<   "Longitude": "",
+---
+>   "Latitude": null,
+>   "Longitude": null,
+51c47,48
+<   "Reporting Badge No.": "830"
+---
+>   "Reporting Badge No.": "830",
+>   "Date": "2001-12-21 18:34:00 +00:00"
+```
+
+This reflects the schema changes above.
+
+</details>
+
+#### Row count diff <a id="parquet-row-count-diff"></a>
+```bash
+parquet_row_count() {
+    parquet2json "$1" rowcount
+}
+export -f parquet_row_count
+dvc-utils diff -r $commit^..$commit parquet_row_count $path
+```
+
+This time we get no output; [the given `$commit`][commit] didn't change the row count in the DVC-tracked Parquet file [`$path`][commit path].
+
+[DVC]: https://dvc.org/
+[`parquet2json`]: https://github.com/jupiter/parquet2json
+[hudcostreets/nj-crashes]: https://github.com/hudcostreets/nj-crashes
+[Parquet]: https://parquet.apache.org/
+[commit]: https://github.com/hudcostreets/nj-crashes/commit/c8ae28e64f4917895d84074913f48e0a7afbc3d7
+[commit path]: https://github.com/hudcostreets/nj-crashes/commit/c8ae28e64f4917895d84074913f48e0a7afbc3d7#diff-7f812dce61e0996354f4af414203e0933ccdfe9613cb406c40c1c41a14b9769c
+[hudcostreets/nj-crashes]: https://github.com/hudcostreets/nj-crashes
+[`jq`]: https://jqlang.github.io/jq/

dvc_utils-0.0.3.dist-info/RECORD

@@ -0,0 +1,9 @@
+dvc_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dvc_utils/main.py,sha256=0rpJptELszWdFLoCKoYOY6VQtAm2mp-3_-kN075T-TE,4743
+dvc_utils/named_pipes.py,sha256=VQ2t9BYCazFq_-MABj4t2HS7GHDvSqXXx8fOLz5DsTc,492
+dvc_utils-0.0.3.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
+dvc_utils-0.0.3.dist-info/METADATA,sha256=ExDOJDxXQTUrklQfYm6qTdWNW8Le6xknle4mG2sQFpg,6572
+dvc_utils-0.0.3.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+dvc_utils-0.0.3.dist-info/entry_points.txt,sha256=W9OuZ6CX8QF9ojbqLtfXFo8Q2hnJ-zlcGY4_7nO8paM,49
+dvc_utils-0.0.3.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
+dvc_utils-0.0.3.dist-info/RECORD,,

{dvc_utils-0.0.1.dist-info → dvc_utils-0.0.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dvc_utils-0.0.1.dist-info/METADATA

@@ -1,8 +0,0 @@
-Metadata-Version: 2.1
-Name: dvc-utils
-Version: 0.0.1
-License-File: LICENSE
-Requires-Dist: click
-Requires-Dist: pyyaml
-Requires-Dist: utz
-

dvc_utils-0.0.1.dist-info/RECORD

@@ -1,9 +0,0 @@
-dvc_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dvc_utils/main.py,sha256=_D_FVRVnBGRHb6XIeTLOOFKc0v5J9D8_O6thG8_lSmU,3863
-dvc_utils/named_pipes.py,sha256=GqWvsvTMmnkjk0gPM1aXBIW5dUsSkW-eblerHJ18B68,485
-dvc_utils-0.0.1.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
-dvc_utils-0.0.1.dist-info/METADATA,sha256=YRQeC1fJaUdBHluoCt6BXsEM5LozJesFVqRnILU9Nhs,138
-dvc_utils-0.0.1.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-dvc_utils-0.0.1.dist-info/entry_points.txt,sha256=W9OuZ6CX8QF9ojbqLtfXFo8Q2hnJ-zlcGY4_7nO8paM,49
-dvc_utils-0.0.1.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
-dvc_utils-0.0.1.dist-info/RECORD,,