dvc-utils 0.0.8__py3-none-any.whl → 0.1.0__py3-none-any.whl

dvc_utils/__init__.py

@@ -0,0 +1,2 @@
+from . import cli, path
+from .path import dvc_cache_dir, dvc_md5, dvc_paths, dvc_path

dvc_utils/{main.py → cli.py}

@@ -1,13 +1,13 @@
-from functools import cache
-from os import environ as env, getcwd
-from os.path import join, relpath
 import shlex
-from typing import Optional, Tuple
+from os import environ as env
+from typing import Tuple
 
-from click import option, argument, group
 import click
-import yaml
-from utz import diff_cmds, process, err, singleton
+from click import option, argument, group
+from utz import process, err
+from qmdx import join_pipelines
+
+from dvc_utils.path import dvc_paths, dvc_path as dvc_cache_path
 
 
 @group()
@@ -15,57 +15,6 @@ def cli():
     pass
 
 
-def dvc_paths(path: str) -> Tuple[str, str]:
-    if path.endswith('.dvc'):
-        dvc_path = path
-        path = dvc_path[:-len('.dvc')]
-    else:
-        dvc_path = f'{path}.dvc'
-    return path, dvc_path
-
-
-@cache
-def get_git_root() -> str:
-    return process.line('git', 'rev-parse', '--show-toplevel', log=False)
-
-
-@cache
-def get_dir_path() -> str:
-    return relpath(getcwd(), get_git_root())
-
-
-@cache
-def dvc_cache_dir(log: bool = False) -> str:
-    dvc_cache_relpath = env.get('DVC_UTILS_CACHE_DIR')
-    if dvc_cache_relpath:
-        return join(get_git_root(), dvc_cache_relpath)
-    else:
-        return process.line('dvc', 'cache', 'dir', log=log)
-
-
-def dvc_md5(git_ref: str, dvc_path: str, log: bool = False) -> str:
-    dir_path = get_dir_path()
-    dir_path = '' if dir_path == '.' else f'{dir_path}/'
-    dvc_spec = process.output('git', 'show', f'{git_ref}:{dir_path}{dvc_path}', log=err if log else None)
-    dvc_obj = yaml.safe_load(dvc_spec)
-    out = singleton(dvc_obj['outs'], dedupe=False)
-    md5 = out['md5']
-    return md5
-
-
-def dvc_cache_path(ref: str, dvc_path: Optional[str] = None, log: bool = False) -> str:
-    if dvc_path:
-        md5 = dvc_md5(ref, dvc_path, log=log)
-    elif ':' in ref:
-        git_ref, dvc_path = ref.split(':', 1)
-        md5 = dvc_md5(git_ref, dvc_path, log=log)
-    else:
-        md5 = ref
-    dirname = md5[:2]
-    basename = md5[2:]
-    return join(dvc_cache_dir(log=log), 'files', 'md5', dirname, basename)
-
-
 @cli.command('diff', short_help='Diff a DVC-tracked file at two commits (or one commit vs. current worktree), optionally passing both through another command first')
 @option('-c', '--color', is_flag=True, help='Colorize the output')
 @option('-r', '--refspec', default='HEAD', help='<commit 1>..<commit 2> (compare two commits) or <commit> (compare <commit> to the worktree)')
@@ -114,39 +63,32 @@ def dvc_utils_diff(
         raise ValueError(f"Invalid refspec: {refspec}")
 
     log = err if verbose else False
-    before_path = dvc_cache_path(before, dvc_path, log=log)
-    after_path = path if after is None else dvc_cache_path(after, dvc_path, log=log)
-
+    path1 = dvc_cache_path(before, dvc_path, log=log)
+    path2 = path if after is None else dvc_cache_path(after, dvc_path, log=log)
+
+    diff_args = [
+        *(['-w'] if ignore_whitespace else []),
+        *(['-U', str(unified)] if unified is not None else []),
+        *(['--color=always'] if color else []),
+    ]
     if cmds:
         cmd, *sub_cmds = cmds
+        cmds1 = [ f'{cmd} {path1}', *sub_cmds ]
+        cmds2 = [ f'{cmd} {path2}', *sub_cmds ]
         if not shell:
-            sub_cmds = [ shlex.split(c) for c in sub_cmds ]
-            before_cmds = [
-                shlex.split(f'{cmd} {before_path}'),
-                *sub_cmds,
-            ]
-            after_cmds = [
-                shlex.split(f'{cmd} {after_path}'),
-                *sub_cmds,
-            ]
-            shell_kwargs = {}
-        else:
-            before_cmds = [ f'{cmd} {before_path}', *sub_cmds ]
-            after_cmds = [ f'{cmd} {after_path}', *sub_cmds ]
-            shell_kwargs = dict(shell=shell)
-
-        diff_cmds(
-            before_cmds,
-            after_cmds,
+            cmds1 = [ shlex.split(cmd) for cmd in cmds1 ]
+            cmds2 = [ shlex.split(cmd) for cmd in cmds2 ]
+
+        join_pipelines(
+            base_cmd=['diff', *diff_args],
+            cmds1=cmds1,
+            cmds2=cmds2,
             verbose=verbose,
-            color=color,
-            unified=unified,
-            ignore_whitespace=ignore_whitespace,
+            shell=shell,
             shell_executable=shell_executable,
-            **shell_kwargs,
         )
     else:
-        process.run('diff', before_path, after_path, log=log)
+        process.run('diff', *diff_args, path1, path2, log=log)
 
 
 if __name__ == '__main__':

dvc_utils/path.py

@@ -0,0 +1,60 @@
+from functools import cache
+from os import environ as env, getcwd
+from os.path import join, relpath
+from typing import Optional, Tuple
+
+import yaml
+from utz import process, err, singleton
+
+
+def dvc_paths(path: str) -> Tuple[str, str]:
+    if path.endswith('.dvc'):
+        dvc_path = path
+        path = dvc_path[:-len('.dvc')]
+    else:
+        dvc_path = f'{path}.dvc'
+    return path, dvc_path
+
+
+@cache
+def get_git_root() -> str:
+    return process.line('git', 'rev-parse', '--show-toplevel', log=False)
+
+
+@cache
+def get_dir_path() -> str:
+    return relpath(getcwd(), get_git_root())
+
+
+@cache
+def dvc_cache_dir(log: bool = False) -> str:
+    dvc_cache_relpath = env.get('DVC_UTILS_CACHE_DIR')
+    if dvc_cache_relpath:
+        return join(get_git_root(), dvc_cache_relpath)
+    else:
+        return process.line('dvc', 'cache', 'dir', log=log)
+
+
+def dvc_md5(git_ref: str, dvc_path: str, log: bool = False) -> str:
+    dir_path = get_dir_path()
+    dir_path = '' if dir_path == '.' else f'{dir_path}/'
+    dvc_spec = process.output('git', 'show', f'{git_ref}:{dir_path}{dvc_path}', log=err if log else None)
+    dvc_obj = yaml.safe_load(dvc_spec)
+    out = singleton(dvc_obj['outs'], dedupe=False)
+    md5 = out['md5']
+    return md5
+
+
+def dvc_path(ref: str, dvc_path: Optional[str] = None, log: bool = False) -> str:
+    if dvc_path and not dvc_path.endswith('.dvc'):
+        dvc_path += '.dvc'
+    if dvc_path:
+        md5 = dvc_md5(ref, dvc_path, log=log)
+    elif ':' in ref:
+        git_ref, dvc_path = ref.split(':', 1)
+        md5 = dvc_md5(git_ref, dvc_path, log=log)
+    else:
+        md5 = ref
+    dirname = md5[:2]
+    basename = md5[2:]
+    return join(dvc_cache_dir(log=log), 'files', 'md5', dirname, basename)

{dvc_utils-0.0.8.dist-info → dvc_utils-0.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dvc-utils
-Version: 0.0.8
+Version: 0.1.0
 Summary: CLI for diffing DVC files at two commits (or one commit vs. current worktree), optionally passing both through another command first
 Home-page: https://github.com/runsascoded/dvc-utils
 Author: Ryan Williams
@@ -8,19 +8,26 @@ Author-email: ryan@runsascoded.com
 License: MIT
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: click
+Requires-Dist: pyyaml
+Requires-Dist: qmdx
+Requires-Dist: utz>=0.11.3
 
 # dvc-utils
-CLI for diffing [DVC] files, optionally passing both through another command first
+Diff [DVC] files, optionally piping through other commands first.
+
+[![dvc-utils on PyPI](https://img.shields.io/pypi/v/dvc-utils?label=dvc-utils)][PyPI]
 
 <!-- toc -->
 - [Installation](#installation)
 - [Usage](#usage)
     - [`dvc-diff`](#dvc-diff)
 - [Examples](#examples)
-    - [Parquet file](#parquet-diff)
+    - [Parquet](#parquet-diff)
         - [Schema diff](#parquet-schema-diff)
         - [Row diff](#parquet-row-diff)
         - [Row count diff](#parquet-row-count-diff)
+    - [GZipped CSVs](#csv-gz)
 <!-- /toc -->
 
 ## Installation <a id="installation"></a>
@@ -83,7 +90,7 @@ dvc-diff --help
 
 ## Examples <a id="examples"></a>
 
-### Parquet file <a id="parquet-diff"></a>
+### Parquet <a id="parquet-diff"></a>
 See sample commands and output below for inspecting changes to [a DVC-tracked Parquet file][commit path] in [a given commit][commit].
 
 Setup:
@@ -222,11 +229,112 @@ dvc-diff -r $commit^..$commit parquet_row_count $path
 
 This time we get no output; [the given `$commit`][commit] didn't change the row count in the DVC-tracked Parquet file [`$path`][commit path].
 
+### GZipped CSVs <a id="csv-gz"></a>
+
+Here's a "one-liner" I used in [ctbk.dev][ctbk.dev gh], to normalize and compare headers of `.csv.gz.dvc` files between two commits:
+
+```bash
+# Save some `sed` substitution commands to file `seds`:
+cat <<EOF >seds
+s/station_//
+s/latitude/lat/
+s/longitude/lng/
+s/starttime/started_at/
+s/stoptime/ended_at/
+s/usertype/member_casual/
+EOF
+# Commit range to diff; branch `c0` is an initial commit of some `.csv.gz` files, branch `c1` is a later commit after some updates
+r=c0..c1
+# List files changed in commit range `$r`, in the `s3/ctbk/csvs/` dir, piping through several post-processing commands:
+gdno $r s3/ctbk/csvs/ | \
+pel "ddcr $r guc h1 spc kq kcr snc 'sdf seds' sort"
+```
+
+<details>
+<summary>
+Explanation of aliases
+</summary>
+
+- [`gdno`] (`git diff --name-only`): list files changed in the given commit range and directory
+- [`pel`]: [`parallel`] alias that prepends an `echo {}` to the command
+- [`ddcr`] (`dvc-diff -cr`): colorized `diff` output, revision range `$r`
+- [`guc`] (`gunzip -c`): uncompress the `.csv.gz` files
+- [`h1`] (`head -n1`): only examine each file's header line
+- [`spc`] (`tr , $'\n'`): **sp**lit the header line by **c**ommas (so each column name will be on one line, for easier `diff`ing below)
+- [`kq`] (`tr -d '"'`): **k**ill **q**uote characters (in this case, header-column name quoting changed, but I don't care about that)
+- [`kcr`] (`tr -d '\r'`): **k**ill **c**arriage **r**eturns (line endings also changed)
+- [`snc`] (`sed -f 'snake_case.sed'`): snake-case column names
+- [`sdf`] (`sed -f`): execute the `sed` substitution commands defined in the `seds` file above
+- `sort`: sort the column names alphabetically (to identify missing or added columns, ignore rearrangements)
+
+Note:
+- Most of these are exported Bash functions, allowing them to be used inside the [`parallel`] command.
+- I was able to build this pipeline iteratively, adding steps to normalize out the bits I didn't care about (and accumulating the `seds` commands).
+</details>
+
+Example output:
+```diff
+…
+s3/ctbk/csvs/201910-citibike-tripdata.csv.gz.dvc:
+s3/ctbk/csvs/201911-citibike-tripdata.csv.gz.dvc:
+s3/ctbk/csvs/201912-citibike-tripdata.csv.gz.dvc:
+s3/ctbk/csvs/202001-citibike-tripdata.csv.gz.dvc:
+1,2d0
+< bikeid
+< birth_year
+8d5
+< gender
+9a7,8
+> ride_id
+> rideable_type
+15d13
+< tripduration
+s3/ctbk/csvs/202002-citibike-tripdata.csv.gz.dvc:
+1,2d0
+< bikeid
+< birth_year
+8d5
+< gender
+9a7,8
+> ride_id
+> rideable_type
+15d13
+< tripduration
+s3/ctbk/csvs/202003-citibike-tripdata.csv.gz.dvc:
+1,2d0
+< bikeid
+< birth_year
+8d5
+< gender
+9a7,8
+> ride_id
+> rideable_type
+15d13
+< tripduration
+…
+```
+
+This helped me see that the data update in question (`c0..c1`) dropped some fields (`bikeid, birth_year`, `gender`, `tripduration`) and added others (`ride_id`, `rideable_type`), for `202001` and later.
+
 [DVC]: https://dvc.org/
+[PyPI]: https://pypi.org/project/dvc-utils/
 [`parquet2json`]: https://github.com/jupiter/parquet2json
 [hudcostreets/nj-crashes]: https://github.com/hudcostreets/nj-crashes
 [Parquet]: https://parquet.apache.org/
 [commit]: https://github.com/hudcostreets/nj-crashes/commit/c8ae28e64f4917895d84074913f48e0a7afbc3d7
 [commit path]: https://github.com/hudcostreets/nj-crashes/commit/c8ae28e64f4917895d84074913f48e0a7afbc3d7#diff-7f812dce61e0996354f4af414203e0933ccdfe9613cb406c40c1c41a14b9769c
 [hudcostreets/nj-crashes]: https://github.com/hudcostreets/nj-crashes
+[ctbk.dev gh]: https://github.com/neighbor-ryan/ctbk.dev
 [`jq`]: https://jqlang.github.io/jq/
+[`parallel`]: https://www.gnu.org/software/parallel/
+
+[`gdno`]: https://github.com/ryan-williams/git-helpers/blob/96560df1406f41676f293becefb423895a755faf/diff/.gitconfig#L31
+[`pel`]: https://github.com/ryan-williams/parallel-helpers/blob/e7ee109c4085c04036840ea78999cff73fcf9502/.parallel-rc#L6-L17
+[`ddcr`]: https://github.com/ryan-williams/aws-helpers/blob/8a314f1e6b336833c772459de6b739f5c06a51a3/.dvc-rc#L84
+[`guc`]: https://github.com/ryan-williams/zip-helpers/blob/c67d84fb06c0ab3609dacb68d900344d3b8e8f04/.zip-rc#L16
+[`h1`]: https://github.com/ryan-williams/head-tail-helpers/blob/9715690f47ceeff6b6948b2093901f2b0830114b/.head-tail-rc#L3
+[`spc`]: https://github.com/ryan-williams/col-helpers/blob/9493d003224249ee240d023f71ab03bdd4174b88/.cols-rc#L8
+[`kq`]: https://github.com/ryan-williams/arg-helpers/blob/a8c60809f8878fa38b3c03614778fcf29132538e/.arg-rc#L115
+[`kcr`]: https://github.com/ryan-williams/arg-helpers/blob/a8c60809f8878fa38b3c03614778fcf29132538e/.arg-rc#L118
+[`snc`]: https://github.com/ryan-williams/case-helpers/blob/c40a62a9656f0d52d68fb3a108ae6bb3eed3c7bd/.case-rc#L9
+[`sdf`]: https://github.com/ryan-williams/arg-helpers/blob/a8c60809f8878fa38b3c03614778fcf29132538e/.arg-rc#L138

dvc_utils-0.1.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+dvc_utils/__init__.py,sha256=mP-p1Sl2JMMShM_hRhu86pFNfIq_8E_feh1CN47LWcs,86
+dvc_utils/cli.py,sha256=CcOa5Qmkry3PASz8nu_XqcBB6GnGmcf0e8zsuqSDsCM,3534
+dvc_utils/path.py,sha256=PoAbeaqRPDksY2hcUeF8xZ6Nr6hLIZprey3VNT4V5bc,1727
+dvc_utils-0.1.0.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
+dvc_utils-0.1.0.dist-info/METADATA,sha256=F3GUp8NMg0oEqsqtI_jECiWkoO7-sGYb8KJT4upGqNM,11722
+dvc_utils-0.1.0.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
+dvc_utils-0.1.0.dist-info/entry_points.txt,sha256=luxCQr8OS-jMSyyDhB9KDQhUbP8UH6UMcy-vkfXX7Gg,88
+dvc_utils-0.1.0.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
+dvc_utils-0.1.0.dist-info/RECORD,,

dvc_utils-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dvc-diff = dvc_utils.cli:dvc_utils_diff
+dvc-utils = dvc_utils.cli:cli

dvc_utils-0.0.8.dist-info/RECORD

@@ -1,8 +0,0 @@
-dvc_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-dvc_utils/main.py,sha256=05Cv6jlKfNnmRQjXOs9dq1eBzT2gIdwnayOsiEjmVZE,5205
-dvc_utils-0.0.8.dist-info/LICENSE,sha256=ZS8AReay7xmQzBAHwxIuTouGXz3SKgUa2_Sz8Ip0EzQ,1070
-dvc_utils-0.0.8.dist-info/METADATA,sha256=EplLe4Eaqnfu1LGalvWaVOvoKdUVpTjZjxpljp4YtmU,7384
-dvc_utils-0.0.8.dist-info/WHEEL,sha256=bFJAMchF8aTQGUgMZzHJyDDMPTO3ToJ7x23SLJa1SVo,92
-dvc_utils-0.0.8.dist-info/entry_points.txt,sha256=0JqFeb29N5ZrvwmIJrg4ii32xNFOXRul-HMq-3GOIw4,90
-dvc_utils-0.0.8.dist-info/top_level.txt,sha256=jT0-PJa2t_eFRE9rn-52AjdnZ8nQeEHllf2kJmaGh80,10
-dvc_utils-0.0.8.dist-info/RECORD,,

dvc_utils-0.0.8.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-dvc-diff = dvc_utils.main:dvc_utils_diff
-dvc-utils = dvc_utils.main:cli