calkit-python 0.37.1__py3-none-any.whl → 0.37.2__py3-none-any.whl

calkit/dvc/zip.py

@@ -25,6 +25,7 @@ from tqdm import tqdm
 import calkit
 import calkit.git
 from calkit.core import DVC_SIZE_THRESH_BYTES
+from calkit.dvc.core import run_dvc_command
 
 LOCAL_DIR = ".calkit/local"
 ZIPS_DIR = ".calkit/zip"
@@ -38,6 +39,12 @@ ZIP_CANDIDATE_AVG_FILE_SIZE_BYTES = 10_000_000  # 10 MB
 # Minimum number of files a directory must contain to be a zip candidate;
 # a single large file is better tracked directly by DVC
 ZIP_CANDIDATE_MIN_FILE_COUNT = 10
+# Favor speed for dvc-zip by default; users can tune 0..9 via env var
+# CALKIT_DVC_ZIP_COMPRESS_LEVEL
+ZIP_COMPRESS_LEVEL = 1
+# Use Python zipfile by default; set CALKIT_DVC_ZIP_USE_SYSTEM=1 to try
+# the system zip/unzip tools instead (may be faster for very large files)
+ZIP_USE_SYSTEM_CLI = False
 
 
 def _check_local_dir() -> Path:
@@ -285,16 +292,59 @@ def get_zip_path(workspace_path: str) -> str:
     raise ValueError(f"No zip path defined for {workspace_path}")
 
 
+def _get_zip_compress_level() -> int:
+    raw = os.getenv("CALKIT_DVC_ZIP_COMPRESS_LEVEL", str(ZIP_COMPRESS_LEVEL))
+    try:
+        level = int(raw)
+    except ValueError:
+        typer.echo(
+            "Invalid CALKIT_DVC_ZIP_COMPRESS_LEVEL value; "
+            f"using default {ZIP_COMPRESS_LEVEL}.",
+            err=True,
+        )
+        return ZIP_COMPRESS_LEVEL
+    return min(max(level, 0), 9)
+
+
+def _should_use_system_zip_cli() -> bool:
+    raw = os.getenv("CALKIT_DVC_ZIP_USE_SYSTEM")
+    if raw is None:
+        return ZIP_USE_SYSTEM_CLI
+    return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def _iter_files(path: str):
+    for foldername, _, filenames in os.walk(path):
+        for filename in filenames:
+            yield os.path.join(foldername, filename)
+
+
 def zip_(workspace_path: str, zip_path: str):
     """Zip a path."""
+    zip_path = os.path.abspath(zip_path)
     output_dir = os.path.dirname(zip_path)
     os.makedirs(output_dir, exist_ok=True)
-    all_files = [
-        os.path.join(foldername, filename)
-        for foldername, _, filenames in os.walk(workspace_path)
-        for filename in filenames
-    ]
-    with ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zip_file:
+    compress_level = _get_zip_compress_level()
+    if _should_use_system_zip_cli() and shutil.which("zip"):
+        try:
+            subprocess.run(
+                ["zip", "-q", f"-{compress_level}", "-r", zip_path, "."],
+                check=True,
+                cwd=workspace_path,
+            )
+            return
+        except Exception:
+            typer.echo(
+                "System zip failed; falling back to Python zipfile.",
+                err=True,
+            )
+    all_files = list(_iter_files(workspace_path))
+    with ZipFile(
+        zip_path,
+        "w",
+        compression=zipfile.ZIP_DEFLATED,
+        compresslevel=compress_level,
+    ) as zip_file:
         for file_path in tqdm(all_files, desc="Zipping", unit="file"):
             zip_file.write(
                 file_path, os.path.relpath(file_path, workspace_path)
@@ -303,11 +353,25 @@ def zip_(workspace_path: str, zip_path: str):
 
 def unzip(workspace_path: str, zip_path: str):
     """Unzip from zip to workspace."""
+    zip_path = os.path.abspath(zip_path)
     input_dir = os.path.dirname(workspace_path)
     if input_dir:
         os.makedirs(input_dir, exist_ok=True)
+    if _should_use_system_zip_cli() and shutil.which("unzip"):
+        try:
+            os.makedirs(workspace_path, exist_ok=True)
+            subprocess.run(
+                ["unzip", "-oq", zip_path, "-d", workspace_path],
+                check=True,
+            )
+            return
+        except Exception:
+            typer.echo(
+                "System unzip failed; falling back to Python zipfile.",
+                err=True,
+            )
     with ZipFile(zip_path, "r") as zip_file:
-        members = zip_file.namelist()
+        members = zip_file.infolist()
         for member in tqdm(members, desc="Unzipping", unit="file"):
             zip_file.extract(member, workspace_path)
 
@@ -407,7 +471,7 @@ def sync_one(
     if zip_deleted and direction == "to-zip":
         typer.echo(f"Rezipping '{workspace_path}' (zip was deleted)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        subprocess.run(["dvc", "add", zip_path], check=True)
+        run_dvc_command(["add", zip_path])
         zip_hash = get_hash(zip_path)
     # Workspace was deleted but zip exists and direction is to-workspace:
     # restore workspace
@@ -429,7 +493,7 @@ def sync_one(
     if workspace_changed and (direction in ["to-zip", "both"]):
         typer.echo(f"Zipping '{workspace_path}' (workspace has changed)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        subprocess.run(["dvc", "add", zip_path], check=True)
+        run_dvc_command(["add", zip_path])
         zip_hash = get_hash(zip_path)
     # If we unzip, we need to update the hash
     if zip_changed and (direction in ["to-workspace", "both"]):

calkit/tests/dvc/test_zip.py

@@ -200,7 +200,14 @@ def test_get_sync_status(tmp_dir):
 
 
 def test_sync_one(tmp_dir, monkeypatch):
-    monkeypatch.setattr("calkit.dvc.zip.subprocess.run", lambda *_, **__: None)
+    real_run = calkit.dvc.zip.subprocess.run
+
+    def _run(cmd, *args, **kwargs):
+        if cmd and cmd[0] == "dvc":
+            return None
+        return real_run(cmd, *args, **kwargs)
+
+    monkeypatch.setattr("calkit.dvc.zip.subprocess.run", _run)
     # to-zip: zips input and writes sync record
     src = tmp_dir / "src"
     src.mkdir()

{calkit_python-0.37.1.dist-info → calkit_python-0.37.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.37.1
+Version: 0.37.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.37.1.dist-info → calkit_python-0.37.2.dist-info}/RECORD

@@ -47,7 +47,7 @@ calkit/cli/main/core.py,sha256=Ds9bhOww5p6vB85zKy_KvS_sS2g9fTyF_03KJH4Fb0o,92127
 calkit/cli/main/xr.py,sha256=-hiIS6Xr3RiVAFGgH19yqvyzbcAwnEyaVBUH-qDehxY,25575
 calkit/dvc/__init__.py,sha256=g_K9o7zVXS0d_6in8UqhPpkPfxXAjGGmjLVLl7k6Nd0,34
 calkit/dvc/core.py,sha256=relc7smVSQLVxpWjt_aCCbHZiWF8ZFD2nKckOjk96eE,18274
-calkit/dvc/zip.py,sha256=lt7kITGfeBs1Bw086V-GywTLgRtrHGXdp1krbU6J3D4,16514
+calkit/dvc/zip.py,sha256=Rcf7oEbkuazGsPcvKVhdiRrqNMqRIIp9rmnOPidEhKU,18572
 calkit/jupyterlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/jupyterlab/routes.py,sha256=vipSlBXVHIzdnZryNhZBpeTWnqcw2wGiOJYwO7Lxd_o,51010
 calkit/models/__init__.py,sha256=vGYF4MZBGzDk1BPBzpp1bRb59RCJ2Bj2kzHuVVRacug,120
@@ -99,29 +99,29 @@ calkit/tests/cli/main/test_core.py,sha256=ZRWujJmI2Dtr5FfQxUSXnNfpa-SagkV90y1dHU
 calkit/tests/cli/main/test_xr.py,sha256=UhtzJEjRiHgE_p_tdjGyYS7I9yy_Ubs2W2lSmLPpBLU,22386
 calkit/tests/dvc/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/tests/dvc/test_core.py,sha256=urN4FIPMVgYfVJxqzH1d7x0hpMI3h2KeArnslfFRDYI,4616
-calkit/tests/dvc/test_zip.py,sha256=HClb6kGg0MFnBrTHeMQsUje_syR9CgYT7YCtYK-Rw9w,11040
+calkit/tests/dvc/test_zip.py,sha256=H6L3uJdUpsaNQcOPKnmRxMb5Imr_MaDQFoAnMrZMErU,11212
 calkit/tests/jupyterlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/tests/jupyterlab/test_routes.py,sha256=TEEyoHiKnzzKOkQsJcMPKdqpKFQ25yBCvVSquqe9nsE,408
 calkit/tests/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 calkit/tests/models/test_iteration.py,sha256=vjuDKwyYQAV4w1qF9VJWQEHksq3KRd6gtT39_a-RhnI,644
 calkit/tests/models/test_pipeline.py,sha256=yefZIG0oHO3xBSwEc3BLx6JOuqKr8GwZvlx1eMcr6SI,9429
-calkit_python-0.37.1.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/package.json,sha256=HWflZoLFLgne6BqHLI_cZTBzro050PoPTabnCp5NRQc,6651
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=6w0b5y8LztEv2tErajJ5d39yBryld02crX0CG3zZFNs,6509
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/909.651be47ca47390b78a92.js,sha256=ZRvkfKRzkLeKkhSUg3qX-df9A80XTd0F0CPcXMfmb-Q,114572
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.c091821b3d7f2d287a67.js,sha256=wJGCGz1_LSh6Z9lt5VFlfXs92iKQNlLUJshrAQl4ziw,8737
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
-calkit_python-0.37.1.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
-calkit_python-0.37.1.dist-info/METADATA,sha256=5h2ujrZJV8kjw7XsrbZKLytWb95sZQv_9hntwAFJj_s,8777
-calkit_python-0.37.1.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
-calkit_python-0.37.1.dist-info/entry_points.txt,sha256=59JWYjwz2N3yBOk_9vYUzsMsSs-It1LW8gYE8NsTUJ4,113
-calkit_python-0.37.1.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
-calkit_python-0.37.1.dist-info/RECORD,,
+calkit_python-0.37.2.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/package.json,sha256=HWflZoLFLgne6BqHLI_cZTBzro050PoPTabnCp5NRQc,6651
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=6w0b5y8LztEv2tErajJ5d39yBryld02crX0CG3zZFNs,6509
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/909.651be47ca47390b78a92.js,sha256=ZRvkfKRzkLeKkhSUg3qX-df9A80XTd0F0CPcXMfmb-Q,114572
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.c091821b3d7f2d287a67.js,sha256=wJGCGz1_LSh6Z9lt5VFlfXs92iKQNlLUJshrAQl4ziw,8737
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
+calkit_python-0.37.2.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
+calkit_python-0.37.2.dist-info/METADATA,sha256=1w6BYdTyAKHWTh-mOZ6_3UFXRHlg9cE_f2Iof4qWQng,8777
+calkit_python-0.37.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+calkit_python-0.37.2.dist-info/entry_points.txt,sha256=59JWYjwz2N3yBOk_9vYUzsMsSs-It1LW8gYE8NsTUJ4,113
+calkit_python-0.37.2.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
+calkit_python-0.37.2.dist-info/RECORD,,