calkit-python 0.37.1__tar.gz → 0.37.2__tar.gz

{calkit_python-0.37.1 → calkit_python-0.37.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.37.1
+Version: 0.37.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.37.1 → calkit_python-0.37.2}/calkit/dvc/zip.py

@@ -25,6 +25,7 @@ from tqdm import tqdm
 import calkit
 import calkit.git
 from calkit.core import DVC_SIZE_THRESH_BYTES
+from calkit.dvc.core import run_dvc_command
 
 LOCAL_DIR = ".calkit/local"
 ZIPS_DIR = ".calkit/zip"
@@ -38,6 +39,12 @@ ZIP_CANDIDATE_AVG_FILE_SIZE_BYTES = 10_000_000  # 10 MB
 # Minimum number of files a directory must contain to be a zip candidate;
 # a single large file is better tracked directly by DVC
 ZIP_CANDIDATE_MIN_FILE_COUNT = 10
+# Favor speed for dvc-zip by default; users can tune 0..9 via env var
+# CALKIT_DVC_ZIP_COMPRESS_LEVEL
+ZIP_COMPRESS_LEVEL = 1
+# Use Python zipfile by default; set CALKIT_DVC_ZIP_USE_SYSTEM=1 to try
+# the system zip/unzip tools instead (may be faster for very large files)
+ZIP_USE_SYSTEM_CLI = False
 
 
 def _check_local_dir() -> Path:
@@ -285,16 +292,59 @@ def get_zip_path(workspace_path: str) -> str:
     raise ValueError(f"No zip path defined for {workspace_path}")
 
 
+def _get_zip_compress_level() -> int:
+    raw = os.getenv("CALKIT_DVC_ZIP_COMPRESS_LEVEL", str(ZIP_COMPRESS_LEVEL))
+    try:
+        level = int(raw)
+    except ValueError:
+        typer.echo(
+            "Invalid CALKIT_DVC_ZIP_COMPRESS_LEVEL value; "
+            f"using default {ZIP_COMPRESS_LEVEL}.",
+            err=True,
+        )
+        return ZIP_COMPRESS_LEVEL
+    return min(max(level, 0), 9)
+
+
+def _should_use_system_zip_cli() -> bool:
+    raw = os.getenv("CALKIT_DVC_ZIP_USE_SYSTEM")
+    if raw is None:
+        return ZIP_USE_SYSTEM_CLI
+    return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def _iter_files(path: str):
+    for foldername, _, filenames in os.walk(path):
+        for filename in filenames:
+            yield os.path.join(foldername, filename)
+
+
 def zip_(workspace_path: str, zip_path: str):
     """Zip a path."""
+    zip_path = os.path.abspath(zip_path)
     output_dir = os.path.dirname(zip_path)
     os.makedirs(output_dir, exist_ok=True)
-    all_files = [
-        os.path.join(foldername, filename)
-        for foldername, _, filenames in os.walk(workspace_path)
-        for filename in filenames
-    ]
-    with ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zip_file:
+    compress_level = _get_zip_compress_level()
+    if _should_use_system_zip_cli() and shutil.which("zip"):
+        try:
+            subprocess.run(
+                ["zip", "-q", f"-{compress_level}", "-r", zip_path, "."],
+                check=True,
+                cwd=workspace_path,
+            )
+            return
+        except Exception:
+            typer.echo(
+                "System zip failed; falling back to Python zipfile.",
+                err=True,
+            )
+    all_files = list(_iter_files(workspace_path))
+    with ZipFile(
+        zip_path,
+        "w",
+        compression=zipfile.ZIP_DEFLATED,
+        compresslevel=compress_level,
+    ) as zip_file:
         for file_path in tqdm(all_files, desc="Zipping", unit="file"):
             zip_file.write(
                 file_path, os.path.relpath(file_path, workspace_path)
@@ -303,11 +353,25 @@ def zip_(workspace_path: str, zip_path: str):
 
 def unzip(workspace_path: str, zip_path: str):
     """Unzip from zip to workspace."""
+    zip_path = os.path.abspath(zip_path)
     input_dir = os.path.dirname(workspace_path)
     if input_dir:
         os.makedirs(input_dir, exist_ok=True)
+    if _should_use_system_zip_cli() and shutil.which("unzip"):
+        try:
+            os.makedirs(workspace_path, exist_ok=True)
+            subprocess.run(
+                ["unzip", "-oq", zip_path, "-d", workspace_path],
+                check=True,
+            )
+            return
+        except Exception:
+            typer.echo(
+                "System unzip failed; falling back to Python zipfile.",
+                err=True,
+            )
     with ZipFile(zip_path, "r") as zip_file:
-        members = zip_file.namelist()
+        members = zip_file.infolist()
         for member in tqdm(members, desc="Unzipping", unit="file"):
             zip_file.extract(member, workspace_path)
 
@@ -407,7 +471,7 @@ def sync_one(
     if zip_deleted and direction == "to-zip":
         typer.echo(f"Rezipping '{workspace_path}' (zip was deleted)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        subprocess.run(["dvc", "add", zip_path], check=True)
+        run_dvc_command(["add", zip_path])
         zip_hash = get_hash(zip_path)
     # Workspace was deleted but zip exists and direction is to-workspace:
     # restore workspace
@@ -429,7 +493,7 @@ def sync_one(
     if workspace_changed and (direction in ["to-zip", "both"]):
         typer.echo(f"Zipping '{workspace_path}' (workspace has changed)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        subprocess.run(["dvc", "add", zip_path], check=True)
+        run_dvc_command(["add", zip_path])
         zip_hash = get_hash(zip_path)
     # If we unzip, we need to update the hash
     if zip_changed and (direction in ["to-workspace", "both"]):

{calkit_python-0.37.1 → calkit_python-0.37.2}/calkit/tests/dvc/test_zip.py

@@ -200,7 +200,14 @@ def test_get_sync_status(tmp_dir):
 
 
 def test_sync_one(tmp_dir, monkeypatch):
-    monkeypatch.setattr("calkit.dvc.zip.subprocess.run", lambda *_, **__: None)
+    real_run = calkit.dvc.zip.subprocess.run
+
+    def _run(cmd, *args, **kwargs):
+        if cmd and cmd[0] == "dvc":
+            return None
+        return real_run(cmd, *args, **kwargs)
+
+    monkeypatch.setattr("calkit.dvc.zip.subprocess.run", _run)
     # to-zip: zips input and writes sync record
     src = tmp_dir / "src"
     src.mkdir()