calkit-python 0.37.0__tar.gz → 0.37.2__tar.gz

{calkit_python-0.37.0 → calkit_python-0.37.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.37.0
+Version: 0.37.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/cli/config.py

@@ -70,6 +70,13 @@ def unset_config_value(key: str):
 @config_app.command(name="setup-remote", help="Alias for 'remote'.")
 @config_app.command(name="remote")
 def setup_remote(
+    ck: Annotated[
+        bool,
+        typer.Option(
+            "--ck",
+            help="Use a ck:// URL for the 'calkit' DVC remote.",
+        ),
+    ] = False,
     no_commit: Annotated[
         bool,
         typer.Option(
@@ -81,7 +88,7 @@ def setup_remote(
     the local config.
     """
     try:
-        configure_remote()
+        configure_remote(use_ck=ck)
         set_remote_auth()
     except subprocess.CalledProcessError:
         if not os.path.isfile(".dvc/config"):

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/cli/main/core.py

@@ -347,8 +347,6 @@ def get_status(
             clean_notebooks=True,
             compile_to_dvc=True,
         )
-        for error in pipeline_status.errors:
-            warn(error)
         if pipeline_status.failed_environment_checks:
             warn(
                 "Failed pipeline environment checks for: "
@@ -469,7 +467,9 @@ def get_status(
         print_sep("Pipeline")
         # Nicely format the results from pipeline status
         if pipeline_status and pipeline_status.errors:
-            warn("Pipeline status unavailable due to errors.")
+            warn("Pipeline status unavailable due to errors:")
+            for error in pipeline_status.errors:
+                warn(error)
             return
         if pipeline_status and not pipeline_status.has_pipeline:
             typer.echo("This project has no pipeline.")
@@ -482,7 +482,7 @@ def get_status(
                     continue
                 typer.echo(f"        {typer.style(stage_name, fg='yellow')}:")
                 if stale_stage.modified_command:
-                    typer.echo("            modified command")
+                    typer.echo("          modified command")
                 # Show stale outputs for this stage
                 if stale_stage.stale_outputs:
                     typer.echo("          stale outputs:")

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/cli/new.py

@@ -7,7 +7,6 @@ import os
 import pathlib
 import shutil
 import subprocess
-import zipfile
 from enum import Enum
 
 import dotenv
@@ -2686,8 +2685,7 @@ def new_release(
         zip_path = release_files_dir + "/archive.zip"
         all_paths = calkit.releases.ls_files()
         typer.echo(f"Adding files to {zip_path}")
-        with zipfile.ZipFile(zip_path, "w") as zipf:
-            calkit.releases.add_paths_to_zip(zipf, all_paths)
+        calkit.releases.zip_paths(zip_path, all_paths)
         typer.echo("Checking extracted project release archive")
         try:
             calkit.releases.check_project_release_archive(

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/cli/update.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import json
 import os
-import zipfile
 from datetime import datetime
 
 import git
@@ -266,8 +265,7 @@ def update_release(
         zip_path = release_files_dir + "/archive.zip"
         all_paths = calkit.releases.ls_files()
         typer.echo(f"Adding files to {zip_path}")
-        with zipfile.ZipFile(zip_path, "w") as zipf:
-            calkit.releases.add_paths_to_zip(zipf, all_paths)
+        calkit.releases.zip_paths(zip_path, all_paths)
         typer.echo("Checking project release archive")
         try:
             calkit.releases.check_project_release_archive(zip_path)

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/dvc/core.py

@@ -11,6 +11,7 @@ from typing import Any
 
 import dvc.repo
 import git
+from configobj import ConfigObj
 from dvc.utils.objects import cached_property
 from dvc_objects.fs.base import ObjectFileSystem
 from fsspec import Callback
@@ -251,10 +252,10 @@ def run_dvc_command(argv: list[str], cwd: str | None = None) -> int:
     return run_dvc_cli(argv)
 
 
-def configure_remote(wdir: str | None = None) -> str:
+def configure_remote(wdir: str | None = None, use_ck: bool = False) -> str:
     """Configure a DVC remote for the current project.
 
-    TODO: Use the ck:// scheme.
+    TODO: Use the ck:// scheme by default once it's deemed stable.
     """
     try:
         project_name = calkit.detect_project_name(wdir=wdir)
@@ -275,8 +276,12 @@ def configure_remote(wdir: str | None = None) -> str:
         if not url.endswith(".git"):
             url += ".git"
         repo.git.remote(["add", "origin", url])
-    base_url = calkit.cloud.get_base_url()
-    remote_url = f"{base_url}/projects/{project_name}/dvc"
+    if use_ck:
+        clear_remote_local_http_auth(wdir=wdir)
+        remote_url = f"ck://{project_name}"
+    else:
+        base_url = calkit.cloud.get_base_url()
+        remote_url = f"{base_url}/projects/{project_name}/dvc"
     remote_name = get_app_name()
     result = run_dvc_command(
         ["remote", "add", "-d", "-f", remote_name, remote_url],
@@ -284,17 +289,47 @@ def configure_remote(wdir: str | None = None) -> str:
     )
     if result != 0:
         raise RuntimeError(f"Failed to add DVC remote {remote_name}")
-    result = run_dvc_command(
-        ["remote", "modify", remote_name, "auth", "custom"],
-        cwd=wdir,
-    )
-    if result != 0:
-        raise RuntimeError(
-            f"Failed to configure auth for DVC remote {remote_name}"
+    if not use_ck:
+        result = run_dvc_command(
+            ["remote", "modify", remote_name, "auth", "custom"],
+            cwd=wdir,
         )
+        if result != 0:
+            raise RuntimeError(
+                f"Failed to configure auth for DVC remote {remote_name}"
+            )
     return remote_name
 
 
+def clear_remote_local_http_auth(
+    remote_name: str | None = None, wdir: str | None = None
+) -> None:
+    """Remove HTTP-specific local auth settings for a DVC remote.
+
+    This clears values written to ``.dvc/config.local`` by HTTP auth setup.
+    """
+    if remote_name is None:
+        remote_name = get_app_name()
+    config_local = Path(wdir or ".") / ".dvc" / "config.local"
+    if not config_local.is_file():
+        return
+    cfg = ConfigObj(str(config_local), encoding="utf-8")
+    section_name = f'remote "{remote_name}"'
+    remote = cfg.get(section_name)
+    if not isinstance(remote, dict):
+        return
+    changed = False
+    for option in ("custom_auth_header", "password", "auth"):
+        if option in remote:
+            remote.pop(option)
+            changed = True
+    if not changed:
+        return
+    if not remote:
+        cfg.pop(section_name, None)
+    cfg.write()
+
+
 def set_remote_auth(
     remote_name: str | None = None,
     always_auth: bool = False,
@@ -312,6 +347,7 @@ def set_remote_auth(
     remotes = get_remotes(wdir=wdir)
     remote_url = remotes.get(remote_name, "")
     if remote_url.startswith("ck://"):
+        clear_remote_local_http_auth(remote_name=remote_name, wdir=wdir)
         logger.info(
             f"Remote {remote_name} uses ck:// scheme; skipping HTTP auth setup"
         )

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/dvc/zip.py

@@ -25,6 +25,7 @@ from tqdm import tqdm
 import calkit
 import calkit.git
 from calkit.core import DVC_SIZE_THRESH_BYTES
+from calkit.dvc.core import run_dvc_command
 
 LOCAL_DIR = ".calkit/local"
 ZIPS_DIR = ".calkit/zip"
@@ -38,6 +39,12 @@ ZIP_CANDIDATE_AVG_FILE_SIZE_BYTES = 10_000_000  # 10 MB
 # Minimum number of files a directory must contain to be a zip candidate;
 # a single large file is better tracked directly by DVC
 ZIP_CANDIDATE_MIN_FILE_COUNT = 10
+# Favor speed for dvc-zip by default; users can tune 0..9 via env var
+# CALKIT_DVC_ZIP_COMPRESS_LEVEL
+ZIP_COMPRESS_LEVEL = 1
+# Use Python zipfile by default; set CALKIT_DVC_ZIP_USE_SYSTEM=1 to try
+# the system zip/unzip tools instead (may be faster for very large files)
+ZIP_USE_SYSTEM_CLI = False
 
 
 def _check_local_dir() -> Path:
@@ -285,16 +292,59 @@ def get_zip_path(workspace_path: str) -> str:
     raise ValueError(f"No zip path defined for {workspace_path}")
 
 
+def _get_zip_compress_level() -> int:
+    raw = os.getenv("CALKIT_DVC_ZIP_COMPRESS_LEVEL", str(ZIP_COMPRESS_LEVEL))
+    try:
+        level = int(raw)
+    except ValueError:
+        typer.echo(
+            "Invalid CALKIT_DVC_ZIP_COMPRESS_LEVEL value; "
+            f"using default {ZIP_COMPRESS_LEVEL}.",
+            err=True,
+        )
+        return ZIP_COMPRESS_LEVEL
+    return min(max(level, 0), 9)
+
+
+def _should_use_system_zip_cli() -> bool:
+    raw = os.getenv("CALKIT_DVC_ZIP_USE_SYSTEM")
+    if raw is None:
+        return ZIP_USE_SYSTEM_CLI
+    return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def _iter_files(path: str):
+    for foldername, _, filenames in os.walk(path):
+        for filename in filenames:
+            yield os.path.join(foldername, filename)
+
+
 def zip_(workspace_path: str, zip_path: str):
     """Zip a path."""
+    zip_path = os.path.abspath(zip_path)
     output_dir = os.path.dirname(zip_path)
     os.makedirs(output_dir, exist_ok=True)
-    all_files = [
-        os.path.join(foldername, filename)
-        for foldername, _, filenames in os.walk(workspace_path)
-        for filename in filenames
-    ]
-    with ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zip_file:
+    compress_level = _get_zip_compress_level()
+    if _should_use_system_zip_cli() and shutil.which("zip"):
+        try:
+            subprocess.run(
+                ["zip", "-q", f"-{compress_level}", "-r", zip_path, "."],
+                check=True,
+                cwd=workspace_path,
+            )
+            return
+        except Exception:
+            typer.echo(
+                "System zip failed; falling back to Python zipfile.",
+                err=True,
+            )
+    all_files = list(_iter_files(workspace_path))
+    with ZipFile(
+        zip_path,
+        "w",
+        compression=zipfile.ZIP_DEFLATED,
+        compresslevel=compress_level,
+    ) as zip_file:
         for file_path in tqdm(all_files, desc="Zipping", unit="file"):
             zip_file.write(
                 file_path, os.path.relpath(file_path, workspace_path)
@@ -303,11 +353,25 @@ def zip_(workspace_path: str, zip_path: str):
 
 def unzip(workspace_path: str, zip_path: str):
     """Unzip from zip to workspace."""
+    zip_path = os.path.abspath(zip_path)
     input_dir = os.path.dirname(workspace_path)
     if input_dir:
         os.makedirs(input_dir, exist_ok=True)
+    if _should_use_system_zip_cli() and shutil.which("unzip"):
+        try:
+            os.makedirs(workspace_path, exist_ok=True)
+            subprocess.run(
+                ["unzip", "-oq", zip_path, "-d", workspace_path],
+                check=True,
+            )
+            return
+        except Exception:
+            typer.echo(
+                "System unzip failed; falling back to Python zipfile.",
+                err=True,
+            )
     with ZipFile(zip_path, "r") as zip_file:
-        members = zip_file.namelist()
+        members = zip_file.infolist()
         for member in tqdm(members, desc="Unzipping", unit="file"):
             zip_file.extract(member, workspace_path)
 
@@ -407,7 +471,7 @@ def sync_one(
     if zip_deleted and direction == "to-zip":
         typer.echo(f"Rezipping '{workspace_path}' (zip was deleted)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        subprocess.run(["dvc", "add", zip_path], check=True)
+        run_dvc_command(["add", zip_path])
         zip_hash = get_hash(zip_path)
     # Workspace was deleted but zip exists and direction is to-workspace:
     # restore workspace
@@ -429,7 +493,7 @@ def sync_one(
     if workspace_changed and (direction in ["to-zip", "both"]):
         typer.echo(f"Zipping '{workspace_path}' (workspace has changed)")
         zip_(workspace_path=workspace_path, zip_path=zip_path)
-        subprocess.run(["dvc", "add", zip_path], check=True)
+        run_dvc_command(["add", zip_path])
         zip_hash = get_hash(zip_path)
     # If we unzip, we need to update the hash
     if zip_changed and (direction in ["to-workspace", "both"]):

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/pipeline.py

@@ -383,10 +383,31 @@ def get_status(
             if v != ["always changed"] and not k.endswith(".dvc")
         }
         stages_config = ck_info.get("pipeline", {}).get("stages", {})
+        # Build an ordered list of stage names from dvc.yaml to preserve
+        # pipeline order, since dvc_repo.status() returns stages alphabetically
+        dvc_yaml_stages: list[str] = []
+        if os.path.isfile("dvc.yaml"):
+            try:
+                with open("dvc.yaml") as f:
+                    dvc_yaml = calkit.ryaml.load(f)
+                dvc_yaml_stages = list(
+                    (dvc_yaml or {}).get("stages", {}).keys()
+                )
+            except Exception:
+                pass
         ordered_stale_stages = {}
-        for stage_name in stages_config.keys():
-            if stage_name not in raw_stale_stages:
-                continue
+        # First, add stages in dvc.yaml order, matching expanded stage names
+        # (e.g. benchmark-boom@1-3-1) against their base template name
+        # (benchmark-boom) using the position in dvc.yaml
+        dvc_yaml_stage_order = {
+            name: i for i, name in enumerate(dvc_yaml_stages)
+        }
+
+        def _stage_sort_key(stage_name: str) -> int:
+            base = stage_name.split("@")[0]
+            return dvc_yaml_stage_order.get(base, len(dvc_yaml_stages))
+
+        for stage_name in sorted(raw_stale_stages.keys(), key=_stage_sort_key):
             status_data = raw_stale_stages[stage_name]
             ordered_stale_stages[stage_name] = StaleStage.from_status_data(
                 status_data=status_data,
@@ -399,23 +420,14 @@ def get_status(
                     )
                 ],
             )
-        # Keep any stale stages not present in ck_info at the end
-        for stage_name, status_data in raw_stale_stages.items():
-            if stage_name in ordered_stale_stages:
-                continue
-            ordered_stale_stages[stage_name] = StaleStage.from_status_data(
-                status_data=status_data,
-                configured_outputs=[
-                    output.get("path", str(output))
-                    if isinstance(output, dict)
-                    else str(output)
-                    for output in stages_config.get(stage_name, {}).get(
-                        "outputs", []
-                    )
-                ],
-            )
         result["stale_stages"] = ordered_stale_stages
-        return PipelineStatus.model_validate(result)
+        return PipelineStatus(
+            has_pipeline=result["has_pipeline"],
+            environment_checks=result["environment_checks"],
+            cleaned_notebooks=result["cleaned_notebooks"],
+            stale_stages=result["stale_stages"],
+            errors=result["errors"],
+        )
     finally:
         if wdir is not None:
             os.chdir(prev_cwd)

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/releases.py

@@ -184,16 +184,21 @@ def make_dvc_md5s(
     return resp
 
 
-def add_paths_to_zip(zipf: zipfile.ZipFile, paths: list[str]) -> None:
-    """Add files to a zip archive, expanding directories recursively."""
-    for path in paths:
-        if os.path.isdir(path):
-            for root, _, files in os.walk(path):
-                for filename in files:
-                    fpath = os.path.join(root, filename)
-                    zipf.write(fpath)
-        elif os.path.isfile(path):
-            zipf.write(path)
+def zip_paths(zip_path: str, paths: list[str]) -> None:
+    """Create a compressed ZIP from a list of file or directory paths."""
+    with zipfile.ZipFile(
+        zip_path,
+        "w",
+        compression=zipfile.ZIP_DEFLATED,
+    ) as zipf:
+        for path in paths:
+            if os.path.isdir(path):
+                for root, _, files in os.walk(path):
+                    for filename in files:
+                        fpath = os.path.join(root, filename)
+                        zipf.write(fpath)
+            elif os.path.isfile(path):
+                zipf.write(path)
 
 
 def populate_dvc_cache():

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/tests/dvc/test_core.py

@@ -4,6 +4,8 @@ import os
 import subprocess
 
 import dvc.repo
+import git
+from configobj import ConfigObj
 from dvc.config_schema import SCHEMA, Invalid
 from dvc_objects.fs import known_implementations
 
@@ -89,3 +91,70 @@ def test_list_files_paths(tmp_dir):
     repo = dvc.repo.Repo()
     repo.add("file1.txt")  # type: ignore
     assert "file1.txt" in calkit.dvc.list_paths()
+
+
+def test_configure_remote_ck_uses_ck_scheme_and_skips_http_auth(monkeypatch):
+    monkeypatch.setattr(calkit, "detect_project_name", lambda wdir=None: "o/p")
+
+    class DummyRepo:
+        def remote(self):
+            return "origin"
+
+    monkeypatch.setattr(git, "Repo", lambda wdir=None: DummyRepo())
+    calls = []
+    events = []
+
+    def fake_run(argv, cwd=None):
+        events.append("run")
+        calls.append((argv, cwd))
+        return 0
+
+    monkeypatch.setattr(calkit.dvc.core, "run_dvc_command", fake_run)
+    monkeypatch.setattr(
+        calkit.dvc.core,
+        "clear_remote_local_http_auth",
+        lambda remote_name=None, wdir=None: events.append("clear"),
+    )
+    out = calkit.dvc.configure_remote(use_ck=True)
+    assert out == calkit.config.get_app_name()
+    assert events and events[0] == "clear"
+    assert calls == [
+        (
+            [
+                "remote",
+                "add",
+                "-d",
+                "-f",
+                calkit.config.get_app_name(),
+                "ck://o/p",
+            ],
+            None,
+        )
+    ]
+
+
+def test_set_remote_auth_ck_remote_clears_local_http_auth(
+    monkeypatch, tmp_path
+):
+    remote_name = calkit.config.get_app_name()
+    monkeypatch.setattr(
+        calkit.dvc.core,
+        "get_remotes",
+        lambda wdir=None: {remote_name: "ck://owner/proj"},
+    )
+    dvc_dir = tmp_path / ".dvc"
+    dvc_dir.mkdir()
+    fpath = dvc_dir / "config.local"
+    with open(fpath, "w") as f:
+        f.write(
+            f'[remote "{remote_name}"]\n'
+            "    custom_auth_header = Authorization\n"
+            "    password = Bearer token\n"
+            "    url = https://example.com\n"
+        )
+    calkit.dvc.set_remote_auth(wdir=str(tmp_path))
+    cfg = ConfigObj(str(fpath), encoding="utf-8")
+    section = cfg[f'remote "{remote_name}"']
+    assert "custom_auth_header" not in section
+    assert "password" not in section
+    assert section["url"] == "https://example.com"  # type: ignore

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/tests/dvc/test_zip.py

@@ -200,7 +200,14 @@ def test_get_sync_status(tmp_dir):
 
 
 def test_sync_one(tmp_dir, monkeypatch):
-    monkeypatch.setattr("calkit.dvc.zip.subprocess.run", lambda *_, **__: None)
+    real_run = calkit.dvc.zip.subprocess.run
+
+    def _run(cmd, *args, **kwargs):
+        if cmd and cmd[0] == "dvc":
+            return None
+        return real_run(cmd, *args, **kwargs)
+
+    monkeypatch.setattr("calkit.dvc.zip.subprocess.run", _run)
     # to-zip: zips input and writes sync record
     src = tmp_dir / "src"
     src.mkdir()

{calkit_python-0.37.0 → calkit_python-0.37.2}/calkit/tests/test_releases.py

@@ -10,10 +10,10 @@ import git
 import pytest
 
 from calkit.releases import (
-    add_paths_to_zip,
     check_project_release_archive,
     create_bibtex,
     ls_files,
+    zip_paths,
 )
 
 
@@ -135,16 +135,16 @@ def test_create_bibtex():
     assert len(entries) == 1
 
 
-def test_add_paths_to_zip_expands_directory_contents(tmp_dir):
+def test_zip_paths(tmp_dir):
     os.makedirs("data/sub", exist_ok=True)
     with open("data/sub/file.txt", "w") as f:
         f.write("hello")
     with open("root.txt", "w") as f:
         f.write("root")
     zip_path = "archive.zip"
-    with zipfile.ZipFile(zip_path, "w") as zipf:
-        add_paths_to_zip(zipf, ["data", "root.txt"])
+    zip_paths(zip_path, ["data", "root.txt"])
     with zipfile.ZipFile(zip_path) as zipf:
         names = set(zipf.namelist())
-    assert "data/sub/file.txt" in names
-    assert "root.txt" in names
+        assert "data/sub/file.txt" in names
+        assert "root.txt" in names
+        assert zipf.getinfo("root.txt").compress_type == zipfile.ZIP_DEFLATED