calkit-python 0.37.3__tar.gz → 0.37.5__tar.gz

{calkit_python-0.37.3 → calkit_python-0.37.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.37.3
+Version: 0.37.5
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/cli/check.py

@@ -876,15 +876,26 @@ def check_docker_env(
                     typer.echo(f"Found modified dependency: {dep}")
                     rebuild_or_pull = True
                     break
+
+    def delete_lock_on_failure() -> None:
+        if lock_fpath and os.path.exists(lock_fpath):
+            os.remove(lock_fpath)
+
     if fpath is not None and rebuild_or_pull:
-        wdir, fname = os.path.split(fpath)
-        if not wdir:
-            wdir = None
-        cmd = ["docker", "build", "-t", tag, "-f", fname]
+        dockerfile_dir, dockerfile_name = os.path.split(fpath)
+        if not dockerfile_dir:
+            dockerfile_dir = None
+        cmd = ["docker", "build", "-t", tag, "-f", dockerfile_name]
         if platform is not None:
             cmd += ["--platform", platform]
         cmd.append(".")
-        subprocess.check_output(cmd, cwd=wdir)
+        try:
+            subprocess.check_output(cmd, cwd=dockerfile_dir)
+        except subprocess.CalledProcessError:
+            delete_lock_on_failure()
+            raise_error(
+                f"Failed to build Docker image with tag {tag} from {fpath}"
+            )
     elif fpath is None and rebuild_or_pull:
         # First try to pull by repo digest
         pulled = False
@@ -901,6 +912,7 @@ def check_docker_env(
                     subprocess.check_output(tag_cmd)
                     pulled = True
                 except subprocess.CalledProcessError:
+                    delete_lock_on_failure()
                     warn(
                         f"Failed to pull image by digest: {image_with_digest}; "
                         "falling back to pulling by tag"
@@ -912,6 +924,7 @@ def check_docker_env(
             try:
                 subprocess.check_output(cmd)
             except subprocess.CalledProcessError:
+                delete_lock_on_failure()
                 raise_error(f"Failed to pull image: {tag}")
     # Write the lock file
     inspect = get_docker_inspect()
@@ -993,7 +1006,7 @@ def check_conda_env(
     try:
         calkit.conda.check_env(
             env_fpath=env_fpath,
-            output_fpath=output_fpath,
+            lock_fpath=output_fpath,
             alt_lock_fpaths=alt_lock_fpaths,
             alt_lock_fpaths_delete=alt_lock_fpaths_delete,
             log_func=log_func,

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/cli/main/core.py

@@ -958,10 +958,9 @@ def push(
 ):
     """Push with both Git and DVC."""
     if not no_dvc:
-        typer.echo("Pushing to DVC remote")
+        remotes = calkit.dvc.get_remotes()
         if not no_check_auth:
             # Check that our dvc remotes all have our DVC token set for them
-            remotes = calkit.dvc.get_remotes()
             ck_remote_name = calkit.config.get_app_name()
             for name, url in remotes.items():
                 if (
@@ -972,9 +971,13 @@ def push(
                         f"Checking authentication for DVC remote: {name}"
                     )
                     calkit.dvc.set_remote_auth(remote_name=name)
-        result = run_dvc_command(["push"] + dvc_args)
-        if result != 0:
-            raise_error("DVC push failed")
+        if remotes:
+            typer.echo("Pushing to DVC remote")
+            result = run_dvc_command(["push"] + dvc_args)
+            if result != 0:
+                raise_error("DVC push failed")
+        else:
+            warn("No DVC remotes configured; skipping DVC push")
     if not no_git:
         typer.echo("Pushing to Git remote")
         try:

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/cli/new.py

@@ -1204,10 +1204,8 @@ def new_conda_env(
             "(use -f to overwrite)"
         )
     if conda_name is None:
-        project_name = ck_info.get("name")
-        if project_name is None:
-            project_name = os.path.basename(os.getcwd())
-        conda_name = calkit.to_kebab_case(project_name) + "-" + name
+        project_name = calkit.detect_project_name(prepend_owner=False)
+        conda_name = calkit.to_kebab_case(project_name) + "." + name
     if packages is not None:
         assert isinstance(packages, list)
         # Write environment to path

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/cli/notebooks.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import base64
 import json
 import os
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -122,6 +123,17 @@ def _check_ijulia_available(
     return res.returncode == 0
 
 
+def _sanitize_kernel_name_component(name: str, label: str) -> str:
+    """Keep readable names while removing kernelspec-unfriendly characters."""
+    sanitized = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("._-")
+    if not sanitized:
+        raise_error(
+            f"{label} cannot be empty after sanitizing for kernel name"
+        )
+        return ""  # For typing analysis since raise_error exits
+    return sanitized
+
+
 @notebooks_app.command("check-kernel")
 def check_env_kernel(
     env_name: Annotated[
@@ -191,8 +203,14 @@ def check_env_kernel(
     project_name = calkit.detect_project_name(prepend_owner=False)
     if not project_name:
         raise_error("Project name cannot be empty")
-    kernel_name = calkit.to_kebab_case(f"{project_name}-{env_name}")
+    kernel_name = (
+        f"{_sanitize_kernel_name_component(project_name, 'Project name')}"
+        f".{_sanitize_kernel_name_component(env_name, 'Environment name')}"
+    )
     display_name = f"{project_name}: {env_name}"
+    if verbose and not json_output:
+        typer.echo(f"Using kernel name: {kernel_name}")
+        typer.echo(f"Using display name: {display_name}")
     if language == "python":
         cmd = [
             "python",
@@ -295,9 +313,10 @@ def check_env_kernel(
         julia_cmd = (
             "import IJulia;"
             "kp=IJulia.installkernel("
-            f'"{display_name}",'
+            f'"{kernel_name}",'
             f'"--project={env_dir_abs}",'
             '"--startup-file=no",'
+            f'displayname="{display_name}",'
             'env=Dict("JULIA_LOAD_PATH" => "@:@stdlib")'
             ");"
             "println(kp);"
@@ -316,7 +335,7 @@ def check_env_kernel(
         cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
         res = subprocess.run(cmd, capture_output=True, text=True)
         if res.returncode != 0:
-            raise_error(f"Failed to create kernel:\n{res.stdout}")
+            raise_error(f"Failed to create kernel:\n{res.stderr}")
         kernel_path = res.stdout.strip()
         kernel_name = os.path.basename(kernel_path)
         # Update display_name to include version for matching in VS Code
@@ -567,9 +586,9 @@ def execute_notebook(
         run_in_env(cmd, env_name=env_name, no_check=no_check, verbose=verbose)
     if not no_replace:
         # Replace original notebook outputs with those from executed version
-        with open(fpath_out_exec, "r") as f:
+        with open(fpath_out_exec, "r", encoding="utf-8") as f:
             executed_nb = json.load(f)
-        with open(path, "r") as f:
+        with open(path, "r", encoding="utf-8") as f:
             original_nb = json.load(f)
         for orig_cell, exec_cell in zip(
             original_nb.get("cells", []), executed_nb.get("cells", [])
@@ -581,7 +600,7 @@ def execute_notebook(
                 and "execution_count" in exec_cell
             ):
                 orig_cell["execution_count"] = exec_cell["execution_count"]
-        with open(path, "w") as f:
+        with open(path, "w", encoding="utf-8") as f:
             json.dump(original_nb, f, indent=1)
     for to_fmt in to:
         if to_fmt != "notebook":

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/conda.py

@@ -262,7 +262,7 @@ class EnvCheckResult(BaseModel):
 def check_env(
     env_fpath: str = "environment.yml",
     log_func=None,
-    output_fpath: str | None = None,
+    lock_fpath: str | None = None,
     alt_lock_fpaths: list[str] = [],
     alt_lock_fpaths_delete: list[str] = [],
     relaxed: bool = False,
@@ -284,7 +284,7 @@ def check_env(
     # Determine which lock file to use for creating the environment
     lock_to_use_for_creation = None
     used_legacy_lock = None
-    if output_fpath and not os.path.isfile(output_fpath):
+    if lock_fpath and not os.path.isfile(lock_fpath):
         # Try alternative lock files first
         for alt_fpath in alt_lock_fpaths:
             if os.path.isfile(alt_fpath):
@@ -302,9 +302,31 @@ def check_env(
                     f"Using legacy lock file for creation: {legacy_fpath}"
                 )
                 break
-    elif output_fpath and os.path.isfile(output_fpath):
-        lock_to_use_for_creation = output_fpath
-        log_func(f"Using existing lock file for creation: {output_fpath}")
+    elif lock_fpath and os.path.isfile(lock_fpath):
+        lock_to_use_for_creation = lock_fpath
+        log_func(f"Using existing lock file for creation: {lock_fpath}")
+    # Make sure the lock file has the correct env name in it
+    if lock_to_use_for_creation:
+        with open(lock_to_use_for_creation) as f:
+            lock_spec = ryaml.load(f)
+        lock_env_name = lock_spec.get("name")
+        if lock_env_name is not None:
+            with open(env_fpath) as f:
+                env_spec = ryaml.load(f)
+            env_spec_env_name = env_spec.get("name")
+            if (
+                env_spec_env_name is not None
+                and lock_env_name != env_spec_env_name
+            ):
+                log_func(
+                    f"Lock file {lock_to_use_for_creation} has env name "
+                    f"{lock_env_name}, which does not match env spec "
+                    f"name {env_spec_env_name}; deleting mismatched lock file "
+                    "and ignoring it for creation"
+                )
+                if os.path.isfile(lock_to_use_for_creation):
+                    os.remove(lock_to_use_for_creation)
+                lock_to_use_for_creation = None
     res = EnvCheckResult()
     if verbose:
         log_func("Getting conda info")
@@ -538,15 +560,15 @@ def check_env(
         )
         with open(env_check_fpath, "w") as f:
             ryaml.dump(env_check, f)
-    if output_fpath is None:
+    if lock_fpath is None:
         fname, ext = os.path.splitext(env_fpath)
-        output_fpath = fname + "-lock" + ext
+        lock_fpath = fname + "-lock" + ext
     if (
         not res.env_exists
         or res.env_needs_rebuild
-        or not os.path.isfile(output_fpath)
+        or not os.path.isfile(lock_fpath)
     ):
-        log_func(f"Exporting lock file to {output_fpath}")
+        log_func(f"Exporting lock file to {lock_fpath}")
         env_export = json.loads(
             subprocess.check_output(
                 [a for a in export_cmd if a != "--no-builds"]
@@ -584,16 +606,16 @@ def check_env(
                 dep_name = re.split("[=<>]+", dep, maxsplit=1)[0]
                 if dep_name in editable_pip_deps:
                     path_rel_to_project_root = editable_pip_deps[dep_name]
-                    lock_dir = os.path.dirname(output_fpath)
+                    lock_dir = os.path.dirname(lock_fpath)
                     path_rel_to_lock = os.path.relpath(
                         path_rel_to_project_root, start=lock_dir
                     )
                     export_pip_deps[i] = (
                         "-e " + Path(path_rel_to_lock).as_posix()
                     )
-        out_dir = os.path.dirname(output_fpath)
+        out_dir = os.path.dirname(lock_fpath)
         if out_dir:
             os.makedirs(out_dir, exist_ok=True)
-        with open(output_fpath, "w") as f:
+        with open(lock_fpath, "w") as f:
             ryaml.dump(env_export, f)
     return res

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/fs.py

@@ -298,10 +298,45 @@ class CalkitFileSystem(AbstractFileSystem):
         operation_info: dict[str, Any],
         operation: str,
         data: bytes | None = None,
+        file_obj: Any = None,
+        file_size: int | None = None,
         headers: dict | None = None,
         callback=DEFAULT_CALLBACK,
     ) -> requests.Response:
-        """Execute a file operation using the provided operation info."""
+        """Execute a file operation using the provided operation info.
+
+        Exactly one of ``data`` or ``file_obj`` should be provided for put
+        operations. When ``file_obj`` is given it must be a seekable binary
+        file-like object and ``file_size`` must be set; part/chunk bodies are
+        streamed from it instead of being held in memory, which keeps memory
+        bounded for large uploads.
+        """
+        if data is not None and file_obj is not None:
+            raise ValueError("Provide data or file_obj, not both")
+        if file_obj is not None:
+            if not callable(getattr(file_obj, "seek", None)):
+                raise TypeError(
+                    "file_obj must be a seekable binary file-like object"
+                )
+            if file_size is None:
+                raise ValueError(
+                    "file_size must be provided when file_obj is given"
+                )
+        # Total payload size for progress reporting.
+        payload_size = len(data) if data is not None else (file_size or 0)
+
+        def _prepare_stream(request_headers: dict) -> Any:
+            """Return the request body, setting Content-Length if streaming."""
+            if data is not None:
+                return data
+            if file_obj is None:
+                return None
+            file_obj.seek(0)
+            # Explicit Content-Length avoids chunked transfer-encoding, which
+            # presigned URL signatures typically reject.
+            request_headers.setdefault("Content-Length", str(file_size))
+            return file_obj
+
         # Extract access info from the operation info
         access = operation_info.get("access")
         if not access:
@@ -322,16 +357,17 @@ class CalkitFileSystem(AbstractFileSystem):
             if headers:
                 request_headers.update(headers)
             params = access.get("params")
+            body: Any = _prepare_stream(request_headers)
             resp = self._session.request(
                 method=http_method.upper(),
                 url=url,
                 headers=request_headers,
                 params=params,
-                data=data,
+                data=body,
                 timeout=120,
             )
-            if data is not None and operation == "put":
-                callback.relative_update(len(data))
+            if operation == "put" and payload_size:
+                callback.relative_update(payload_size)
             return resp
         elif kind == "http-request":
             # Generic HTTP request with custom headers
@@ -345,20 +381,21 @@ class CalkitFileSystem(AbstractFileSystem):
             if headers:
                 request_headers.update(headers)
             params = access.get("params")
+            body = _prepare_stream(request_headers)
             resp = self._session.request(
                 method=http_method.upper(),
                 url=url,
                 headers=request_headers,
                 params=params,
-                data=data,
+                data=body,
                 timeout=120,
             )
-            if data is not None and operation == "put":
-                callback.relative_update(len(data))
+            if operation == "put" and payload_size:
+                callback.relative_update(payload_size)
             return resp
         elif kind == "presigned-multipart":
             # S3 multipart upload using server-provided part URLs
-            if data is None:
+            if data is None and file_obj is None:
                 raise ValueError("Data required for multipart upload")
             upload_id = access.get("upload_id")
             part_urls = access.get("part_urls")
@@ -382,7 +419,8 @@ class CalkitFileSystem(AbstractFileSystem):
                     "presigned-multipart"
                 )
             content_type = access.get("content_type")
-            total_parts_needed = (len(data) + part_size - 1) // part_size
+            total_bytes = len(data) if data is not None else int(file_size)
+            total_parts_needed = (total_bytes + part_size - 1) // part_size
             if total_parts_needed > len(part_urls):
                 raise ValueError(
                     "Insufficient part URLs for multipart upload "
@@ -391,8 +429,12 @@ class CalkitFileSystem(AbstractFileSystem):
             uploaded_parts: list[tuple[int, str]] = []
             for part_num in range(1, total_parts_needed + 1):
                 start = (part_num - 1) * part_size
-                end = min(start + part_size, len(data))
-                part_data = data[start:end]
+                end = min(start + part_size, total_bytes)
+                if data is not None:
+                    part_data = data[start:end]
+                else:
+                    file_obj.seek(start)
+                    part_data = file_obj.read(end - start)
                 part_url = part_urls[part_num - 1]
                 part_headers = {}
                 if content_type:
@@ -429,7 +471,7 @@ class CalkitFileSystem(AbstractFileSystem):
             return complete_resp
         elif kind == "presigned-chunked":
             # GCS resumable upload - requires multiple requests
-            if data is None:
+            if data is None and file_obj is None:
                 raise ValueError("Data required for chunked upload")
             init_url = access.get("init_url")
             if not init_url:
@@ -477,7 +519,7 @@ class CalkitFileSystem(AbstractFileSystem):
                         pass
 
             # Step 2: Upload data in chunks
-            total_size = len(data)
+            total_size = len(data) if data is not None else int(file_size)
             offset = 0
             chunk_headers_base = dict(
                 access.get("chunk_headers") or access.get("headers") or {}
@@ -510,7 +552,11 @@ class CalkitFileSystem(AbstractFileSystem):
             try:
                 while offset < total_size:
                     chunk_end = min(offset + chunk_size, total_size)
-                    chunk_data = data[offset:chunk_end]
+                    if data is not None:
+                        chunk_data = data[offset:chunk_end]
+                    else:
+                        file_obj.seek(offset)
+                        chunk_data = file_obj.read(chunk_end - offset)
                     # Set Content-Range header for the chunk
                     chunk_headers = {
                         **chunk_headers_base,
@@ -915,12 +961,9 @@ class CalkitFileSystem(AbstractFileSystem):
         if os.path.isdir(lpath):
             self.makedirs(rpath, exist_ok=True)
             return None
-        with open(lpath, "rb") as f:
-            size = f.seek(0, 2)
-            callback.set_size(size)
-            f.seek(0)
-            data = f.read()
         owner, project, file_path = _parse_path(rpath)
+        size = os.path.getsize(lpath)
+        callback.set_size(size)
         operation_info = self._get_fs_op_info(
             owner,
             project,
@@ -928,12 +971,17 @@ class CalkitFileSystem(AbstractFileSystem):
             "put",
             content_length=size,
         )
-        resp = self._execute_operation(
-            operation_info,
-            "put",
-            data=data,
-            callback=callback,
-        )
+        # Stream the file from disk rather than loading it into memory. This
+        # keeps RAM bounded when DVC pushes many (or very large) files in
+        # parallel via its worker pool.
+        with open(lpath, "rb") as f:
+            resp = self._execute_operation(
+                operation_info,
+                "put",
+                file_obj=f,
+                file_size=size,
+                callback=callback,
+            )
         resp.raise_for_status()
 
 

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/tests/cli/test_check.py

@@ -122,3 +122,39 @@ def test_check_julia_env_caches_second_run(tmp_dir):
         check=True,
     )
     assert "skipping Pkg.instantiate" not in result3.stdout
+
+
+def test_check_docker_env(tmp_dir):
+    # First create a Dockerfile with a known base image
+    with open("Dockerfile", "w") as f:
+        f.write("FROM python:3.9-slim\n")
+    # Now check the environment
+    subprocess.check_call(
+        [
+            "calkit",
+            "check",
+            "docker-env",
+            "python-3.9-slim",
+            "-i",
+            "Dockerfile",
+            "-o",
+            "Dockerfile-lock.json",
+        ]
+    )
+    # Now modify the image to fail to build and ensure the lock file is deleted
+    with open("Dockerfile", "w") as f:
+        f.write("FROM non-existent-image:latest\n")
+    with pytest.raises(subprocess.CalledProcessError):
+        subprocess.check_call(
+            [
+                "calkit",
+                "check",
+                "docker-env",
+                "python-3.9-slim",
+                "-i",
+                "Dockerfile",
+                "-o",
+                "Dockerfile-lock.json",
+            ]
+        )
+    assert not os.path.exists("Dockerfile-lock.json")

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/tests/cli/test_new.py

@@ -277,7 +277,7 @@ def test_new_conda_env(tmp_dir):
     )
     with open("environment.yml") as f:
         env = calkit.ryaml.load(f)
-    assert env["name"] == "test-e1"
+    assert env["name"] == "test.e1"
     assert env["dependencies"] == ["python", "requests"]
 
 

{calkit_python-0.37.3 → calkit_python-0.37.5}/calkit/tests/test_conda.py

@@ -69,7 +69,7 @@ def delete_env(name: str):
 
 @pytest.fixture
 def conda_env_name():
-    name = calkit.to_kebab_case(os.path.basename(os.getcwd())) + "-" + ENV_NAME
+    name = calkit.to_kebab_case(os.path.basename(os.getcwd())) + "." + ENV_NAME
     yield name
     # Teardown code
     delete_env(name)