calkit-python 0.37.2__py3-none-any.whl → 0.37.4__py3-none-any.whl

calkit/cli/check.py

@@ -31,6 +31,217 @@ from calkit.environments import (
 check_app = typer.Typer(no_args_is_help=True)
 
 
+def _juliaup_version_installed(julia_version: str) -> bool:
+    """Return True if juliaup already has the given channel installed locally."""
+    try:
+        result = subprocess.run(
+            ["juliaup", "status"],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+    except FileNotFoundError:
+        return False
+    except subprocess.TimeoutExpired:
+        warn("Timed out while running `juliaup status`.")
+        return False
+    if result.returncode != 0:
+        err_output = (result.stderr or result.stdout or "").strip()
+        if err_output:
+            warn(f"`juliaup status` failed: {err_output}")
+        else:
+            warn(
+                f"`juliaup status` failed with exit code {result.returncode}."
+            )
+        return False
+    for line in result.stdout.splitlines():
+        parts = line.split()
+        if not parts:
+            continue
+        # Columns: [Default?] Channel Version [Update?]
+        # Default column is either empty or "*", so channel is parts[0] or parts[1]
+        for col in parts[:2]:
+            if col == julia_version:
+                return True
+    return False
+
+
+def _check_julia_env(
+    env_path: str,
+    julia_version: str | None = None,
+    verbose: bool = False,
+    cache_key: str | None = None,
+) -> str:
+    """Check a Julia environment and instantiate only when needed."""
+    abs_env_path = os.path.abspath(env_path)
+    env_fname = os.path.basename(abs_env_path)
+    if env_fname != "Project.toml":
+        raise_error(
+            "Julia environments require a path pointing to Project.toml"
+        )
+    env_dir = os.path.dirname(abs_env_path) or "."
+    env_path_for_cache = os.path.basename(abs_env_path)
+    env = {
+        "kind": "julia",
+        "path": env_path_for_cache,
+        "julia": julia_version or "",
+    }
+    cache_env_name = cache_key or (
+        f"julia::{abs_env_path}::{julia_version or ''}"
+    )
+    if calkit.environments.check_cache(
+        env_name=cache_env_name,
+        env=env,
+        wdir=env_dir,
+        respect_ttl=False,
+    ):
+        if verbose:
+            typer.echo(
+                "Julia environment cache is valid; skipping Pkg.instantiate()"
+            )
+        lock_fpath = calkit.environments.get_env_lock_fpath(
+            env=env,
+            env_name=cache_env_name,
+            wdir=env_dir,
+            as_posix=False,
+        )
+        return lock_fpath or os.path.join(env_dir, "Manifest.toml")
+    if julia_version:
+        if shutil.which("juliaup") is not None:
+            if _juliaup_version_installed(julia_version):
+                if verbose:
+                    typer.echo(
+                        f"Julia {julia_version} is already installed; "
+                        "skipping juliaup add"
+                    )
+            else:
+                cmd = ["juliaup", "add", julia_version]
+                if verbose:
+                    typer.echo(f"Running command: {cmd}")
+                try:
+                    subprocess.run(cmd, check=True)
+                except subprocess.CalledProcessError:
+                    raise_error(
+                        f"Failed to install Julia version {julia_version}"
+                    )
+        else:
+            try:
+                compatible = calkit.julia.current_version_is_compatible(
+                    julia_version
+                )
+            except ValueError as e:
+                raise_error(str(e))
+            if not compatible:
+                raise_error(
+                    "Current Julia version is not compatible with required "
+                    f"version ({julia_version}), and juliaup is not "
+                    "available to install it"
+                )
+    deps_to_add: list[str] = []
+    try:
+        with open(abs_env_path, "r") as f:
+            content = f.read()
+        lines = [line.rstrip() for line in content.splitlines()]
+        deps_section = False
+        deps_found = False
+        for line in lines:
+            stripped = line.strip()
+            if stripped == "[deps]":
+                deps_section = True
+                continue
+            if deps_section:
+                if stripped.startswith("[") and stripped.endswith("]"):
+                    break
+                if stripped and not stripped.startswith("#"):
+                    if "=" in stripped:
+                        deps_found = True
+                        break
+        if not deps_found:
+            for idx, line in enumerate(lines):
+                marker = "# Dependencies (add with Julia's Pkg.add):"
+                if line.strip() == marker and idx + 1 < len(lines):
+                    dep_line = lines[idx + 1].strip()
+                    if dep_line.startswith("#"):
+                        dep_line = dep_line.lstrip("#").strip()
+                    deps_to_add = [
+                        dep.strip()
+                        for dep in dep_line.split(",")
+                        if dep.strip()
+                    ]
+                    break
+    except OSError:
+        deps_to_add = []
+    if deps_to_add:
+        pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
+        cmd = ["julia"]
+        if julia_version:
+            cmd.append(f"+{julia_version}")
+        cmd += [
+            f"--project={env_dir}",
+            "-e",
+            f"using Pkg; Pkg.add([{pkg_list}]);",
+        ]
+        if julia_version:
+            try:
+                cmd = calkit.julia.check_version_in_command(cmd)
+            except Exception as e:
+                raise_error(f"Failed to check Julia version: {e}")
+        cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
+        try:
+            subprocess.check_call(
+                cmd,
+                env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
+            )
+        except subprocess.CalledProcessError:
+            calkit.environments.save_cache(
+                env_name=cache_env_name,
+                env=env,
+                wdir=env_dir,
+                success=False,
+            )
+            raise_error("Failed to add Julia dependencies")
+    cmd = ["julia"]
+    if julia_version:
+        cmd.append(f"+{julia_version}")
+    cmd += [
+        f"--project={env_dir}",
+        "-e",
+        "using Pkg; Pkg.instantiate();",
+    ]
+    if julia_version:
+        try:
+            cmd = calkit.julia.check_version_in_command(cmd)
+        except Exception as e:
+            raise_error(f"Failed to check Julia version: {e}")
+    cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
+    try:
+        subprocess.check_call(
+            cmd, env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"}
+        )
+    except subprocess.CalledProcessError:
+        calkit.environments.save_cache(
+            env_name=cache_env_name,
+            env=env,
+            wdir=env_dir,
+            success=False,
+        )
+        raise_error("Failed to check julia environment")
+    calkit.environments.save_cache(
+        env_name=cache_env_name,
+        env=env,
+        wdir=env_dir,
+        success=True,
+    )
+    lock_fpath = calkit.environments.get_env_lock_fpath(
+        env=env,
+        env_name=cache_env_name,
+        wdir=env_dir,
+        as_posix=False,
+    )
+    return lock_fpath or os.path.join(env_dir, "Manifest.toml")
+
+
 @check_app.command(name="repro")
 def check_repro(
     wdir: Annotated[
@@ -190,116 +401,47 @@ def check_environment(
         julia_version = env.get("julia")
         if julia_version is None:
             raise_error("Julia environments require a Julia version")
-        env_fname = os.path.basename(env_path)
-        if not env_fname == "Project.toml":
-            raise_error(
-                "Julia environments require a path pointing to Project.toml"
-            )
-        # First ensure the Julia version exists
-        if shutil.which("juliaup") is not None:
-            cmd = ["juliaup", "add", julia_version]
-            if verbose:
-                typer.echo(f"Running command: {cmd}")
-            try:
-                subprocess.run(cmd, check=True)
-            except subprocess.CalledProcessError:
-                raise_error(f"Failed to install Julia version {julia_version}")
-        else:
-            try:
-                compatible = calkit.julia.current_version_is_compatible(
-                    julia_version
-                )
-            except ValueError as e:
-                raise_error(str(e))
-            if not compatible:
-                raise_error(
-                    f"Current Julia version is not compatible with required "
-                    f"version ({julia_version}), and juliaup is not available to "
-                    "install it"
-                )
-        env_dir = os.path.dirname(env_path)
-        if not env_dir:
-            env_dir = "."
-        # If auto-detection couldn't resolve UUIDs, Project.toml includes a
-        # commented dependency list
-        # In that case, add those packages with
-        # Pkg.add before instantiating so the env is usable at run time
-        deps_to_add: list[str] = []
-        try:
-            with open(env_path, "r") as f:
-                content = f.read()
-            lines = [line.rstrip() for line in content.splitlines()]
-            deps_section = False
-            deps_found = False
-            for line in lines:
-                stripped = line.strip()
-                if stripped == "[deps]":
-                    deps_section = True
-                    continue
-                if deps_section:
-                    if stripped.startswith("[") and stripped.endswith("]"):
-                        break
-                    if stripped and not stripped.startswith("#"):
-                        if "=" in stripped:
-                            deps_found = True
-                            break
-            if not deps_found:
-                for idx, line in enumerate(lines):
-                    marker = "# Dependencies (add with Julia's Pkg.add):"
-                    if line.strip() == marker and idx + 1 < len(lines):
-                        dep_line = lines[idx + 1].strip()
-                        if dep_line.startswith("#"):
-                            dep_line = dep_line.lstrip("#").strip()
-                        deps_to_add = [
-                            dep.strip()
-                            for dep in dep_line.split(",")
-                            if dep.strip()
-                        ]
-                        break
-        except OSError:
-            deps_to_add = []
-        if deps_to_add:
-            pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
-            cmd = [
-                "julia",
-                f"+{julia_version}",
-                f"--project={env_dir}",
-                "-e",
-                f"using Pkg; Pkg.add([{pkg_list}]);",
-            ]
-            try:
-                cmd = calkit.julia.check_version_in_command(cmd)
-            except Exception as e:
-                raise_error(f"Failed to check Julia version: {e}")
-            try:
-                subprocess.check_call(
-                    cmd,
-                    env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
-                )
-            except subprocess.CalledProcessError:
-                raise_error("Failed to add Julia dependencies")
-        cmd = [
-            "julia",
-            f"+{julia_version}",
-            f"--project={env_dir}",
-            "-e",
-            "using Pkg; Pkg.instantiate();",
-        ]
-        try:
-            cmd = calkit.julia.check_version_in_command(cmd)
-        except Exception as e:
-            raise_error(f"Failed to check Julia version: {e}")
-        try:
-            subprocess.check_call(
-                cmd, env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"}
-            )
-        except subprocess.CalledProcessError:
-            raise_error("Failed to check julia environment")
+        _check_julia_env(
+            env_path=env_path,
+            julia_version=julia_version,
+            verbose=verbose,
+            cache_key=env_name,
+        )
     else:
         raise_error(f"Environment kind '{env['kind']}' not supported")
     return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)
 
 
+@check_app.command(
+    name="julia-env",
+    help=(
+        "Check a Julia environment and instantiate only when project, "
+        "manifest, and package cache state have changed."
+    ),
+)
+def check_julia_env(
+    env_path: Annotated[
+        str,
+        typer.Argument(help="Path to Julia Project.toml file."),
+    ] = "Project.toml",
+    julia_version: Annotated[
+        str | None,
+        typer.Option(
+            "--julia",
+            help="Julia version to enforce (e.g., 1.11).",
+        ),
+    ] = None,
+    verbose: Annotated[
+        bool, typer.Option("--verbose", help="Print verbose output.")
+    ] = False,
+) -> str:
+    return _check_julia_env(
+        env_path=env_path,
+        julia_version=julia_version,
+        verbose=verbose,
+    )
+
+
 @check_app.command(
     name="envs",
     help="Check that all environments are up-to-date.",
@@ -734,15 +876,26 @@ def check_docker_env(
                     typer.echo(f"Found modified dependency: {dep}")
                     rebuild_or_pull = True
                     break
+
+    def delete_lock_on_failure() -> None:
+        if lock_fpath and os.path.exists(lock_fpath):
+            os.remove(lock_fpath)
+
     if fpath is not None and rebuild_or_pull:
-        wdir, fname = os.path.split(fpath)
-        if not wdir:
-            wdir = None
-        cmd = ["docker", "build", "-t", tag, "-f", fname]
+        dockerfile_dir, dockerfile_name = os.path.split(fpath)
+        if not dockerfile_dir:
+            dockerfile_dir = None
+        cmd = ["docker", "build", "-t", tag, "-f", dockerfile_name]
         if platform is not None:
             cmd += ["--platform", platform]
         cmd.append(".")
-        subprocess.check_output(cmd, cwd=wdir)
+        try:
+            subprocess.check_output(cmd, cwd=dockerfile_dir)
+        except subprocess.CalledProcessError:
+            delete_lock_on_failure()
+            raise_error(
+                f"Failed to build Docker image with tag {tag} from {fpath}"
+            )
     elif fpath is None and rebuild_or_pull:
         # First try to pull by repo digest
         pulled = False
@@ -759,6 +912,7 @@ def check_docker_env(
                     subprocess.check_output(tag_cmd)
                     pulled = True
                 except subprocess.CalledProcessError:
+                    delete_lock_on_failure()
                     warn(
                         f"Failed to pull image by digest: {image_with_digest}; "
                         "falling back to pulling by tag"
@@ -770,6 +924,7 @@ def check_docker_env(
             try:
                 subprocess.check_output(cmd)
             except subprocess.CalledProcessError:
+                delete_lock_on_failure()
                 raise_error(f"Failed to pull image: {tag}")
     # Write the lock file
     inspect = get_docker_inspect()
@@ -851,7 +1006,7 @@ def check_conda_env(
     try:
         calkit.conda.check_env(
             env_fpath=env_fpath,
-            output_fpath=output_fpath,
+            lock_fpath=output_fpath,
             alt_lock_fpaths=alt_lock_fpaths,
             alt_lock_fpaths_delete=alt_lock_fpaths_delete,
             log_func=log_func,

calkit/cli/main/core.py

@@ -958,10 +958,9 @@ def push(
 ):
     """Push with both Git and DVC."""
     if not no_dvc:
-        typer.echo("Pushing to DVC remote")
+        remotes = calkit.dvc.get_remotes()
         if not no_check_auth:
             # Check that our dvc remotes all have our DVC token set for them
-            remotes = calkit.dvc.get_remotes()
             ck_remote_name = calkit.config.get_app_name()
             for name, url in remotes.items():
                 if (
@@ -972,9 +971,13 @@ def push(
                         f"Checking authentication for DVC remote: {name}"
                     )
                     calkit.dvc.set_remote_auth(remote_name=name)
-        result = run_dvc_command(["push"] + dvc_args)
-        if result != 0:
-            raise_error("DVC push failed")
+        if remotes:
+            typer.echo("Pushing to DVC remote")
+            result = run_dvc_command(["push"] + dvc_args)
+            if result != 0:
+                raise_error("DVC push failed")
+        else:
+            warn("No DVC remotes configured; skipping DVC push")
     if not no_git:
         typer.echo("Pushing to Git remote")
         try:
@@ -1910,6 +1913,9 @@ def run_in_env(
             julia_cmd = calkit.julia.check_version_in_command(julia_cmd)
         except Exception as e:
             raise_error(f"Failed to check Julia version: {e}")
+        julia_cmd = calkit.julia.ensure_startup_file_disabled_in_command(
+            julia_cmd
+        )
         if verbose:
             typer.echo(f"Running command: {julia_cmd}")
         try:

calkit/cli/new.py

@@ -1204,10 +1204,8 @@ def new_conda_env(
             "(use -f to overwrite)"
         )
     if conda_name is None:
-        project_name = ck_info.get("name")
-        if project_name is None:
-            project_name = os.path.basename(os.getcwd())
-        conda_name = calkit.to_kebab_case(project_name) + "-" + name
+        project_name = calkit.detect_project_name(prepend_owner=False)
+        conda_name = calkit.to_kebab_case(project_name) + "." + name
     if packages is not None:
         assert isinstance(packages, list)
         # Write environment to path
@@ -1351,6 +1349,17 @@ def new_slurm_env(
             ),
         ),
     ] = [],
+    default_setup: Annotated[
+        list[str],
+        typer.Option(
+            "--default-setup",
+            help=(
+                "Default shell setup command to run before SLURM jobs "
+                "(for example 'module load julia/1.11'). Repeat for "
+                "multiple commands."
+            ),
+        ),
+    ] = [],
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
@@ -1383,9 +1392,14 @@ def new_slurm_env(
     normalized_default_options = [
         opt.strip() for opt in default_options if opt.strip()
     ]
+    normalized_default_setup = [
+        cmd.strip() for cmd in default_setup if cmd.strip()
+    ]
     env = {"kind": "slurm", "host": host}
     if normalized_default_options:
         env["default_options"] = normalized_default_options  # type: ignore
+    if normalized_default_setup:
+        env["default_setup"] = normalized_default_setup  # type: ignore
     if description is not None:
         env["description"] = description
     envs[name] = env

calkit/cli/notebooks.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import base64
 import json
 import os
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -109,6 +110,11 @@ def _check_ijulia_available(
     except Exception as e:
         raise_error(f"Failed to check Julia version: {e}")
         return False
+    ijulia_check_cmd_checked = (
+        calkit.julia.ensure_startup_file_disabled_in_command(
+            ijulia_check_cmd_checked
+        )
+    )
     res = subprocess.run(
         ijulia_check_cmd_checked,
         capture_output=True,
@@ -117,6 +123,17 @@ def _check_ijulia_available(
     return res.returncode == 0
 
 
+def _sanitize_kernel_name_component(name: str, label: str) -> str:
+    """Keep readable names while removing kernelspec-unfriendly characters."""
+    sanitized = re.sub(r"[^A-Za-z0-9._-]+", "-", name).strip("._-")
+    if not sanitized:
+        raise_error(
+            f"{label} cannot be empty after sanitizing for kernel name"
+        )
+        return ""  # For typing analysis since raise_error exits
+    return sanitized
+
+
 @notebooks_app.command("check-kernel")
 def check_env_kernel(
     env_name: Annotated[
@@ -186,8 +203,14 @@ def check_env_kernel(
     project_name = calkit.detect_project_name(prepend_owner=False)
     if not project_name:
         raise_error("Project name cannot be empty")
-    kernel_name = calkit.to_kebab_case(f"{project_name}-{env_name}")
+    kernel_name = (
+        f"{_sanitize_kernel_name_component(project_name, 'Project name')}"
+        f".{_sanitize_kernel_name_component(env_name, 'Environment name')}"
+    )
     display_name = f"{project_name}: {env_name}"
+    if verbose and not json_output:
+        typer.echo(f"Using kernel name: {kernel_name}")
+        typer.echo(f"Using display name: {display_name}")
     if language == "python":
         cmd = [
             "python",
@@ -290,8 +313,10 @@ def check_env_kernel(
         julia_cmd = (
             "import IJulia;"
             "kp=IJulia.installkernel("
-            f'"{display_name}",'
+            f'"{kernel_name}",'
             f'"--project={env_dir_abs}",'
+            '"--startup-file=no",'
+            f'displayname="{display_name}",'
             'env=Dict("JULIA_LOAD_PATH" => "@:@stdlib")'
             ");"
             "println(kp);"
@@ -307,9 +332,10 @@ def check_env_kernel(
             cmd = calkit.julia.check_version_in_command(cmd)
         except Exception as e:
             raise_error(f"Failed to check Julia version: {e}")
+        cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
         res = subprocess.run(cmd, capture_output=True, text=True)
         if res.returncode != 0:
-            raise_error(f"Failed to create kernel:\n{res.stdout}")
+            raise_error(f"Failed to create kernel:\n{res.stderr}")
         kernel_path = res.stdout.strip()
         kernel_name = os.path.basename(kernel_path)
         # Update display_name to include version for matching in VS Code

calkit/cli/slurm.py

@@ -81,6 +81,16 @@ def run_sbatch(
             help="Additional options to pass to sbatch (no spaces allowed).",
         ),
     ] = [],
+    setup_cmds: Annotated[
+        list[str],
+        typer.Option(
+            "--setup",
+            help=(
+                "Shell setup command to run before launching the target "
+                "(repeat for multiple commands)."
+            ),
+        ),
+    ] = [],
     log_path: Annotated[
         str | None, typer.Option("--log-path", help="Output log path.")
     ] = None,
@@ -97,7 +107,7 @@ def run_sbatch(
     Duplicates are not allowed, so if one is already running or queued with
     the same name, we'll wait for it to finish. The only exception is if the
     dependencies have changed, in which case any queued or running jobs will
-    be cancelled and a new one submitted.
+    be canceled and a new one submitted.
     """
 
     def check_job_running_or_queued(job_id: str) -> bool:
@@ -132,13 +142,6 @@ def run_sbatch(
     ] + sbatch_opts
     if is_command is None:
         is_command = not os.path.isfile(target)
-    if is_command:
-        # Use shlex.join to properly escape target and args for shell execution
-        cmd += ["--wrap", shlex.join([target] + args)]
-    else:
-        cmd += [target] + args
-        if target not in deps:
-            deps = [target] + deps
     if environment != "_system":
         ck_info = calkit.load_calkit_info()
         env = ck_info.get("environments", {}).get(environment, {})
@@ -163,6 +166,26 @@ def run_sbatch(
                     f"Environment '{environment}' is for host '{env_host}', but "
                     f"this is '{current_host}'"
                 )
+        env_setup_cmds = env.get("default_setup", [])
+        if env_setup_cmds:
+            setup_cmds = [
+                s for s in [*env_setup_cmds, *setup_cmds] if s.strip()
+            ]
+    if setup_cmds:
+        # Run setup and target in the same shell so setup side effects (like
+        # module loads) persist for the target command.
+        wrapped_target = shlex.join([target] + args)
+        setup_chain = " && ".join([*setup_cmds, wrapped_target])
+        cmd += ["--wrap", setup_chain]
+        if not is_command and target not in deps:
+            deps = [target] + deps
+    elif is_command:
+        # Use shlex.join to properly escape target and args for shell execution
+        cmd += ["--wrap", shlex.join([target] + args)]
+    else:
+        cmd += [target] + args
+        if target not in deps:
+            deps = [target] + deps
     slurm_dir = os.path.join(".calkit", "slurm")
     os.makedirs(slurm_dir, exist_ok=True)
     logs_dir = os.path.dirname(log_path)
@@ -187,6 +210,7 @@ def run_sbatch(
         job_deps = job_info["deps"]
         job_target = job_info.get("target")
         job_args = job_info.get("args", [])
+        job_setup = job_info.get("setup", [])
         running_or_queued = check_job_running_or_queued(job_id)
         should_wait = True
         if running_or_queued:
@@ -207,6 +231,13 @@ def run_sbatch(
                     job_id=job_id,
                     reason=f"Arguments for job '{name}' have changed",
                 )
+            # Check if setup commands have changed
+            if job_setup != setup_cmds:
+                should_wait = False
+                cancel_job(
+                    job_id=job_id,
+                    reason=f"Setup commands for job '{name}' have changed",
+                )
             # Check if dependency paths have changed
             if set(job_deps) != set(deps):
                 should_wait = False
@@ -258,6 +289,7 @@ def run_sbatch(
         "deps": deps,
         "target": target,
         "args": args,
+        "setup": setup_cmds,
         "dep_md5s": current_dep_md5s,
     }
     jobs[name] = new_job
@@ -336,7 +368,7 @@ def cancel_jobs(
         )
         if p.returncode != 0:
             raise_error(f"Failed to cancel job ID {job_id}: {p.stderr}")
-        typer.echo(f"Cancelled job '{name}' with ID {job_id}")
+        typer.echo(f"Canceled job '{name}' with ID {job_id}")
 
 
 @slurm_app.command(name="logs")