calkit-python 0.37.2__py3-none-any.whl → 0.37.3__py3-none-any.whl

calkit/cli/check.py

@@ -31,6 +31,217 @@ from calkit.environments import (
 check_app = typer.Typer(no_args_is_help=True)
 
 
+def _juliaup_version_installed(julia_version: str) -> bool:
+    """Return True if juliaup already has the given channel installed locally."""
+    try:
+        result = subprocess.run(
+            ["juliaup", "status"],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=10,
+        )
+    except FileNotFoundError:
+        return False
+    except subprocess.TimeoutExpired:
+        warn("Timed out while running `juliaup status`.")
+        return False
+    if result.returncode != 0:
+        err_output = (result.stderr or result.stdout or "").strip()
+        if err_output:
+            warn(f"`juliaup status` failed: {err_output}")
+        else:
+            warn(
+                f"`juliaup status` failed with exit code {result.returncode}."
+            )
+        return False
+    for line in result.stdout.splitlines():
+        parts = line.split()
+        if not parts:
+            continue
+        # Columns: [Default?] Channel Version [Update?]
+        # Default column is either empty or "*", so channel is parts[0] or parts[1]
+        for col in parts[:2]:
+            if col == julia_version:
+                return True
+    return False
+
+
+def _check_julia_env(
+    env_path: str,
+    julia_version: str | None = None,
+    verbose: bool = False,
+    cache_key: str | None = None,
+) -> str:
+    """Check a Julia environment and instantiate only when needed."""
+    abs_env_path = os.path.abspath(env_path)
+    env_fname = os.path.basename(abs_env_path)
+    if env_fname != "Project.toml":
+        raise_error(
+            "Julia environments require a path pointing to Project.toml"
+        )
+    env_dir = os.path.dirname(abs_env_path) or "."
+    env_path_for_cache = os.path.basename(abs_env_path)
+    env = {
+        "kind": "julia",
+        "path": env_path_for_cache,
+        "julia": julia_version or "",
+    }
+    cache_env_name = cache_key or (
+        f"julia::{abs_env_path}::{julia_version or ''}"
+    )
+    if calkit.environments.check_cache(
+        env_name=cache_env_name,
+        env=env,
+        wdir=env_dir,
+        respect_ttl=False,
+    ):
+        if verbose:
+            typer.echo(
+                "Julia environment cache is valid; skipping Pkg.instantiate()"
+            )
+        lock_fpath = calkit.environments.get_env_lock_fpath(
+            env=env,
+            env_name=cache_env_name,
+            wdir=env_dir,
+            as_posix=False,
+        )
+        return lock_fpath or os.path.join(env_dir, "Manifest.toml")
+    if julia_version:
+        if shutil.which("juliaup") is not None:
+            if _juliaup_version_installed(julia_version):
+                if verbose:
+                    typer.echo(
+                        f"Julia {julia_version} is already installed; "
+                        "skipping juliaup add"
+                    )
+            else:
+                cmd = ["juliaup", "add", julia_version]
+                if verbose:
+                    typer.echo(f"Running command: {cmd}")
+                try:
+                    subprocess.run(cmd, check=True)
+                except subprocess.CalledProcessError:
+                    raise_error(
+                        f"Failed to install Julia version {julia_version}"
+                    )
+        else:
+            try:
+                compatible = calkit.julia.current_version_is_compatible(
+                    julia_version
+                )
+            except ValueError as e:
+                raise_error(str(e))
+            if not compatible:
+                raise_error(
+                    "Current Julia version is not compatible with required "
+                    f"version ({julia_version}), and juliaup is not "
+                    "available to install it"
+                )
+    deps_to_add: list[str] = []
+    try:
+        with open(abs_env_path, "r") as f:
+            content = f.read()
+        lines = [line.rstrip() for line in content.splitlines()]
+        deps_section = False
+        deps_found = False
+        for line in lines:
+            stripped = line.strip()
+            if stripped == "[deps]":
+                deps_section = True
+                continue
+            if deps_section:
+                if stripped.startswith("[") and stripped.endswith("]"):
+                    break
+                if stripped and not stripped.startswith("#"):
+                    if "=" in stripped:
+                        deps_found = True
+                        break
+        if not deps_found:
+            for idx, line in enumerate(lines):
+                marker = "# Dependencies (add with Julia's Pkg.add):"
+                if line.strip() == marker and idx + 1 < len(lines):
+                    dep_line = lines[idx + 1].strip()
+                    if dep_line.startswith("#"):
+                        dep_line = dep_line.lstrip("#").strip()
+                    deps_to_add = [
+                        dep.strip()
+                        for dep in dep_line.split(",")
+                        if dep.strip()
+                    ]
+                    break
+    except OSError:
+        deps_to_add = []
+    if deps_to_add:
+        pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
+        cmd = ["julia"]
+        if julia_version:
+            cmd.append(f"+{julia_version}")
+        cmd += [
+            f"--project={env_dir}",
+            "-e",
+            f"using Pkg; Pkg.add([{pkg_list}]);",
+        ]
+        if julia_version:
+            try:
+                cmd = calkit.julia.check_version_in_command(cmd)
+            except Exception as e:
+                raise_error(f"Failed to check Julia version: {e}")
+        cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
+        try:
+            subprocess.check_call(
+                cmd,
+                env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
+            )
+        except subprocess.CalledProcessError:
+            calkit.environments.save_cache(
+                env_name=cache_env_name,
+                env=env,
+                wdir=env_dir,
+                success=False,
+            )
+            raise_error("Failed to add Julia dependencies")
+    cmd = ["julia"]
+    if julia_version:
+        cmd.append(f"+{julia_version}")
+    cmd += [
+        f"--project={env_dir}",
+        "-e",
+        "using Pkg; Pkg.instantiate();",
+    ]
+    if julia_version:
+        try:
+            cmd = calkit.julia.check_version_in_command(cmd)
+        except Exception as e:
+            raise_error(f"Failed to check Julia version: {e}")
+    cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
+    try:
+        subprocess.check_call(
+            cmd, env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"}
+        )
+    except subprocess.CalledProcessError:
+        calkit.environments.save_cache(
+            env_name=cache_env_name,
+            env=env,
+            wdir=env_dir,
+            success=False,
+        )
+        raise_error("Failed to check julia environment")
+    calkit.environments.save_cache(
+        env_name=cache_env_name,
+        env=env,
+        wdir=env_dir,
+        success=True,
+    )
+    lock_fpath = calkit.environments.get_env_lock_fpath(
+        env=env,
+        env_name=cache_env_name,
+        wdir=env_dir,
+        as_posix=False,
+    )
+    return lock_fpath or os.path.join(env_dir, "Manifest.toml")
+
+
 @check_app.command(name="repro")
 def check_repro(
     wdir: Annotated[
@@ -190,116 +401,47 @@ def check_environment(
         julia_version = env.get("julia")
         if julia_version is None:
             raise_error("Julia environments require a Julia version")
-        env_fname = os.path.basename(env_path)
-        if not env_fname == "Project.toml":
-            raise_error(
-                "Julia environments require a path pointing to Project.toml"
-            )
-        # First ensure the Julia version exists
-        if shutil.which("juliaup") is not None:
-            cmd = ["juliaup", "add", julia_version]
-            if verbose:
-                typer.echo(f"Running command: {cmd}")
-            try:
-                subprocess.run(cmd, check=True)
-            except subprocess.CalledProcessError:
-                raise_error(f"Failed to install Julia version {julia_version}")
-        else:
-            try:
-                compatible = calkit.julia.current_version_is_compatible(
-                    julia_version
-                )
-            except ValueError as e:
-                raise_error(str(e))
-            if not compatible:
-                raise_error(
-                    f"Current Julia version is not compatible with required "
-                    f"version ({julia_version}), and juliaup is not available to "
-                    "install it"
-                )
-        env_dir = os.path.dirname(env_path)
-        if not env_dir:
-            env_dir = "."
-        # If auto-detection couldn't resolve UUIDs, Project.toml includes a
-        # commented dependency list
-        # In that case, add those packages with
-        # Pkg.add before instantiating so the env is usable at run time
-        deps_to_add: list[str] = []
-        try:
-            with open(env_path, "r") as f:
-                content = f.read()
-            lines = [line.rstrip() for line in content.splitlines()]
-            deps_section = False
-            deps_found = False
-            for line in lines:
-                stripped = line.strip()
-                if stripped == "[deps]":
-                    deps_section = True
-                    continue
-                if deps_section:
-                    if stripped.startswith("[") and stripped.endswith("]"):
-                        break
-                    if stripped and not stripped.startswith("#"):
-                        if "=" in stripped:
-                            deps_found = True
-                            break
-            if not deps_found:
-                for idx, line in enumerate(lines):
-                    marker = "# Dependencies (add with Julia's Pkg.add):"
-                    if line.strip() == marker and idx + 1 < len(lines):
-                        dep_line = lines[idx + 1].strip()
-                        if dep_line.startswith("#"):
-                            dep_line = dep_line.lstrip("#").strip()
-                        deps_to_add = [
-                            dep.strip()
-                            for dep in dep_line.split(",")
-                            if dep.strip()
-                        ]
-                        break
-        except OSError:
-            deps_to_add = []
-        if deps_to_add:
-            pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
-            cmd = [
-                "julia",
-                f"+{julia_version}",
-                f"--project={env_dir}",
-                "-e",
-                f"using Pkg; Pkg.add([{pkg_list}]);",
-            ]
-            try:
-                cmd = calkit.julia.check_version_in_command(cmd)
-            except Exception as e:
-                raise_error(f"Failed to check Julia version: {e}")
-            try:
-                subprocess.check_call(
-                    cmd,
-                    env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
-                )
-            except subprocess.CalledProcessError:
-                raise_error("Failed to add Julia dependencies")
-        cmd = [
-            "julia",
-            f"+{julia_version}",
-            f"--project={env_dir}",
-            "-e",
-            "using Pkg; Pkg.instantiate();",
-        ]
-        try:
-            cmd = calkit.julia.check_version_in_command(cmd)
-        except Exception as e:
-            raise_error(f"Failed to check Julia version: {e}")
-        try:
-            subprocess.check_call(
-                cmd, env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"}
-            )
-        except subprocess.CalledProcessError:
-            raise_error("Failed to check julia environment")
+        _check_julia_env(
+            env_path=env_path,
+            julia_version=julia_version,
+            verbose=verbose,
+            cache_key=env_name,
+        )
     else:
         raise_error(f"Environment kind '{env['kind']}' not supported")
     return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)
 
 
+@check_app.command(
+    name="julia-env",
+    help=(
+        "Check a Julia environment and instantiate only when project, "
+        "manifest, and package cache state have changed."
+    ),
+)
+def check_julia_env(
+    env_path: Annotated[
+        str,
+        typer.Argument(help="Path to Julia Project.toml file."),
+    ] = "Project.toml",
+    julia_version: Annotated[
+        str | None,
+        typer.Option(
+            "--julia",
+            help="Julia version to enforce (e.g., 1.11).",
+        ),
+    ] = None,
+    verbose: Annotated[
+        bool, typer.Option("--verbose", help="Print verbose output.")
+    ] = False,
+) -> str:
+    return _check_julia_env(
+        env_path=env_path,
+        julia_version=julia_version,
+        verbose=verbose,
+    )
+
+
 @check_app.command(
     name="envs",
     help="Check that all environments are up-to-date.",

calkit/cli/main/core.py

@@ -1910,6 +1910,9 @@ def run_in_env(
             julia_cmd = calkit.julia.check_version_in_command(julia_cmd)
         except Exception as e:
             raise_error(f"Failed to check Julia version: {e}")
+        julia_cmd = calkit.julia.ensure_startup_file_disabled_in_command(
+            julia_cmd
+        )
         if verbose:
             typer.echo(f"Running command: {julia_cmd}")
         try:

calkit/cli/new.py

@@ -1351,6 +1351,17 @@ def new_slurm_env(
             ),
         ),
     ] = [],
+    default_setup: Annotated[
+        list[str],
+        typer.Option(
+            "--default-setup",
+            help=(
+                "Default shell setup command to run before SLURM jobs "
+                "(for example 'module load julia/1.11'). Repeat for "
+                "multiple commands."
+            ),
+        ),
+    ] = [],
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
@@ -1383,9 +1394,14 @@ def new_slurm_env(
     normalized_default_options = [
         opt.strip() for opt in default_options if opt.strip()
     ]
+    normalized_default_setup = [
+        cmd.strip() for cmd in default_setup if cmd.strip()
+    ]
     env = {"kind": "slurm", "host": host}
     if normalized_default_options:
         env["default_options"] = normalized_default_options  # type: ignore
+    if normalized_default_setup:
+        env["default_setup"] = normalized_default_setup  # type: ignore
     if description is not None:
         env["description"] = description
     envs[name] = env

calkit/cli/notebooks.py

@@ -109,6 +109,11 @@ def _check_ijulia_available(
     except Exception as e:
         raise_error(f"Failed to check Julia version: {e}")
         return False
+    ijulia_check_cmd_checked = (
+        calkit.julia.ensure_startup_file_disabled_in_command(
+            ijulia_check_cmd_checked
+        )
+    )
     res = subprocess.run(
         ijulia_check_cmd_checked,
         capture_output=True,
@@ -292,6 +297,7 @@ def check_env_kernel(
             "kp=IJulia.installkernel("
             f'"{display_name}",'
             f'"--project={env_dir_abs}",'
+            '"--startup-file=no",'
             'env=Dict("JULIA_LOAD_PATH" => "@:@stdlib")'
             ");"
             "println(kp);"
@@ -307,6 +313,7 @@ def check_env_kernel(
             cmd = calkit.julia.check_version_in_command(cmd)
         except Exception as e:
             raise_error(f"Failed to check Julia version: {e}")
+        cmd = calkit.julia.ensure_startup_file_disabled_in_command(cmd)
         res = subprocess.run(cmd, capture_output=True, text=True)
         if res.returncode != 0:
             raise_error(f"Failed to create kernel:\n{res.stdout}")

calkit/cli/slurm.py

@@ -81,6 +81,16 @@ def run_sbatch(
             help="Additional options to pass to sbatch (no spaces allowed).",
         ),
     ] = [],
+    setup_cmds: Annotated[
+        list[str],
+        typer.Option(
+            "--setup",
+            help=(
+                "Shell setup command to run before launching the target "
+                "(repeat for multiple commands)."
+            ),
+        ),
+    ] = [],
     log_path: Annotated[
         str | None, typer.Option("--log-path", help="Output log path.")
     ] = None,
@@ -97,7 +107,7 @@ def run_sbatch(
     Duplicates are not allowed, so if one is already running or queued with
     the same name, we'll wait for it to finish. The only exception is if the
     dependencies have changed, in which case any queued or running jobs will
-    be cancelled and a new one submitted.
+    be canceled and a new one submitted.
     """
 
     def check_job_running_or_queued(job_id: str) -> bool:
@@ -132,13 +142,6 @@ def run_sbatch(
     ] + sbatch_opts
     if is_command is None:
         is_command = not os.path.isfile(target)
-    if is_command:
-        # Use shlex.join to properly escape target and args for shell execution
-        cmd += ["--wrap", shlex.join([target] + args)]
-    else:
-        cmd += [target] + args
-        if target not in deps:
-            deps = [target] + deps
     if environment != "_system":
         ck_info = calkit.load_calkit_info()
         env = ck_info.get("environments", {}).get(environment, {})
@@ -163,6 +166,26 @@ def run_sbatch(
                     f"Environment '{environment}' is for host '{env_host}', but "
                     f"this is '{current_host}'"
                 )
+        env_setup_cmds = env.get("default_setup", [])
+        if env_setup_cmds:
+            setup_cmds = [
+                s for s in [*env_setup_cmds, *setup_cmds] if s.strip()
+            ]
+    if setup_cmds:
+        # Run setup and target in the same shell so setup side effects (like
+        # module loads) persist for the target command.
+        wrapped_target = shlex.join([target] + args)
+        setup_chain = " && ".join([*setup_cmds, wrapped_target])
+        cmd += ["--wrap", setup_chain]
+        if not is_command and target not in deps:
+            deps = [target] + deps
+    elif is_command:
+        # Use shlex.join to properly escape target and args for shell execution
+        cmd += ["--wrap", shlex.join([target] + args)]
+    else:
+        cmd += [target] + args
+        if target not in deps:
+            deps = [target] + deps
     slurm_dir = os.path.join(".calkit", "slurm")
     os.makedirs(slurm_dir, exist_ok=True)
     logs_dir = os.path.dirname(log_path)
@@ -187,6 +210,7 @@ def run_sbatch(
         job_deps = job_info["deps"]
         job_target = job_info.get("target")
         job_args = job_info.get("args", [])
+        job_setup = job_info.get("setup", [])
         running_or_queued = check_job_running_or_queued(job_id)
         should_wait = True
         if running_or_queued:
@@ -207,6 +231,13 @@ def run_sbatch(
                     job_id=job_id,
                     reason=f"Arguments for job '{name}' have changed",
                 )
+            # Check if setup commands have changed
+            if job_setup != setup_cmds:
+                should_wait = False
+                cancel_job(
+                    job_id=job_id,
+                    reason=f"Setup commands for job '{name}' have changed",
+                )
             # Check if dependency paths have changed
             if set(job_deps) != set(deps):
                 should_wait = False
@@ -258,6 +289,7 @@ def run_sbatch(
         "deps": deps,
         "target": target,
         "args": args,
+        "setup": setup_cmds,
         "dep_md5s": current_dep_md5s,
     }
     jobs[name] = new_job
@@ -336,7 +368,7 @@ def cancel_jobs(
         )
         if p.returncode != 0:
             raise_error(f"Failed to cancel job ID {job_id}: {p.stderr}")
-        typer.echo(f"Cancelled job '{name}' with ID {job_id}")
+        typer.echo(f"Canceled job '{name}' with ID {job_id}")
 
 
 @slurm_app.command(name="logs")

calkit/dvc/zip.py

@@ -185,17 +185,23 @@ def hash_path(path: str, alg="md5") -> str:
 def calc_dir_sig(path: str) -> str:
     """Calculate a fast signature for a directory to know if we should
     rehash.
+
+    The signature includes file count, total size, and latest mtime in
+    nanoseconds over all files in the directory tree.
     """
     if not os.path.isdir(path):
         return ""
     file_count = 0
     total_size = 0
     latest_mtime = 0
-    for foldername, subfolders, filenames in os.walk(path):
+    for foldername, _, filenames in os.walk(path):
         for filename in filenames:
-            file_count += 1
             fpath = os.path.join(foldername, filename)
-            st = os.stat(fpath)
+            try:
+                st = os.stat(fpath)
+            except OSError:
+                continue
+            file_count += 1
             total_size += st.st_size
             if st.st_mtime_ns > latest_mtime:
                 latest_mtime = st.st_mtime_ns