calkit-python 0.41.9__py3-none-any.whl → 0.41.11__py3-none-any.whl

calkit/cli/check.py

@@ -163,7 +163,7 @@ def _check_julia_env(
         deps_to_add = []
     if deps_to_add:
         pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
-        cmd = ["julia"]
+        cmd = [calkit.julia.get_julia_exe()]
         if julia_version:
             cmd.append(f"+{julia_version}")
         cmd += [
@@ -190,7 +190,7 @@ def _check_julia_env(
                 success=False,
             )
             raise_error("Failed to add Julia dependencies")
-    cmd = ["julia"]
+    cmd = [calkit.julia.get_julia_exe()]
     if julia_version:
         cmd.append(f"+{julia_version}")
     cmd += [
@@ -318,6 +318,7 @@ def check_environment(
         get_all_conda_lock_fpaths,
         get_all_docker_lock_fpaths,
         get_all_venv_lock_fpaths,
+        get_default_venv_prefix,
         get_env_lock_fpath,
         write_scheduler_env_lock,
     )
@@ -403,12 +404,14 @@ def check_environment(
         except subprocess.CalledProcessError:
             raise_error("Failed to check uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
-        if "prefix" not in env:
-            raise_error("venv environments require a prefix")
         if "path" not in env:
             raise_error("venv environments require a path")
-        prefix = os.path.expandvars(env["prefix"])
         path = os.path.expandvars(env["path"])
+        # Resolve the prefix on the fly if it isn't pinned in calkit.yaml
+        prefix = env.get("prefix")
+        if prefix is None:
+            prefix = get_default_venv_prefix(envs, path, env_name)
+        prefix = os.path.expandvars(prefix)
         lock_fpath = get_env_lock_fpath(
             env=env, env_name=env_name, as_posix=False
         )
@@ -543,6 +546,41 @@ def check_environments(
         )
 
 
+def _renv_snapshot_from_description(env_dir: str, verbose: bool) -> None:
+    """Install packages from DESCRIPTION and (re)write renv.lock.
+
+    Used to create the initial lock, to add newly declared packages, and as a
+    fallback when an existing lock can't be restored (e.g. its versions don't
+    build against the installed R version).
+    """
+    hydrate_cmd = [
+        "Rscript",
+        "--vanilla",
+        "-e",
+        "renv::load(); renv::hydrate()",
+    ]
+    if verbose:
+        typer.echo(f"Running: {' '.join(hydrate_cmd)}")
+    try:
+        subprocess.check_call(hydrate_cmd, cwd=env_dir)
+    except subprocess.CalledProcessError:
+        # Hydrate may fail if some packages aren't available; snapshot anyway
+        if verbose:
+            typer.echo("Warning: hydrate had issues, continuing to snapshot")
+    snapshot_cmd = [
+        "Rscript",
+        "--vanilla",
+        "-e",
+        "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
+    ]
+    if verbose:
+        typer.echo(f"Running: {' '.join(snapshot_cmd)}")
+    try:
+        subprocess.check_call(snapshot_cmd, cwd=env_dir)
+    except subprocess.CalledProcessError:
+        raise_error(f"Failed to snapshot renv in {env_dir}")
+
+
 @check_app.command(name="renv")
 def check_renv(
     env_path: Annotated[
@@ -606,153 +644,74 @@ def check_renv(
             subprocess.check_call(init_cmd, cwd=env_dir)
         except subprocess.CalledProcessError:
             raise_error(f"Failed to initialize renv in {env_dir}")
-        # Use hydrate to install packages from DESCRIPTION and snapshot
+        # Install packages from DESCRIPTION and write the lock file
         if verbose:
             typer.echo("Setting up environment from DESCRIPTION")
-        hydrate_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); renv::hydrate()",
-        ]
-        if verbose:
-            typer.echo(f"Running: {' '.join(hydrate_cmd)}")
-        try:
-            subprocess.check_call(hydrate_cmd, cwd=env_dir)
-        except subprocess.CalledProcessError:
-            # Hydrate might fail if packages aren't available, continue anyway
-            if verbose:
-                typer.echo(
-                    "Warning: hydrate had issues, continuing to snapshot"
-                )
-        # Always snapshot after hydrate to create lock file from DESCRIPTION
+        _renv_snapshot_from_description(env_dir, verbose=verbose)
+    else:
+        # A lock file exists, so treat it as the source of truth: restore the
+        # library to the exact recorded versions rather than re-resolving from
+        # DESCRIPTION (which would bump packages and overwrite the lock).
         if verbose:
-            typer.echo("Creating lock file from DESCRIPTION")
-        snapshot_cmd = [
+            typer.echo("Restoring library from lockfile")
+        restore_cmd = [
             "Rscript",
             "--vanilla",
             "-e",
-            "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
+            "renv::load(); renv::restore(prompt=FALSE)",
         ]
         if verbose:
-            typer.echo(f"Running: {' '.join(snapshot_cmd)}")
+            typer.echo(f"Running: {' '.join(restore_cmd)}")
         try:
-            subprocess.check_call(snapshot_cmd, cwd=env_dir)
+            subprocess.check_call(restore_cmd, cwd=env_dir)
+            restored = True
         except subprocess.CalledProcessError:
-            raise_error(f"Failed to snapshot renv in {env_dir}")
-    else:
-        # Lock file exists, check if it's in sync with DESCRIPTION
-        if verbose:
-            typer.echo("Checking if lockfile is in sync with DESCRIPTION")
-        # Check status to see if lockfile needs updating
-        status_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); status <- renv::status(); cat(status$synchronized)",
-        ]
-        try:
-            result = subprocess.run(
-                status_cmd,
-                cwd=env_dir,
-                capture_output=True,
-                text=True,
-                check=True,
+            restored = False
+        if not restored:
+            # The locked versions couldn't be installed (e.g. they don't build
+            # against the installed R version). Fall back to re-resolving from
+            # DESCRIPTION, which updates renv.lock to a working set.
+            warn(
+                f"Could not restore renv environment in {env_dir} from "
+                "renv.lock; the locked versions may be incompatible with the "
+                "installed R version. Re-resolving from DESCRIPTION and "
+                "updating renv.lock."
             )
-            lockfile_synced = "TRUE" in result.stdout
-        except subprocess.CalledProcessError:
-            # If status fails, assume we need to update
-            lockfile_synced = False
-            if verbose:
-                typer.echo("Warning: status check failed, will update lock")
-        if not lockfile_synced:
+            _renv_snapshot_from_description(env_dir, verbose=verbose)
+        else:
+            # Only update the lock if DESCRIPTION declares dependencies the
+            # lock doesn't cover (e.g. the user added a package). After the
+            # restore the library matches the lock, so status is unsynchronized
+            # only when DESCRIPTION and the lock genuinely disagree---not merely
+            # because packages were missing on a fresh checkout.
             if verbose:
-                typer.echo("Lockfile out of sync, updating from DESCRIPTION")
-            # Use hydrate to update from DESCRIPTION
-            hydrate_cmd = [
+                typer.echo("Checking if DESCRIPTION matches lockfile")
+            status_cmd = [
                 "Rscript",
                 "--vanilla",
                 "-e",
-                "renv::load(); renv::hydrate()",
+                "renv::load(); cat(renv::status()$synchronized)",
             ]
-            if verbose:
-                typer.echo(f"Running: {' '.join(hydrate_cmd)}")
             try:
-                subprocess.check_call(hydrate_cmd, cwd=env_dir)
+                result = subprocess.run(
+                    status_cmd,
+                    cwd=env_dir,
+                    capture_output=True,
+                    text=True,
+                    check=True,
+                )
+                in_sync = "TRUE" in result.stdout
             except subprocess.CalledProcessError:
+                # If status fails, keep the lock rather than risk clobbering it
+                in_sync = True
                 if verbose:
-                    typer.echo(
-                        "Warning: hydrate had issues, continuing to snapshot"
-                    )
-            # Snapshot to update lock
-            snapshot_cmd = [
-                "Rscript",
-                "--vanilla",
-                "-e",
-                "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
-            ]
-            if verbose:
-                typer.echo(f"Running: {' '.join(snapshot_cmd)}")
-            try:
-                subprocess.check_call(snapshot_cmd, cwd=env_dir)
-            except subprocess.CalledProcessError:
+                    typer.echo("Warning: status check failed, keeping lock")
+            if not in_sync:
                 if verbose:
-                    typer.echo("Warning: snapshot failed, using existing lock")
-        else:
-            if verbose:
+                    typer.echo("DESCRIPTION changed; updating lockfile")
+                _renv_snapshot_from_description(env_dir, verbose=verbose)
+            elif verbose:
                 typer.echo("Lockfile is already in sync with DESCRIPTION")
-    # Check if library needs restoring
-    if verbose:
-        typer.echo("Checking if library is in sync with lockfile")
-    lib_status_cmd = [
-        "Rscript",
-        "--vanilla",
-        "-e",
-        (
-            "renv::load(); "
-            "status <- tryCatch({"
-            "  renv::status();"
-            "  cat('synchronized');"
-            "}, error = function(e) {"
-            "  cat('needs_restore');"
-            "})"
-        ),
-    ]
-    try:
-        result = subprocess.run(
-            lib_status_cmd,
-            cwd=env_dir,
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        needs_restore = "needs_restore" in result.stdout or (
-            "synchronized" not in result.stdout
-        )
-    except subprocess.CalledProcessError:
-        # If check fails, restore to be safe
-        needs_restore = True
-        if verbose:
-            typer.echo("Warning: library status check failed, will restore")
-
-    if needs_restore:
-        if verbose:
-            typer.echo("Restoring library from lockfile")
-        restore_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); renv::restore(prompt=FALSE)",
-        ]
-        if verbose:
-            typer.echo(f"Running: {' '.join(restore_cmd)}")
-        try:
-            subprocess.check_call(restore_cmd, cwd=env_dir)
-        except subprocess.CalledProcessError:
-            raise_error(f"Failed to restore renv in {env_dir}")
-    else:
-        if verbose:
-            typer.echo("Library is already in sync with lockfile")
 
 
 @check_app.command(name="docker-env")

calkit/cli/describe.py

@@ -3,10 +3,13 @@
 from __future__ import annotations
 
 import json
+from typing import Annotated
 
 import typer
 
 import calkit
+from calkit.cli import raise_error
+from calkit.environments import get_env_lock_fpath
 
 describe_app = typer.Typer(no_args_is_help=True)
 
@@ -16,3 +19,45 @@ def describe_system():
     """Describe the system."""
     system_info = calkit.get_system_info()
     typer.echo(json.dumps(system_info, indent=2))
+
+
+@describe_app.command(name="env")
+def describe_env(
+    name: Annotated[
+        str,
+        typer.Option("--name", "-n", help="Environment name."),
+    ],
+):
+    """Describe a single environment, including spec and lock file paths."""
+    ck_info = calkit.load_calkit_info()
+    envs: dict = ck_info.get("environments", {})
+    if name not in envs:
+        raise_error(f"Environment '{name}' not found.")
+    env = envs[name]
+    lock_path = get_env_lock_fpath(env=env, env_name=name)
+    result = {
+        "kind": env.get("kind"),
+        "spec_path": env.get("path"),
+        "lock_path": lock_path,
+        "prefix": env.get("prefix"),
+        "python": env.get("python"),
+    }
+    typer.echo(json.dumps(result, indent=2))
+
+
+@describe_app.command(name="envs")
+def describe_envs():
+    """Describe all environments, including spec and lock file paths."""
+    ck_info = calkit.load_calkit_info()
+    envs: dict = ck_info.get("environments", {})
+    result = {}
+    for env_name, env in envs.items():
+        lock_path = get_env_lock_fpath(env=env, env_name=env_name)
+        result[env_name] = {
+            "kind": env.get("kind"),
+            "spec_path": env.get("path"),
+            "lock_path": lock_path,
+            "prefix": env.get("prefix"),
+            "python": env.get("python"),
+        }
+    typer.echo(json.dumps(result, indent=2))

calkit/cli/list.py

@@ -7,7 +7,7 @@ from typing import Annotated, Literal
 import typer
 
 import calkit
-from calkit.cli import AliasGroup, warn
+from calkit.cli import AliasGroup, raise_error, warn
 
 list_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
@@ -167,12 +167,28 @@ def list_stages(
         list[str] | None,
         typer.Option("--kind", "-k", help="Filter stages by kind."),
     ] = None,
+    stale_only: Annotated[
+        bool, typer.Option("--stale", help="Show only stale stages.")
+    ] = False,
 ):
     """List pipeline stages."""
-    stages = calkit.load_calkit_info().get("pipeline", {}).get("stages", {})
+    ck_info = calkit.load_calkit_info()
+    stages = ck_info.get("pipeline", {}).get("stages", {})
+    # If we only want stale stages, we need to get the status first.
+    # This compiles the pipeline, cleans notebooks, and checks environments,
+    # all of which can affect whether a stage is stale, so we don't skip them.
+    if stale_only:
+        status = calkit.pipeline.get_status(ck_info=ck_info)
+        if status.errors:
+            raise_error(
+                "Failed to determine stale stages: " + "; ".join(status.errors)
+            )
+        stale_stage_names = status.stale_stage_names
     for name, stage in stages.items():
         if kinds is not None and stage.get("kind") not in kinds:
             continue
+        if stale_only and name not in stale_stage_names:
+            continue
         typer.echo(name)
 
 

calkit/cli/main/core.py

@@ -196,7 +196,12 @@ def init(
     repo = calkit.git.get_repo()
     repo.git.add(".dvc")
     repo.git.commit("-m", "Initialize DVC")
-    # TODO: Initialize `calkit.yaml`
+    # Create an empty calkit.yaml if one doesn't already exist
+    if not os.path.isfile("calkit.yaml"):
+        with open("calkit.yaml", "w"):
+            pass
+        repo.git.add("calkit.yaml")
+        repo.git.commit("-m", "Initialize Calkit")
     # TODO: Initialize `dvc.yaml`
     # TODO: Add a sane .gitignore file
     # TODO: Add a sane LICENSE file?
@@ -1762,6 +1767,17 @@ def run(
             "--no-push", help="Do not push to Git and DVC after saving."
         ),
     ] = False,
+    mock_scheduler: Annotated[
+        bool,
+        typer.Option(
+            "--mock-scheduler",
+            "-K",
+            help=(
+                "Run job-scheduler (SLURM/PBS) stages locally instead of "
+                "submitting them to a real scheduler."
+            ),
+        ),
+    ] = False,
 ) -> dict:
     """Check dependencies and run the pipeline."""
     import dvc.log
@@ -1779,6 +1795,10 @@ def run(
     if (target_inputs or target_outputs) and targets:
         raise_error("Cannot specify both targets and inputs")
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
+    # Mock the scheduler for this run (and any subprocesses) so SLURM/PBS
+    # stages execute locally; child processes inherit it via os.environ
+    if mock_scheduler:
+        os.environ[calkit.cli.scheduler.MOCK_ENV_VAR] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     ck_info = calkit.load_calkit_info()
     # Ensure Git is initialized so DVC can be used.
@@ -2463,12 +2483,15 @@ def run_in_env(
         except subprocess.CalledProcessError:
             raise_error("Failed to run in uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
-        if "prefix" not in env:
-            raise_error("venv environments require a prefix")
         if "path" not in env:
             raise_error("venv environments require a path")
-        prefix = env["prefix"]
         path = env["path"]
+        # Resolve the prefix on the fly if it isn't pinned in calkit.yaml
+        prefix = env.get("prefix")
+        if prefix is None:
+            prefix = calkit.environments.get_default_venv_prefix(
+                envs, path, env_name
+            )
         shell_cmd = _to_shell_cmd(cmd)
         if _platform.system() == "Windows":
             activate_cmd = f"{prefix}\\Scripts\\activate"
@@ -2525,7 +2548,7 @@ def run_in_env(
         # specifying the project
         cmd = [arg for arg in cmd if not arg.startswith("--project=")]
         julia_cmd = [
-            "julia",
+            calkit.julia.get_julia_exe(),
             f"+{julia_version}",
             "--project=" + env_dir,
         ] + cmd

calkit/cli/new.py

@@ -1604,8 +1604,15 @@ def new_uv_venv(
         str, typer.Option("--path", help="Path for requirements file.")
     ] = "requirements.txt",
     prefix: Annotated[
-        str, typer.Option("--prefix", help="Prefix for environment location.")
-    ] = ".venv",
+        str | None,
+        typer.Option(
+            "--prefix",
+            help=(
+                "Prefix for environment location (defaults to .venv, or "
+                ".calkit/envs/<name>/.venv if .venv is already taken)."
+            ),
+        ),
+    ] = None,
     python_version: Annotated[
         str | None, typer.Option("--python", "-p", help="Python version.")
     ] = None,
@@ -1654,12 +1661,13 @@ def new_uv_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Check prefixes
-    if not overwrite:
-        for env_name, env in envs.items():
-            if env.get("prefix") == prefix:
+    # Only pin a prefix if one was explicitly given; otherwise it is resolved
+    # on the fly (defaulting to .venv) at check/run time
+    if prefix is not None and not overwrite:
+        for other_name, other_env in envs.items():
+            if other_env.get("prefix") == prefix:
                 raise_error(
-                    f"Environment '{env_name}' already exists with "
+                    f"Environment '{other_name}' already exists with "
                     f"prefix '{prefix}'"
                 )
     if packages is not None:
@@ -1669,7 +1677,9 @@ def new_uv_venv(
         with open(path, "w") as f:
             f.write(packages_txt)
     typer.echo("Adding environment to calkit.yaml")
-    env = dict(path=path, kind="uv-venv", prefix=prefix)
+    env = dict(path=path, kind="uv-venv")
+    if prefix is not None:
+        env["prefix"] = prefix
     if python_version is not None:
         env["python"] = python_version
     if description is not None:
@@ -1703,8 +1713,15 @@ def new_venv(
         str, typer.Option("--path", help="Path for requirements file.")
     ] = "requirements.txt",
     prefix: Annotated[
-        str, typer.Option("--prefix", help="Prefix for environment location.")
-    ] = ".venv",
+        str | None,
+        typer.Option(
+            "--prefix",
+            help=(
+                "Prefix for environment location (defaults to .venv, or "
+                ".calkit/envs/<name>/.venv if .venv is already taken)."
+            ),
+        ),
+    ] = None,
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
@@ -1745,12 +1762,13 @@ def new_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Check prefixes
-    if not overwrite:
-        for env_name, env in envs.items():
-            if env.get("prefix") == prefix:
+    # Only pin a prefix if one was explicitly given; otherwise it is resolved
+    # on the fly (defaulting to .venv) at check/run time
+    if prefix is not None and not overwrite:
+        for other_name, other_env in envs.items():
+            if other_env.get("prefix") == prefix:
                 raise_error(
-                    f"Environment '{env_name}' already exists with "
+                    f"Environment '{other_name}' already exists with "
                     f"prefix '{prefix}'"
                 )
     packages_txt = "\n".join(packages)
@@ -1760,7 +1778,9 @@ def new_venv(
         f.write(packages_txt)
     repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
-    env = dict(path=path, kind="venv", prefix=prefix)
+    env = dict(path=path, kind="venv")
+    if prefix is not None:
+        env["prefix"] = prefix
     if description is not None:
         env["description"] = description
     envs[name] = env

calkit/cli/notebooks.py

@@ -90,7 +90,7 @@ def _check_ijulia_available(
     env_dir: str,
 ) -> bool:
     ijulia_check_cmd = [
-        "julia",
+        calkit.julia.get_julia_exe(),
         f"+{julia_version}",
         "--project=" + env_dir,
         "-e",
@@ -322,7 +322,7 @@ def check_env_kernel(
             "println(kp);"
         )
         cmd = [
-            "julia",
+            calkit.julia.get_julia_exe(),
             f"+{julia_version}",
             "--project=" + env_dir,
             "-e",

calkit/cli/scheduler.py

@@ -251,20 +251,19 @@ def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
 
 
 def _wait_until_done(kind: str, job_id: str, name: str) -> None:
-    """Poll until the job finishes, leaving it running on Ctrl+C.
+    """Poll until the job finishes, canceling it on Ctrl+C.
 
-    The job is already submitted and tracked, so an interrupt should only stop
-    local waiting---the scheduler keeps running it and a later ``calkit run``
-    resumes from here.
+    The job is submitted and tracked, so interrupting the local wait cancels
+    the scheduler job before exiting rather than leaving it orphaned.
     """
     try:
         while _is_active(kind, job_id):
             time.sleep(1)
     except KeyboardInterrupt:
-        typer.echo(
-            f"Interrupted; job '{name}' ({job_id}) left running on the "
-            "scheduler. Resume with `calkit run`."
-        )
+        typer.echo(f"Interrupted; canceling job '{name}' ({job_id})")
+        ok, stderr = _cancel(kind, job_id)
+        if not ok:
+            typer.echo(f"Failed to cancel job '{name}' ({job_id}): {stderr}")
         raise typer.Exit(130)