calkit-python 0.41.9__py3-none-any.whl → 0.41.10__py3-none-any.whl

calkit/cli/check.py

@@ -163,7 +163,7 @@ def _check_julia_env(
         deps_to_add = []
     if deps_to_add:
         pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
-        cmd = ["julia"]
+        cmd = [calkit.julia.get_julia_exe()]
         if julia_version:
             cmd.append(f"+{julia_version}")
         cmd += [
@@ -190,7 +190,7 @@ def _check_julia_env(
                 success=False,
             )
             raise_error("Failed to add Julia dependencies")
-    cmd = ["julia"]
+    cmd = [calkit.julia.get_julia_exe()]
     if julia_version:
         cmd.append(f"+{julia_version}")
     cmd += [
@@ -318,6 +318,7 @@ def check_environment(
         get_all_conda_lock_fpaths,
         get_all_docker_lock_fpaths,
         get_all_venv_lock_fpaths,
+        get_default_venv_prefix,
         get_env_lock_fpath,
         write_scheduler_env_lock,
     )
@@ -403,12 +404,14 @@ def check_environment(
         except subprocess.CalledProcessError:
             raise_error("Failed to check uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
-        if "prefix" not in env:
-            raise_error("venv environments require a prefix")
         if "path" not in env:
             raise_error("venv environments require a path")
-        prefix = os.path.expandvars(env["prefix"])
         path = os.path.expandvars(env["path"])
+        # Resolve the prefix on the fly if it isn't pinned in calkit.yaml
+        prefix = env.get("prefix")
+        if prefix is None:
+            prefix = get_default_venv_prefix(envs, path, env_name)
+        prefix = os.path.expandvars(prefix)
         lock_fpath = get_env_lock_fpath(
             env=env, env_name=env_name, as_posix=False
         )
@@ -543,6 +546,41 @@ def check_environments(
         )
 
 
+def _renv_snapshot_from_description(env_dir: str, verbose: bool) -> None:
+    """Install packages from DESCRIPTION and (re)write renv.lock.
+
+    Used to create the initial lock, to add newly declared packages, and as a
+    fallback when an existing lock can't be restored (e.g. its versions don't
+    build against the installed R version).
+    """
+    hydrate_cmd = [
+        "Rscript",
+        "--vanilla",
+        "-e",
+        "renv::load(); renv::hydrate()",
+    ]
+    if verbose:
+        typer.echo(f"Running: {' '.join(hydrate_cmd)}")
+    try:
+        subprocess.check_call(hydrate_cmd, cwd=env_dir)
+    except subprocess.CalledProcessError:
+        # Hydrate may fail if some packages aren't available; snapshot anyway
+        if verbose:
+            typer.echo("Warning: hydrate had issues, continuing to snapshot")
+    snapshot_cmd = [
+        "Rscript",
+        "--vanilla",
+        "-e",
+        "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
+    ]
+    if verbose:
+        typer.echo(f"Running: {' '.join(snapshot_cmd)}")
+    try:
+        subprocess.check_call(snapshot_cmd, cwd=env_dir)
+    except subprocess.CalledProcessError:
+        raise_error(f"Failed to snapshot renv in {env_dir}")
+
+
 @check_app.command(name="renv")
 def check_renv(
     env_path: Annotated[
@@ -606,153 +644,74 @@ def check_renv(
             subprocess.check_call(init_cmd, cwd=env_dir)
         except subprocess.CalledProcessError:
             raise_error(f"Failed to initialize renv in {env_dir}")
-        # Use hydrate to install packages from DESCRIPTION and snapshot
+        # Install packages from DESCRIPTION and write the lock file
         if verbose:
             typer.echo("Setting up environment from DESCRIPTION")
-        hydrate_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); renv::hydrate()",
-        ]
-        if verbose:
-            typer.echo(f"Running: {' '.join(hydrate_cmd)}")
-        try:
-            subprocess.check_call(hydrate_cmd, cwd=env_dir)
-        except subprocess.CalledProcessError:
-            # Hydrate might fail if packages aren't available, continue anyway
-            if verbose:
-                typer.echo(
-                    "Warning: hydrate had issues, continuing to snapshot"
-                )
-        # Always snapshot after hydrate to create lock file from DESCRIPTION
+        _renv_snapshot_from_description(env_dir, verbose=verbose)
+    else:
+        # A lock file exists, so treat it as the source of truth: restore the
+        # library to the exact recorded versions rather than re-resolving from
+        # DESCRIPTION (which would bump packages and overwrite the lock).
         if verbose:
-            typer.echo("Creating lock file from DESCRIPTION")
-        snapshot_cmd = [
+            typer.echo("Restoring library from lockfile")
+        restore_cmd = [
             "Rscript",
             "--vanilla",
             "-e",
-            "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
+            "renv::load(); renv::restore(prompt=FALSE)",
         ]
         if verbose:
-            typer.echo(f"Running: {' '.join(snapshot_cmd)}")
+            typer.echo(f"Running: {' '.join(restore_cmd)}")
         try:
-            subprocess.check_call(snapshot_cmd, cwd=env_dir)
+            subprocess.check_call(restore_cmd, cwd=env_dir)
+            restored = True
         except subprocess.CalledProcessError:
-            raise_error(f"Failed to snapshot renv in {env_dir}")
-    else:
-        # Lock file exists, check if it's in sync with DESCRIPTION
-        if verbose:
-            typer.echo("Checking if lockfile is in sync with DESCRIPTION")
-        # Check status to see if lockfile needs updating
-        status_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); status <- renv::status(); cat(status$synchronized)",
-        ]
-        try:
-            result = subprocess.run(
-                status_cmd,
-                cwd=env_dir,
-                capture_output=True,
-                text=True,
-                check=True,
+            restored = False
+        if not restored:
+            # The locked versions couldn't be installed (e.g. they don't build
+            # against the installed R version). Fall back to re-resolving from
+            # DESCRIPTION, which updates renv.lock to a working set.
+            warn(
+                f"Could not restore renv environment in {env_dir} from "
+                "renv.lock; the locked versions may be incompatible with the "
+                "installed R version. Re-resolving from DESCRIPTION and "
+                "updating renv.lock."
             )
-            lockfile_synced = "TRUE" in result.stdout
-        except subprocess.CalledProcessError:
-            # If status fails, assume we need to update
-            lockfile_synced = False
-            if verbose:
-                typer.echo("Warning: status check failed, will update lock")
-        if not lockfile_synced:
+            _renv_snapshot_from_description(env_dir, verbose=verbose)
+        else:
+            # Only update the lock if DESCRIPTION declares dependencies the
+            # lock doesn't cover (e.g. the user added a package). After the
+            # restore the library matches the lock, so status is unsynchronized
+            # only when DESCRIPTION and the lock genuinely disagree---not merely
+            # because packages were missing on a fresh checkout.
             if verbose:
-                typer.echo("Lockfile out of sync, updating from DESCRIPTION")
-            # Use hydrate to update from DESCRIPTION
-            hydrate_cmd = [
+                typer.echo("Checking if DESCRIPTION matches lockfile")
+            status_cmd = [
                 "Rscript",
                 "--vanilla",
                 "-e",
-                "renv::load(); renv::hydrate()",
+                "renv::load(); cat(renv::status()$synchronized)",
             ]
-            if verbose:
-                typer.echo(f"Running: {' '.join(hydrate_cmd)}")
             try:
-                subprocess.check_call(hydrate_cmd, cwd=env_dir)
+                result = subprocess.run(
+                    status_cmd,
+                    cwd=env_dir,
+                    capture_output=True,
+                    text=True,
+                    check=True,
+                )
+                in_sync = "TRUE" in result.stdout
             except subprocess.CalledProcessError:
+                # If status fails, keep the lock rather than risk clobbering it
+                in_sync = True
                 if verbose:
-                    typer.echo(
-                        "Warning: hydrate had issues, continuing to snapshot"
-                    )
-            # Snapshot to update lock
-            snapshot_cmd = [
-                "Rscript",
-                "--vanilla",
-                "-e",
-                "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
-            ]
-            if verbose:
-                typer.echo(f"Running: {' '.join(snapshot_cmd)}")
-            try:
-                subprocess.check_call(snapshot_cmd, cwd=env_dir)
-            except subprocess.CalledProcessError:
+                    typer.echo("Warning: status check failed, keeping lock")
+            if not in_sync:
                 if verbose:
-                    typer.echo("Warning: snapshot failed, using existing lock")
-        else:
-            if verbose:
+                    typer.echo("DESCRIPTION changed; updating lockfile")
+                _renv_snapshot_from_description(env_dir, verbose=verbose)
+            elif verbose:
                 typer.echo("Lockfile is already in sync with DESCRIPTION")
-    # Check if library needs restoring
-    if verbose:
-        typer.echo("Checking if library is in sync with lockfile")
-    lib_status_cmd = [
-        "Rscript",
-        "--vanilla",
-        "-e",
-        (
-            "renv::load(); "
-            "status <- tryCatch({"
-            "  renv::status();"
-            "  cat('synchronized');"
-            "}, error = function(e) {"
-            "  cat('needs_restore');"
-            "})"
-        ),
-    ]
-    try:
-        result = subprocess.run(
-            lib_status_cmd,
-            cwd=env_dir,
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        needs_restore = "needs_restore" in result.stdout or (
-            "synchronized" not in result.stdout
-        )
-    except subprocess.CalledProcessError:
-        # If check fails, restore to be safe
-        needs_restore = True
-        if verbose:
-            typer.echo("Warning: library status check failed, will restore")
-
-    if needs_restore:
-        if verbose:
-            typer.echo("Restoring library from lockfile")
-        restore_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); renv::restore(prompt=FALSE)",
-        ]
-        if verbose:
-            typer.echo(f"Running: {' '.join(restore_cmd)}")
-        try:
-            subprocess.check_call(restore_cmd, cwd=env_dir)
-        except subprocess.CalledProcessError:
-            raise_error(f"Failed to restore renv in {env_dir}")
-    else:
-        if verbose:
-            typer.echo("Library is already in sync with lockfile")
 
 
 @check_app.command(name="docker-env")

calkit/cli/main/core.py

@@ -1762,6 +1762,17 @@ def run(
             "--no-push", help="Do not push to Git and DVC after saving."
         ),
     ] = False,
+    mock_scheduler: Annotated[
+        bool,
+        typer.Option(
+            "--mock-scheduler",
+            "-K",
+            help=(
+                "Run job-scheduler (SLURM/PBS) stages locally instead of "
+                "submitting them to a real scheduler."
+            ),
+        ),
+    ] = False,
 ) -> dict:
     """Check dependencies and run the pipeline."""
     import dvc.log
@@ -1779,6 +1790,10 @@ def run(
     if (target_inputs or target_outputs) and targets:
         raise_error("Cannot specify both targets and inputs")
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
+    # Mock the scheduler for this run (and any subprocesses) so SLURM/PBS
+    # stages execute locally; child processes inherit it via os.environ
+    if mock_scheduler:
+        os.environ[calkit.cli.scheduler.MOCK_ENV_VAR] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     ck_info = calkit.load_calkit_info()
     # Ensure Git is initialized so DVC can be used.
@@ -2463,12 +2478,15 @@ def run_in_env(
         except subprocess.CalledProcessError:
             raise_error("Failed to run in uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
-        if "prefix" not in env:
-            raise_error("venv environments require a prefix")
         if "path" not in env:
             raise_error("venv environments require a path")
-        prefix = env["prefix"]
         path = env["path"]
+        # Resolve the prefix on the fly if it isn't pinned in calkit.yaml
+        prefix = env.get("prefix")
+        if prefix is None:
+            prefix = calkit.environments.get_default_venv_prefix(
+                envs, path, env_name
+            )
         shell_cmd = _to_shell_cmd(cmd)
         if _platform.system() == "Windows":
             activate_cmd = f"{prefix}\\Scripts\\activate"
@@ -2525,7 +2543,7 @@ def run_in_env(
         # specifying the project
         cmd = [arg for arg in cmd if not arg.startswith("--project=")]
         julia_cmd = [
-            "julia",
+            calkit.julia.get_julia_exe(),
             f"+{julia_version}",
             "--project=" + env_dir,
         ] + cmd

calkit/cli/new.py

@@ -1604,8 +1604,15 @@ def new_uv_venv(
         str, typer.Option("--path", help="Path for requirements file.")
     ] = "requirements.txt",
     prefix: Annotated[
-        str, typer.Option("--prefix", help="Prefix for environment location.")
-    ] = ".venv",
+        str | None,
+        typer.Option(
+            "--prefix",
+            help=(
+                "Prefix for environment location (defaults to .venv, or "
+                ".calkit/envs/<name>/.venv if .venv is already taken)."
+            ),
+        ),
+    ] = None,
     python_version: Annotated[
         str | None, typer.Option("--python", "-p", help="Python version.")
     ] = None,
@@ -1654,12 +1661,13 @@ def new_uv_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Check prefixes
-    if not overwrite:
-        for env_name, env in envs.items():
-            if env.get("prefix") == prefix:
+    # Only pin a prefix if one was explicitly given; otherwise it is resolved
+    # on the fly (defaulting to .venv) at check/run time
+    if prefix is not None and not overwrite:
+        for other_name, other_env in envs.items():
+            if other_env.get("prefix") == prefix:
                 raise_error(
-                    f"Environment '{env_name}' already exists with "
+                    f"Environment '{other_name}' already exists with "
                     f"prefix '{prefix}'"
                 )
     if packages is not None:
@@ -1669,7 +1677,9 @@ def new_uv_venv(
         with open(path, "w") as f:
             f.write(packages_txt)
     typer.echo("Adding environment to calkit.yaml")
-    env = dict(path=path, kind="uv-venv", prefix=prefix)
+    env = dict(path=path, kind="uv-venv")
+    if prefix is not None:
+        env["prefix"] = prefix
     if python_version is not None:
         env["python"] = python_version
     if description is not None:
@@ -1703,8 +1713,15 @@ def new_venv(
         str, typer.Option("--path", help="Path for requirements file.")
     ] = "requirements.txt",
     prefix: Annotated[
-        str, typer.Option("--prefix", help="Prefix for environment location.")
-    ] = ".venv",
+        str | None,
+        typer.Option(
+            "--prefix",
+            help=(
+                "Prefix for environment location (defaults to .venv, or "
+                ".calkit/envs/<name>/.venv if .venv is already taken)."
+            ),
+        ),
+    ] = None,
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
@@ -1745,12 +1762,13 @@ def new_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Check prefixes
-    if not overwrite:
-        for env_name, env in envs.items():
-            if env.get("prefix") == prefix:
+    # Only pin a prefix if one was explicitly given; otherwise it is resolved
+    # on the fly (defaulting to .venv) at check/run time
+    if prefix is not None and not overwrite:
+        for other_name, other_env in envs.items():
+            if other_env.get("prefix") == prefix:
                 raise_error(
-                    f"Environment '{env_name}' already exists with "
+                    f"Environment '{other_name}' already exists with "
                     f"prefix '{prefix}'"
                 )
     packages_txt = "\n".join(packages)
@@ -1760,7 +1778,9 @@ def new_venv(
         f.write(packages_txt)
     repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
-    env = dict(path=path, kind="venv", prefix=prefix)
+    env = dict(path=path, kind="venv")
+    if prefix is not None:
+        env["prefix"] = prefix
     if description is not None:
         env["description"] = description
     envs[name] = env

calkit/cli/notebooks.py

@@ -90,7 +90,7 @@ def _check_ijulia_available(
     env_dir: str,
 ) -> bool:
     ijulia_check_cmd = [
-        "julia",
+        calkit.julia.get_julia_exe(),
         f"+{julia_version}",
         "--project=" + env_dir,
         "-e",
@@ -322,7 +322,7 @@ def check_env_kernel(
             "println(kp);"
         )
         cmd = [
-            "julia",
+            calkit.julia.get_julia_exe(),
             f"+{julia_version}",
             "--project=" + env_dir,
             "-e",

calkit/cli/scheduler.py

@@ -251,20 +251,19 @@ def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
 
 
 def _wait_until_done(kind: str, job_id: str, name: str) -> None:
-    """Poll until the job finishes, leaving it running on Ctrl+C.
+    """Poll until the job finishes, canceling it on Ctrl+C.
 
-    The job is already submitted and tracked, so an interrupt should only stop
-    local waiting---the scheduler keeps running it and a later ``calkit run``
-    resumes from here.
+    The job is submitted and tracked, so interrupting the local wait cancels
+    the scheduler job before exiting rather than leaving it orphaned.
     """
     try:
         while _is_active(kind, job_id):
             time.sleep(1)
     except KeyboardInterrupt:
-        typer.echo(
-            f"Interrupted; job '{name}' ({job_id}) left running on the "
-            "scheduler. Resume with `calkit run`."
-        )
+        typer.echo(f"Interrupted; canceling job '{name}' ({job_id})")
+        ok, stderr = _cancel(kind, job_id)
+        if not ok:
+            typer.echo(f"Failed to cancel job '{name}' ({job_id}): {stderr}")
         raise typer.Exit(130)
 
 

calkit/dvc/core.py

@@ -261,6 +261,25 @@ def get_dvc_repo(wdir: str | None = None) -> dvc.repo.Repo:
     return dvc.repo.Repo(wdir)
 
 
+def ensure_dvc_lock_not_ignored(wdir: str | None = None) -> bool:
+    """Ensure ``dvc.lock`` is not Git-ignored.
+
+    DVC raises ``FileIsGitIgnored`` when ``dvc.lock`` is excluded by Git, which
+    breaks ``dvc status`` and pipeline runs. This un-ignores it (editing the
+    relevant ``.gitignore`` as needed) so those operations keep working.
+
+    Returns True if a ``.gitignore`` was modified.
+    """
+    import calkit.git
+
+    lock_path = os.path.join(wdir, "dvc.lock") if wdir else "dvc.lock"
+    try:
+        repo = calkit.git.get_repo(wdir)
+    except Exception:
+        return False
+    return bool(calkit.git.ensure_path_is_not_ignored(repo, path=lock_path))
+
+
 def get_running_pipeline_processes(wdir: str | None = None) -> list[dict]:
     """Return live processes holding DVC's read/write lock.
 

calkit/environments.py

@@ -138,7 +138,7 @@ def _get_julia_version() -> str:
     """
     try:
         result = subprocess.run(
-            ["julia", "--version"],
+            [calkit.julia.get_julia_exe(), "--version"],
             capture_output=True,
             text=True,
             timeout=5,
@@ -449,7 +449,15 @@ def write_scheduler_env_lock(
     parent = os.path.dirname(full_path)
     if parent:
         os.makedirs(parent, exist_ok=True)
-    content = json.dumps(env, indent=2, sort_keys=True) + "\n"
+    lock_data = dict(env)
+    # Record when the scheduler is mocked so switching between a mocked run
+    # (executed locally) and a real scheduler run changes the lock file and
+    # invalidates the cached result
+    from calkit.cli.scheduler import _mock_enabled
+
+    if _mock_enabled():
+        lock_data["mocked"] = True
+    content = json.dumps(lock_data, indent=2, sort_keys=True) + "\n"
     if os.path.isfile(full_path):
         with open(full_path, "r") as f:
             existing = f.read()
@@ -744,6 +752,55 @@ def make_env_name(path: str, all_env_names: list[str], kind: str) -> str:
     return name
 
 
+def get_default_venv_prefix(envs: dict, path: str, name: str) -> str:
+    """Return the default prefix for a venv or uv-venv environment.
+
+    The prefix defaults to ``.venv`` in the same directory as ``path``,
+    unless that location is already claimed by another environment, in which
+    case the virtualenv is nested under ``.calkit/envs/{name}/.venv``. This
+    is resolved on the fly so that the prefix need not be stored in
+    ``calkit.yaml``.
+
+    A location is considered claimed by another environment if it pins that
+    explicit ``prefix``, or if it is a ``uv``, ``venv``, or ``uv-venv``
+    environment whose ``.venv`` would live there (``uv`` always creates its
+    virtualenv at ``.venv`` in its project directory, so a flexible venv
+    yields to it). Sibling venvs that would otherwise collide all nest under
+    their own name-scoped location, which is collision-free.
+
+    Parameters
+    ----------
+    envs : dict
+        All environments, keyed by name.
+    path : str
+        Path to the spec file the environment lives alongside.
+    name : str
+        Name of the environment being resolved, used both to exclude it from
+        the claimed locations and for the nested fallback.
+
+    Returns
+    -------
+    str
+        A POSIX-style prefix that does not collide with another environment.
+    """
+    base = os.path.join(os.path.dirname(path), ".venv")
+    # Collect .venv locations claimed by the other environments
+    claimed = set()
+    for other_name, env in envs.items():
+        if other_name == name:
+            continue
+        prefix = env.get("prefix")
+        if prefix is not None:
+            claimed.add(os.path.normpath(prefix))
+        elif env.get("kind") in ("uv", "venv", "uv-venv"):
+            other_dir = os.path.dirname(env.get("path", ""))
+            claimed.add(os.path.normpath(os.path.join(other_dir, ".venv")))
+    # Nest under .calkit/envs/{name} if the default location is taken
+    if os.path.normpath(base) in claimed:
+        base = os.path.join(".calkit", "envs", name, ".venv")
+    return Path(base).as_posix()
+
+
 def env_from_name_or_path(
     name_or_path: str | None = None,
     ck_info: dict | None = None,
@@ -851,15 +908,14 @@ def env_from_name_or_path(
     if os.path.isfile(env_path):
         if env_path.endswith("requirements.txt"):
             # TODO: Detect if uv is installed, and use a plain venv if not
+            # The prefix is left unset and resolved on the fly at check/run
+            # time via get_default_venv_prefix
             return EnvDetectResult(
                 name=make_env_name(env_path, all_env_names, kind="uv-venv"),
                 env={
                     "kind": "uv-venv",
                     "path": env_path,
                     "python": DEFAULT_PYTHON_VERSION,
-                    "prefix": Path(env_path)
-                    .parent.joinpath(".venv")
-                    .as_posix(),
                 },
                 exists=False,
             )
@@ -1325,7 +1381,7 @@ end
         # Run Julia with the script
         result = subprocess.run(
             [
-                "julia",
+                calkit.julia.get_julia_exe(),
                 script_path,
                 ",".join(package_names),
             ],