calkit-python 0.41.8__py3-none-any.whl → 0.41.10__py3-none-any.whl

calkit/cli/check.py

@@ -163,7 +163,7 @@ def _check_julia_env(
         deps_to_add = []
     if deps_to_add:
         pkg_list = ", ".join(f'"{dep}"' for dep in deps_to_add)
-        cmd = ["julia"]
+        cmd = [calkit.julia.get_julia_exe()]
         if julia_version:
             cmd.append(f"+{julia_version}")
         cmd += [
@@ -190,7 +190,7 @@ def _check_julia_env(
                 success=False,
             )
             raise_error("Failed to add Julia dependencies")
-    cmd = ["julia"]
+    cmd = [calkit.julia.get_julia_exe()]
     if julia_version:
         cmd.append(f"+{julia_version}")
     cmd += [
@@ -318,6 +318,7 @@ def check_environment(
         get_all_conda_lock_fpaths,
         get_all_docker_lock_fpaths,
         get_all_venv_lock_fpaths,
+        get_default_venv_prefix,
         get_env_lock_fpath,
         write_scheduler_env_lock,
     )
@@ -403,12 +404,14 @@ def check_environment(
         except subprocess.CalledProcessError:
             raise_error("Failed to check uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
-        if "prefix" not in env:
-            raise_error("venv environments require a prefix")
         if "path" not in env:
             raise_error("venv environments require a path")
-        prefix = os.path.expandvars(env["prefix"])
         path = os.path.expandvars(env["path"])
+        # Resolve the prefix on the fly if it isn't pinned in calkit.yaml
+        prefix = env.get("prefix")
+        if prefix is None:
+            prefix = get_default_venv_prefix(envs, path, env_name)
+        prefix = os.path.expandvars(prefix)
         lock_fpath = get_env_lock_fpath(
             env=env, env_name=env_name, as_posix=False
         )
@@ -543,6 +546,41 @@ def check_environments(
         )
 
 
+def _renv_snapshot_from_description(env_dir: str, verbose: bool) -> None:
+    """Install packages from DESCRIPTION and (re)write renv.lock.
+
+    Used to create the initial lock, to add newly declared packages, and as a
+    fallback when an existing lock can't be restored (e.g. its versions don't
+    build against the installed R version).
+    """
+    hydrate_cmd = [
+        "Rscript",
+        "--vanilla",
+        "-e",
+        "renv::load(); renv::hydrate()",
+    ]
+    if verbose:
+        typer.echo(f"Running: {' '.join(hydrate_cmd)}")
+    try:
+        subprocess.check_call(hydrate_cmd, cwd=env_dir)
+    except subprocess.CalledProcessError:
+        # Hydrate may fail if some packages aren't available; snapshot anyway
+        if verbose:
+            typer.echo("Warning: hydrate had issues, continuing to snapshot")
+    snapshot_cmd = [
+        "Rscript",
+        "--vanilla",
+        "-e",
+        "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
+    ]
+    if verbose:
+        typer.echo(f"Running: {' '.join(snapshot_cmd)}")
+    try:
+        subprocess.check_call(snapshot_cmd, cwd=env_dir)
+    except subprocess.CalledProcessError:
+        raise_error(f"Failed to snapshot renv in {env_dir}")
+
+
 @check_app.command(name="renv")
 def check_renv(
     env_path: Annotated[
@@ -606,153 +644,74 @@ def check_renv(
             subprocess.check_call(init_cmd, cwd=env_dir)
         except subprocess.CalledProcessError:
             raise_error(f"Failed to initialize renv in {env_dir}")
-        # Use hydrate to install packages from DESCRIPTION and snapshot
+        # Install packages from DESCRIPTION and write the lock file
         if verbose:
             typer.echo("Setting up environment from DESCRIPTION")
-        hydrate_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); renv::hydrate()",
-        ]
-        if verbose:
-            typer.echo(f"Running: {' '.join(hydrate_cmd)}")
-        try:
-            subprocess.check_call(hydrate_cmd, cwd=env_dir)
-        except subprocess.CalledProcessError:
-            # Hydrate might fail if packages aren't available, continue anyway
-            if verbose:
-                typer.echo(
-                    "Warning: hydrate had issues, continuing to snapshot"
-                )
-        # Always snapshot after hydrate to create lock file from DESCRIPTION
+        _renv_snapshot_from_description(env_dir, verbose=verbose)
+    else:
+        # A lock file exists, so treat it as the source of truth: restore the
+        # library to the exact recorded versions rather than re-resolving from
+        # DESCRIPTION (which would bump packages and overwrite the lock).
         if verbose:
-            typer.echo("Creating lock file from DESCRIPTION")
-        snapshot_cmd = [
+            typer.echo("Restoring library from lockfile")
+        restore_cmd = [
             "Rscript",
             "--vanilla",
             "-e",
-            "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
+            "renv::load(); renv::restore(prompt=FALSE)",
         ]
         if verbose:
-            typer.echo(f"Running: {' '.join(snapshot_cmd)}")
+            typer.echo(f"Running: {' '.join(restore_cmd)}")
         try:
-            subprocess.check_call(snapshot_cmd, cwd=env_dir)
+            subprocess.check_call(restore_cmd, cwd=env_dir)
+            restored = True
         except subprocess.CalledProcessError:
-            raise_error(f"Failed to snapshot renv in {env_dir}")
-    else:
-        # Lock file exists, check if it's in sync with DESCRIPTION
-        if verbose:
-            typer.echo("Checking if lockfile is in sync with DESCRIPTION")
-        # Check status to see if lockfile needs updating
-        status_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); status <- renv::status(); cat(status$synchronized)",
-        ]
-        try:
-            result = subprocess.run(
-                status_cmd,
-                cwd=env_dir,
-                capture_output=True,
-                text=True,
-                check=True,
+            restored = False
+        if not restored:
+            # The locked versions couldn't be installed (e.g. they don't build
+            # against the installed R version). Fall back to re-resolving from
+            # DESCRIPTION, which updates renv.lock to a working set.
+            warn(
+                f"Could not restore renv environment in {env_dir} from "
+                "renv.lock; the locked versions may be incompatible with the "
+                "installed R version. Re-resolving from DESCRIPTION and "
+                "updating renv.lock."
             )
-            lockfile_synced = "TRUE" in result.stdout
-        except subprocess.CalledProcessError:
-            # If status fails, assume we need to update
-            lockfile_synced = False
-            if verbose:
-                typer.echo("Warning: status check failed, will update lock")
-        if not lockfile_synced:
+            _renv_snapshot_from_description(env_dir, verbose=verbose)
+        else:
+            # Only update the lock if DESCRIPTION declares dependencies the
+            # lock doesn't cover (e.g. the user added a package). After the
+            # restore the library matches the lock, so status is unsynchronized
+            # only when DESCRIPTION and the lock genuinely disagree---not merely
+            # because packages were missing on a fresh checkout.
             if verbose:
-                typer.echo("Lockfile out of sync, updating from DESCRIPTION")
-            # Use hydrate to update from DESCRIPTION
-            hydrate_cmd = [
+                typer.echo("Checking if DESCRIPTION matches lockfile")
+            status_cmd = [
                 "Rscript",
                 "--vanilla",
                 "-e",
-                "renv::load(); renv::hydrate()",
+                "renv::load(); cat(renv::status()$synchronized)",
             ]
-            if verbose:
-                typer.echo(f"Running: {' '.join(hydrate_cmd)}")
             try:
-                subprocess.check_call(hydrate_cmd, cwd=env_dir)
+                result = subprocess.run(
+                    status_cmd,
+                    cwd=env_dir,
+                    capture_output=True,
+                    text=True,
+                    check=True,
+                )
+                in_sync = "TRUE" in result.stdout
             except subprocess.CalledProcessError:
+                # If status fails, keep the lock rather than risk clobbering it
+                in_sync = True
                 if verbose:
-                    typer.echo(
-                        "Warning: hydrate had issues, continuing to snapshot"
-                    )
-            # Snapshot to update lock
-            snapshot_cmd = [
-                "Rscript",
-                "--vanilla",
-                "-e",
-                "renv::load(); renv::snapshot(type='explicit', prompt=FALSE)",
-            ]
-            if verbose:
-                typer.echo(f"Running: {' '.join(snapshot_cmd)}")
-            try:
-                subprocess.check_call(snapshot_cmd, cwd=env_dir)
-            except subprocess.CalledProcessError:
+                    typer.echo("Warning: status check failed, keeping lock")
+            if not in_sync:
                 if verbose:
-                    typer.echo("Warning: snapshot failed, using existing lock")
-        else:
-            if verbose:
+                    typer.echo("DESCRIPTION changed; updating lockfile")
+                _renv_snapshot_from_description(env_dir, verbose=verbose)
+            elif verbose:
                 typer.echo("Lockfile is already in sync with DESCRIPTION")
-    # Check if library needs restoring
-    if verbose:
-        typer.echo("Checking if library is in sync with lockfile")
-    lib_status_cmd = [
-        "Rscript",
-        "--vanilla",
-        "-e",
-        (
-            "renv::load(); "
-            "status <- tryCatch({"
-            "  renv::status();"
-            "  cat('synchronized');"
-            "}, error = function(e) {"
-            "  cat('needs_restore');"
-            "})"
-        ),
-    ]
-    try:
-        result = subprocess.run(
-            lib_status_cmd,
-            cwd=env_dir,
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        needs_restore = "needs_restore" in result.stdout or (
-            "synchronized" not in result.stdout
-        )
-    except subprocess.CalledProcessError:
-        # If check fails, restore to be safe
-        needs_restore = True
-        if verbose:
-            typer.echo("Warning: library status check failed, will restore")
-
-    if needs_restore:
-        if verbose:
-            typer.echo("Restoring library from lockfile")
-        restore_cmd = [
-            "Rscript",
-            "--vanilla",
-            "-e",
-            "renv::load(); renv::restore(prompt=FALSE)",
-        ]
-        if verbose:
-            typer.echo(f"Running: {' '.join(restore_cmd)}")
-        try:
-            subprocess.check_call(restore_cmd, cwd=env_dir)
-        except subprocess.CalledProcessError:
-            raise_error(f"Failed to restore renv in {env_dir}")
-    else:
-        if verbose:
-            typer.echo("Library is already in sync with lockfile")
 
 
 @check_app.command(name="docker-env")

calkit/cli/main/core.py

@@ -1762,6 +1762,17 @@ def run(
             "--no-push", help="Do not push to Git and DVC after saving."
         ),
     ] = False,
+    mock_scheduler: Annotated[
+        bool,
+        typer.Option(
+            "--mock-scheduler",
+            "-K",
+            help=(
+                "Run job-scheduler (SLURM/PBS) stages locally instead of "
+                "submitting them to a real scheduler."
+            ),
+        ),
+    ] = False,
 ) -> dict:
     """Check dependencies and run the pipeline."""
     import dvc.log
@@ -1779,6 +1790,10 @@ def run(
     if (target_inputs or target_outputs) and targets:
         raise_error("Cannot specify both targets and inputs")
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
+    # Mock the scheduler for this run (and any subprocesses) so SLURM/PBS
+    # stages execute locally; child processes inherit it via os.environ
+    if mock_scheduler:
+        os.environ[calkit.cli.scheduler.MOCK_ENV_VAR] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     ck_info = calkit.load_calkit_info()
     # Ensure Git is initialized so DVC can be used.
@@ -2463,12 +2478,15 @@ def run_in_env(
         except subprocess.CalledProcessError:
             raise_error("Failed to run in uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
-        if "prefix" not in env:
-            raise_error("venv environments require a prefix")
         if "path" not in env:
             raise_error("venv environments require a path")
-        prefix = env["prefix"]
         path = env["path"]
+        # Resolve the prefix on the fly if it isn't pinned in calkit.yaml
+        prefix = env.get("prefix")
+        if prefix is None:
+            prefix = calkit.environments.get_default_venv_prefix(
+                envs, path, env_name
+            )
         shell_cmd = _to_shell_cmd(cmd)
         if _platform.system() == "Windows":
             activate_cmd = f"{prefix}\\Scripts\\activate"
@@ -2525,7 +2543,7 @@ def run_in_env(
         # specifying the project
         cmd = [arg for arg in cmd if not arg.startswith("--project=")]
         julia_cmd = [
-            "julia",
+            calkit.julia.get_julia_exe(),
             f"+{julia_version}",
             "--project=" + env_dir,
         ] + cmd

calkit/cli/new.py

@@ -1604,8 +1604,15 @@ def new_uv_venv(
         str, typer.Option("--path", help="Path for requirements file.")
     ] = "requirements.txt",
     prefix: Annotated[
-        str, typer.Option("--prefix", help="Prefix for environment location.")
-    ] = ".venv",
+        str | None,
+        typer.Option(
+            "--prefix",
+            help=(
+                "Prefix for environment location (defaults to .venv, or "
+                ".calkit/envs/<name>/.venv if .venv is already taken)."
+            ),
+        ),
+    ] = None,
     python_version: Annotated[
         str | None, typer.Option("--python", "-p", help="Python version.")
     ] = None,
@@ -1654,12 +1661,13 @@ def new_uv_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Check prefixes
-    if not overwrite:
-        for env_name, env in envs.items():
-            if env.get("prefix") == prefix:
+    # Only pin a prefix if one was explicitly given; otherwise it is resolved
+    # on the fly (defaulting to .venv) at check/run time
+    if prefix is not None and not overwrite:
+        for other_name, other_env in envs.items():
+            if other_env.get("prefix") == prefix:
                 raise_error(
-                    f"Environment '{env_name}' already exists with "
+                    f"Environment '{other_name}' already exists with "
                     f"prefix '{prefix}'"
                 )
     if packages is not None:
@@ -1669,7 +1677,9 @@ def new_uv_venv(
         with open(path, "w") as f:
             f.write(packages_txt)
     typer.echo("Adding environment to calkit.yaml")
-    env = dict(path=path, kind="uv-venv", prefix=prefix)
+    env = dict(path=path, kind="uv-venv")
+    if prefix is not None:
+        env["prefix"] = prefix
     if python_version is not None:
         env["python"] = python_version
     if description is not None:
@@ -1703,8 +1713,15 @@ def new_venv(
         str, typer.Option("--path", help="Path for requirements file.")
     ] = "requirements.txt",
     prefix: Annotated[
-        str, typer.Option("--prefix", help="Prefix for environment location.")
-    ] = ".venv",
+        str | None,
+        typer.Option(
+            "--prefix",
+            help=(
+                "Prefix for environment location (defaults to .venv, or "
+                ".calkit/envs/<name>/.venv if .venv is already taken)."
+            ),
+        ),
+    ] = None,
     description: Annotated[
         str | None, typer.Option("--description", help="Description.")
     ] = None,
@@ -1745,12 +1762,13 @@ def new_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
-    # Check prefixes
-    if not overwrite:
-        for env_name, env in envs.items():
-            if env.get("prefix") == prefix:
+    # Only pin a prefix if one was explicitly given; otherwise it is resolved
+    # on the fly (defaulting to .venv) at check/run time
+    if prefix is not None and not overwrite:
+        for other_name, other_env in envs.items():
+            if other_env.get("prefix") == prefix:
                 raise_error(
-                    f"Environment '{env_name}' already exists with "
+                    f"Environment '{other_name}' already exists with "
                     f"prefix '{prefix}'"
                 )
     packages_txt = "\n".join(packages)
@@ -1760,7 +1778,9 @@ def new_venv(
         f.write(packages_txt)
     repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
-    env = dict(path=path, kind="venv", prefix=prefix)
+    env = dict(path=path, kind="venv")
+    if prefix is not None:
+        env["prefix"] = prefix
     if description is not None:
         env["description"] = description
     envs[name] = env
@@ -3143,7 +3163,7 @@ def new_release(
             )
             with open("LICENSE", "w") as f:
                 f.write(license_txt)
-                repo.git.add("LICENSE")
+            repo.git.add("LICENSE")
             license_ids = ["mit", "cc-by-4.0"]
         else:
             typer.echo(f"Detecting license(s) from {license_file}")
@@ -3394,38 +3414,23 @@ def new_release(
         doi_base_url = calkit.releases.SERVICES[to]["url"]
         doi_md = (
             f"[![DOI]({doi_base_url}/badge/DOI/{doi}.svg)]"
-            f"(https://handle.stage.datacite.org/{doi})"
+            f"(https://doi.org/{doi})"
         )
         if os.path.isfile("README.md"):
             with open("README.md") as f:
                 readme_txt = f.read()
         else:
-            readme_txt = f"# {title}\n"
-        existing_lines = readme_txt.split("\n")
-        new_lines = []
-        first_content_line_index = None
-        for n, line in enumerate(existing_lines):
-            if line.startswith(doi_md[:6]):
-                pass  # Skip DOI lines
-            else:
-                if (
-                    n != 0
-                    and line.strip()
-                    and first_content_line_index is None
-                ):
-                    first_content_line_index = len(new_lines)
-                new_lines.append(line)
-        # Ensure first 3 lines are title, blank, DOI lines
-        new_lines = (
-            [new_lines[0]]
-            + ["", doi_md, ""]
-            + new_lines[first_content_line_index:]
+            readme_txt = ""
+        readme_txt = calkit.releases.add_doi_badge_to_readme(
+            readme_txt, badge=doi_md, title=title
         )
-        readme_txt = "\n".join(new_lines)
         if not dry_run:
             with open("README.md", "w") as f:
                 f.write(readme_txt)
-                repo.git.add("README.md")
+            # Stage only after the file is closed; otherwise the buffered
+            # contents may not be flushed to disk yet and Git would stage an
+            # empty file
+            repo.git.add("README.md")
         else:
             typer.echo(f"Would have updated README.md to:\n{readme_txt}")
     # Create Git tag
@@ -3480,11 +3485,13 @@ def new_release(
         references = reference_collections[0]
     ref_path = references.get("path", "references.bib")
     try:
+        # Read the existing references as raw text so we can append the new
+        # entry without reformatting the user's existing entries
         if os.path.isfile(ref_path):
             with open(ref_path) as f:
-                reflib = bibtexparser.load(f)
+                existing_text = f.read()
         else:
-            reflib = bibtexparser.bibdatabase.BibDatabase()
+            existing_text = ""
         bibtex_doi = doi
         if bibtex_doi is None and dry_run:
             mock_suffix = "".join(
@@ -3504,31 +3511,38 @@ def new_release(
         if not new_entries:
             raise ValueError("Failed to parse generated BibTeX entry")
         new_entry = new_entries[0]
-        # Search through entries for one with the same DOI, and replace if
-        # there is a match
+        # Search through existing entries for any with the same DOI, and
+        # replace them (by citation key) if there is a match. Tolerate parse
+        # errors here (e.g., non-standard or temporarily-invalid BibTeX) by
+        # skipping replacement detection and still appending the new entry.
         new_doi = new_entry.get("doi")
-        matching_indices = []
+        replace_ids = []
         if new_doi:
-            matching_indices = [
-                n
-                for n, entry in enumerate(reflib.entries)
+            try:
+                existing_entries = bibtexparser.loads(existing_text).entries
+            except Exception as e:
+                warn(f"Could not parse existing references to dedupe: {e}")
+                existing_entries = []
+            replace_ids = [
+                entry["ID"]
+                for entry in existing_entries
                 if entry.get("doi") == new_doi
             ]
-        if matching_indices:
+        if replace_ids:
             typer.echo(
                 "Found matching DOI in existing references "
-                f"({len(matching_indices)} entr"
-                f"{'y' if len(matching_indices) == 1 else 'ies'}); "
+                f"({len(replace_ids)} entr"
+                f"{'y' if len(replace_ids) == 1 else 'ies'}); "
                 "replacing"
             )
-            for n in reversed(matching_indices):
-                _ = reflib.entries.pop(n)
-        reflib.entries.append(new_entry)
+        new_text = calkit.releases.add_bibtex_entry(
+            existing_text, invenio_bibtex, replace_ids=replace_ids
+        )
         if dry_run:
             typer.echo(f"Would write updated references to {ref_path}")
         else:
             with open(ref_path, "w") as f:
-                bibtexparser.dump(reflib, f)
+                f.write(new_text)
             repo.git.add(ref_path)
     except Exception as e:
         warn(f"Failed to add to references: {e}")

calkit/cli/notebooks.py

@@ -90,7 +90,7 @@ def _check_ijulia_available(
     env_dir: str,
 ) -> bool:
     ijulia_check_cmd = [
-        "julia",
+        calkit.julia.get_julia_exe(),
         f"+{julia_version}",
         "--project=" + env_dir,
         "-e",
@@ -322,7 +322,7 @@ def check_env_kernel(
             "println(kp);"
         )
         cmd = [
-            "julia",
+            calkit.julia.get_julia_exe(),
             f"+{julia_version}",
             "--project=" + env_dir,
             "-e",

calkit/cli/scheduler.py

@@ -251,20 +251,19 @@ def _cancel(kind: str, job_id: str) -> tuple[bool, str]:
 
 
 def _wait_until_done(kind: str, job_id: str, name: str) -> None:
-    """Poll until the job finishes, leaving it running on Ctrl+C.
+    """Poll until the job finishes, canceling it on Ctrl+C.
 
-    The job is already submitted and tracked, so an interrupt should only stop
-    local waiting---the scheduler keeps running it and a later ``calkit run``
-    resumes from here.
+    The job is submitted and tracked, so interrupting the local wait cancels
+    the scheduler job before exiting rather than leaving it orphaned.
     """
     try:
         while _is_active(kind, job_id):
             time.sleep(1)
     except KeyboardInterrupt:
-        typer.echo(
-            f"Interrupted; job '{name}' ({job_id}) left running on the "
-            "scheduler. Resume with `calkit run`."
-        )
+        typer.echo(f"Interrupted; canceling job '{name}' ({job_id})")
+        ok, stderr = _cancel(kind, job_id)
+        if not ok:
+            typer.echo(f"Failed to cancel job '{name}' ({job_id}): {stderr}")
         raise typer.Exit(130)
 
 

calkit/dvc/core.py

@@ -261,6 +261,25 @@ def get_dvc_repo(wdir: str | None = None) -> dvc.repo.Repo:
     return dvc.repo.Repo(wdir)
 
 
+def ensure_dvc_lock_not_ignored(wdir: str | None = None) -> bool:
+    """Ensure ``dvc.lock`` is not Git-ignored.
+
+    DVC raises ``FileIsGitIgnored`` when ``dvc.lock`` is excluded by Git, which
+    breaks ``dvc status`` and pipeline runs. This un-ignores it (editing the
+    relevant ``.gitignore`` as needed) so those operations keep working.
+
+    Returns True if a ``.gitignore`` was modified.
+    """
+    import calkit.git
+
+    lock_path = os.path.join(wdir, "dvc.lock") if wdir else "dvc.lock"
+    try:
+        repo = calkit.git.get_repo(wdir)
+    except Exception:
+        return False
+    return bool(calkit.git.ensure_path_is_not_ignored(repo, path=lock_path))
+
+
 def get_running_pipeline_processes(wdir: str | None = None) -> list[dict]:
     """Return live processes holding DVC's read/write lock.