calkit-python 0.39.0__tar.gz → 0.39.2__tar.gz

{calkit_python-0.39.0 → calkit_python-0.39.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.39.0
+Version: 0.39.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.39.0 → calkit_python-0.39.2}/calkit/__init__.py

@@ -1,9 +1,47 @@
+from __future__ import annotations
+
 from importlib.metadata import version as _version
+from typing import TYPE_CHECKING
 
 __version__ = _version("calkit-python")
 
 from .core import *  # noqa: F403, I001
 
+if TYPE_CHECKING:
+    from calkit import (  # noqa: F401
+        calc,
+        check,
+        cloud,
+        conda,
+        config,
+        datasets,
+        detect,
+        docker,
+        dvc,
+        environments,
+        fs,
+        git,
+        github,
+        gui,
+        invenio,
+        julia,
+        jupyter,
+        licenses,
+        magics,
+        matlab,
+        models,
+        notebooks,
+        office,
+        ops,
+        overleaf,
+        pipeline,
+        releases,
+        server,
+        templates,
+    )
+
+    from .notebooks import declare_notebook  # noqa: F401
+
 # Lazy-load submodules to speed up imports
 _SUBMODULES = {
     "git",

{calkit_python-0.39.0 → calkit_python-0.39.2}/calkit/cli/main/core.py

@@ -669,9 +669,11 @@ def add(
             paths.remove(".")
             if dvc_repo is not None:
                 dvc_status = dvc_repo.data_status()
-                for dvc_uncommitted in dvc_status["uncommitted"].get(
+                uncommitted = dvc_status["uncommitted"]
+                dvc_uncommitted_all = uncommitted.get(
                     "modified", []
-                ):
+                ) + uncommitted.get("deleted", [])
+                for dvc_uncommitted in dvc_uncommitted_all:
                     if os.path.exists(dvc_uncommitted):
                         if dry_run:
                             typer.echo(
@@ -679,12 +681,25 @@ def add(
                             )
                         else:
                             typer.echo(f"Adding {dvc_uncommitted} to DVC")
-                            dvc_repo.commit(dvc_uncommitted, force=True)
+                            dvc_repo.commit(
+                                dvc_uncommitted,
+                                force=True,
+                                allow_missing=True,
+                            )
                     else:
-                        warn(
-                            f"DVC uncommitted '{dvc_uncommitted}' does not "
-                            "exist; skipping"
-                        )
+                        if dry_run:
+                            typer.echo(
+                                f"Would commit deleted {dvc_uncommitted} to DVC"
+                            )
+                        else:
+                            typer.echo(
+                                f"Committing deleted {dvc_uncommitted} to DVC"
+                            )
+                            dvc_repo.commit(
+                                dvc_uncommitted,
+                                force=True,
+                                allow_missing=True,
+                            )
             if not disable_auto_ignore:
                 for untracked_file in untracked_git_files:
                     if (
@@ -1053,10 +1068,32 @@ def pull(
             if calkit.dvc.detect_calkit_remote_type(name, url) == "http":
                 typer.echo(f"Checking authentication for DVC remote: {name}")
                 calkit.dvc.set_remote_auth(remote_name=name)
+    if (
+        not no_recursive
+        and "--recursive" not in dvc_args
+        and "-R" not in dvc_args
+    ):
+        dvc_args.append("--recursive")
     result = calkit.dvc.run_dvc_command(["pull"] + dvc_args)
     if result != 0:
         raise_error("DVC pull failed")
     calkit.dvc.zip.sync_all(direction="to-workspace")
+    if not no_recursive:
+        # Pull DVC in isolated subprojects (those with their own .dvc folder)
+        ck_info = calkit.load_calkit_info()
+        for sp in ck_info.get("subprojects", []):
+            if not isinstance(sp, dict) or not sp.get("path"):
+                continue
+            sp_path = sp["path"]
+            if not os.path.isdir(os.path.join(sp_path, ".dvc")):
+                continue
+            typer.echo(f"DVC pulling subproject: {sp_path}")
+            sp_result = calkit.dvc.run_dvc_command(
+                ["pull"] + dvc_args, cwd=sp_path
+            )
+            if sp_result != 0:
+                raise_error(f"DVC pull failed for subproject: {sp_path}")
+            calkit.dvc.zip.sync_all(direction="to-workspace", wdir=sp_path)
 
 
 @app.command(name="push")
@@ -1431,13 +1468,8 @@ def run(
             raise_error("Failed to initialize Git repo")
     # Set env vars
     calkit.set_env_vars(ck_info=ck_info)
-    # Clean all notebooks in the pipeline
-    try:
-        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
-    except Exception as e:
-        raise_error(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
     if not quiet:
-        typer.echo("Getting system information")
+        calkit.echo("💻 Getting system information")
     # Get system information
     system_info = calkit.get_system_info()
     if save_logs:
@@ -1453,7 +1485,7 @@ def run(
             json.dump(system_info, f, indent=2)
     # First check any system-level dependencies exist
     if not quiet:
-        typer.echo("Checking system-level dependencies")
+        calkit.echo("🔗 Checking system-level dependencies")
     try:
         calkit.check_system_deps(ck_info=ck_info, system_info=system_info)
     except Exception as e:
@@ -1461,7 +1493,7 @@ def run(
         raise_error(str(e))
     # Check all environments in the pipeline (with caching)
     # If any failed, warn the user that we might have problems running
-    typer.echo("Checking environments")
+    calkit.echo("📦 Checking environments")
     env_check_results = calkit.environments.check_all_in_pipeline(
         ck_info=ck_info, targets=targets, force=force
     )
@@ -1471,7 +1503,13 @@ def run(
         failed = not result.get("success", False)
         if failed:
             warn(f"Failed to check environment '{env_name}'")
-    # Check environments for each subproject so DVC can detect changes
+    # Clean all notebooks in the pipeline
+    try:
+        calkit.echo("📓 Cleaning notebooks")
+        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
+    except Exception as e:
+        raise_error(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
+    # Check environments and clean notebooks for each subproject
     subprojects = ck_info.get("subprojects", [])
     if subprojects:
         prev_cwd = os.getcwd()
@@ -1481,11 +1519,13 @@ def run(
             sp = Path(subproject["path"]).as_posix()
             if not os.path.isdir(sp):
                 continue
-            if not quiet:
-                typer.echo(f"Checking environments for subproject: {sp}")
             os.chdir(sp)
             try:
                 sp_ck_info = calkit.load_calkit_info()
+                if not quiet:
+                    calkit.echo(
+                        f"📦 Checking environments for subproject: {sp}"
+                    )
                 sp_env_results = calkit.environments.check_all_in_pipeline(
                     ck_info=sp_ck_info, force=force
                 )
@@ -1497,9 +1537,12 @@ def run(
                             f"Failed to check environment '{env_name}' "
                             f"in subproject '{sp}'"
                         )
+                if not quiet:
+                    calkit.echo(f"📓 Cleaning notebooks for subproject: {sp}")
+                calkit.notebooks.clean_all_in_pipeline(ck_info=sp_ck_info)
             except Exception as e:
                 warn(
-                    f"Failed to check environments for subproject '{sp}': "
+                    f"Failed to prepare subproject '{sp}': "
                     f"{e.__class__.__name__}: {e}"
                 )
             finally:
@@ -1511,7 +1554,7 @@ def run(
     dvc_stages = None
     if ck_info.get("pipeline", {}) or ck_info.get("subprojects"):
         if not quiet:
-            typer.echo("Compiling DVC pipeline")
+            calkit.echo("🔀 Compiling DVC pipeline")
         try:
             dvc_stages = calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
         except Exception as e:

{calkit_python-0.39.0 → calkit_python-0.39.2}/calkit/conda.py

@@ -7,6 +7,7 @@ import os
 import re
 import shutil
 import subprocess
+import sys
 import warnings
 from pathlib import Path
 from typing import cast
@@ -137,6 +138,75 @@ def _editable_package_name_from_dir(dir_path: str) -> str:
     raise ValueError(f"Could not determine package name from {dir_path}")
 
 
+def _run_pip_freeze(env_prefix: str) -> list[str]:
+    """Run pip freeze inside a conda env and return the list of packages.
+
+    Uses the env's pip executable directly (avoids ``conda run``, which can
+    touch the env directory and invalidate the stored mtime check).
+    This captures git URLs and exact refs that ``conda env export`` drops.
+    """
+    if sys.platform == "win32":
+        pip_exe = os.path.join(env_prefix, "Scripts", "pip.exe")
+    else:
+        pip_exe = os.path.join(env_prefix, "bin", "pip")
+    if not os.path.isfile(pip_exe):
+        return []
+    output = subprocess.check_output([pip_exe, "freeze"]).decode()
+    return [
+        line.strip()
+        for line in output.splitlines()
+        if line.strip() and not line.startswith("#")
+    ]
+
+
+_GIT_RE = re.compile(r"\s*@\s*git\+", re.IGNORECASE)
+
+
+def _pkg_name_from_dep(dep: str) -> str:
+    """Extract the normalized package name from a pip/conda dep string."""
+    name = _GIT_RE.split(dep)[0]
+    name = re.split(r"[@=<>]", name)[0]
+    return name.strip().lower()
+
+
+def _enrich_pip_deps_from_freeze(
+    pip_deps: list[str],
+    pip_freeze: list[str],
+) -> list[str]:
+    """Replace pip dep entries with pip freeze versions when available.
+
+    Preserves git URLs and exact refs that conda env export drops.
+    Editable installs in pip_deps are kept unchanged.
+    """
+    freeze_by_name = {
+        _pkg_name_from_dep(line): line
+        for line in pip_freeze
+        if not line.startswith("-e ") and "@ file://" not in line
+    }
+    result = []
+    for dep in pip_deps:
+        if dep.startswith("-e ") or dep.startswith("--editable "):
+            result.append(dep)
+            continue
+        name = _pkg_name_from_dep(dep)
+        result.append(freeze_by_name.get(name, dep))
+    return result
+
+
+def _normalize_git_dep_url(dep: str) -> str:
+    """Extract and normalize the git URL+ref from a dep string for comparison.
+
+    Returns the lowercased URL with .git suffix removed.
+    """
+    parts = _GIT_RE.split(dep, maxsplit=1)
+    if len(parts) < 2:
+        return ""
+    url = parts[1].strip().lower()
+    if url.endswith(".git"):
+        url = url[:-4]
+    return url.rstrip("/")
+
+
 def _check_single(
     req: str, actual: str, env_spec_dir: str, conda: bool = False
 ) -> bool:
@@ -156,11 +226,28 @@ def _check_single(
         req = os.path.join(env_spec_dir, req)
         req = _editable_package_name_from_dir(req)
         editable = True
-    # If this is a Git version, we can't check it
-    # TODO: Clone Git repos to check?
-    if "@git" in req:
-        warnings.warn(f"Cannot check Git version for {req}")
-        req = req.split("@git")[0]
+    # Handle git-based requirements — both legacy "pkg@git+url" and PEP 508
+    # "pkg @ git+url" forms.
+    req_is_git = bool(_GIT_RE.search(req))
+    actual_is_git = bool(_GIT_RE.search(actual))
+    if req_is_git and actual_is_git:
+        # Both sides have git URLs: compare normalized URL+ref directly.
+        # A short SHA in the spec matches a full SHA in the installed dep when
+        # the full SHA starts with the short one.
+        req_url = _normalize_git_dep_url(req)
+        actual_url = _normalize_git_dep_url(actual)
+        if req_url and actual_url:
+            req_base, _, req_ref = req_url.rpartition("@")
+            actual_base, _, actual_ref = actual_url.rpartition("@")
+            if req_base != actual_base:
+                return False
+            return actual_ref.startswith(req_ref) or req_ref.startswith(
+                actual_ref
+            )
+        # Fallback: compare names only
+        return _pkg_name_from_dep(req) == _pkg_name_from_dep(actual)
+    if req_is_git:
+        req = _GIT_RE.split(req)[0].strip()
     req_name = re.split("[=<>]", req)[0].strip()
     req_spec = req.removeprefix(req_name).strip().replace(" ", "")
     if "[" in req_name:
@@ -176,9 +263,16 @@ def _check_single(
             numbers_and_dots = re.match(r"^[0-9.]+$", req_spec[2:])
             if numbers_and_dots and len(req_spec.split(".")) < 3:
                 req_spec += ".*"
-    actual_name, actual_vers = re.split("[=<>]+", actual, maxsplit=1)
-    if actual_name != req_name:
+    if actual_is_git:
+        # Spec has no git URL but installed dep does; name match is sufficient
+        actual = _GIT_RE.split(actual)[0].strip()
+    actual_parts = re.split("[=<>]+", actual, maxsplit=1)
+    actual_name = actual_parts[0]
+    actual_vers = actual_parts[1] if len(actual_parts) > 1 else ""
+    if actual_name.strip().lower() != req_name.lower():
         return False
+    if req_is_git or actual_is_git:
+        return True
     actual_spec = actual.removeprefix(actual_name)
     if conda and actual_spec.startswith("="):
         actual_spec = "=" + actual_spec
@@ -328,6 +422,7 @@ def check_env(
                     os.remove(lock_to_use_for_creation)
                 lock_to_use_for_creation = None
     res = EnvCheckResult()
+    early_pip_freeze: list[str] = []
     if verbose:
         log_func("Getting conda info")
     conda_exe = find_conda_exe()
@@ -360,8 +455,10 @@ def check_env(
     prefix_orig = prefix
     if prefix is not None:
         prefix = os.path.abspath(prefix)
+        env_prefix_path = prefix
         env_check_fpath = os.path.join(prefix, "env-export.yml")
     else:
+        env_prefix_path = os.path.join(envs_dir, env_name)
         env_check_fpath = os.path.join(
             os.path.expanduser("~"),
             ".calkit",
@@ -371,6 +468,8 @@ def check_env(
     env_check_dir = os.path.dirname(env_check_fpath)
     os.makedirs(env_check_dir, exist_ok=True)
     env_spec_dir = os.path.dirname(os.path.abspath(env_fpath))
+    spec_pip_deps = _get_pip_dependency_list(env_spec["dependencies"])
+    spec_has_git_pip = any(_GIT_RE.search(d) for d in spec_pip_deps)
     # Create env export command, which will be used later
     export_cmd = [
         conda_exe,  # Mamba output is slightly different
@@ -464,8 +563,26 @@ def check_env(
             env_check["mtime"] = os.path.getmtime(
                 os.path.normpath(env_check["prefix"])
             )
-            with open(env_check_fpath, "w") as f:
-                ryaml.dump(env_check, f)
+        # If the spec has git pip deps, enrich the in-memory env_check pip
+        # section so that git refs are compared correctly during the dep check
+        # rather than falling back to name-only matching.
+        if spec_has_git_pip:
+            log_func("Running pip freeze to enrich git dep comparison")
+            try:
+                early_pip_freeze = _run_pip_freeze(env_prefix_path)
+            except Exception as e:
+                log_func(
+                    f"pip freeze failed; git dep URLs may be missing: {e}"
+                )
+            if early_pip_freeze:
+                check_pip = _get_pip_dependency_list(env_check["dependencies"])
+                enriched_check_pip = _enrich_pip_deps_from_freeze(
+                    check_pip, early_pip_freeze
+                )
+                for dep_entry in env_check["dependencies"]:
+                    if isinstance(dep_entry, dict) and "pip" in dep_entry:
+                        dep_entry["pip"] = enriched_check_pip
+                        break
         # Determine if the env matches
         env_needs_rebuild = False
         existing_conda_deps, existing_pip_deps = _split_env_dependencies(
@@ -552,12 +669,41 @@ def check_env(
                 )
     # If the env was rebuilt, export the env check
     res.env_needs_export = env_needs_export
+    # Determine whether we need pip freeze output for enriching the stored
+    # env check and lock file with exact git URLs/refs.
+    needs_pip_freeze = (
+        env_needs_export
+        or not res.env_exists
+        or res.env_needs_rebuild
+        or spec_has_git_pip
+    )
+    pip_freeze: list[str] = []
+    if needs_pip_freeze:
+        # Reuse the early freeze captured before dep check when possible so
+        # we don't run pip twice; fall back to a fresh run after a rebuild.
+        if early_pip_freeze and not res.env_needs_rebuild:
+            pip_freeze = early_pip_freeze
+        else:
+            log_func("Running pip freeze to capture git deps")
+            try:
+                pip_freeze = _run_pip_freeze(env_prefix_path)
+            except Exception as e:
+                log_func(
+                    f"pip freeze failed; git dep URLs may be missing: {e}"
+                )
     if env_needs_export:
         log_func(f"Exporting existing env to {env_check_fpath}")
         env_check = json.loads(subprocess.check_output(export_cmd).decode())
         env_check["mtime"] = os.path.getmtime(
             os.path.normpath(env_check["prefix"])
         )
+        if pip_freeze:
+            check_pip = _get_pip_dependency_list(env_check["dependencies"])
+            enriched = _enrich_pip_deps_from_freeze(check_pip, pip_freeze)
+            for dep_entry in env_check["dependencies"]:
+                if isinstance(dep_entry, dict) and "pip" in dep_entry:
+                    dep_entry["pip"] = enriched
+                    break
         with open(env_check_fpath, "w") as f:
             ryaml.dump(env_check, f)
     if lock_fpath is None:
@@ -602,8 +748,13 @@ def check_env(
                 editable_pip_deps[pkg_name] = dir_path
         export_pip_deps = _get_pip_dependency_list(env_export["dependencies"])
         if export_pip_deps:
+            # Enrich with pip freeze to preserve git URLs, then fix editable paths
+            if pip_freeze:
+                export_pip_deps = _enrich_pip_deps_from_freeze(
+                    export_pip_deps, pip_freeze
+                )
             for i, dep in enumerate(export_pip_deps):
-                dep_name = re.split("[=<>]+", dep, maxsplit=1)[0]
+                dep_name = _pkg_name_from_dep(dep)
                 if dep_name in editable_pip_deps:
                     path_rel_to_project_root = editable_pip_deps[dep_name]
                     lock_dir = os.path.dirname(lock_fpath)
@@ -613,9 +764,30 @@ def check_env(
                     export_pip_deps[i] = (
                         "-e " + Path(path_rel_to_lock).as_posix()
                     )
+            # Write the modified list back (enrichment returns a new list)
+            for dep_entry in env_export["dependencies"]:
+                if isinstance(dep_entry, dict) and "pip" in dep_entry:
+                    dep_entry["pip"] = export_pip_deps
+                    break
         out_dir = os.path.dirname(lock_fpath)
         if out_dir:
             os.makedirs(out_dir, exist_ok=True)
         with open(lock_fpath, "w") as f:
             ryaml.dump(env_export, f)
+    elif pip_freeze and spec_has_git_pip and os.path.isfile(lock_fpath):
+        # The env matched the spec so no full re-export was done, but the
+        # existing lock file may pre-date pip freeze enrichment. Update its
+        # pip section in place — no conda env export needed.
+        with open(lock_fpath) as f:
+            lock_data = ryaml.load(f) or {}
+        lock_pip = _get_pip_dependency_list(lock_data.get("dependencies", []))
+        enriched = _enrich_pip_deps_from_freeze(lock_pip, pip_freeze)
+        if enriched != lock_pip:
+            log_func("Enriching existing lock file pip section with git URLs")
+            for dep_entry in lock_data.get("dependencies", []):
+                if isinstance(dep_entry, dict) and "pip" in dep_entry:
+                    dep_entry["pip"] = enriched
+                    break
+            with open(lock_fpath, "w") as f:
+                ryaml.dump(lock_data, f)
     return res