calkit-python 0.38.4__tar.gz → 0.39.1__tar.gz

{calkit_python-0.38.4 → calkit_python-0.39.1}/AGENTS.md

@@ -21,3 +21,7 @@ Agents should never make commits to Git.
 
 Prefer tests that include multiple scenarios to comprehensively test
 a feature in one function over many different test functions.
+
+Do not write docstrings in test functions.
+
+For prose, only use one space after punctuation.

{calkit_python-0.38.4 → calkit_python-0.39.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.38.4
+Version: 0.39.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/__init__.py

@@ -1,9 +1,47 @@
+from __future__ import annotations
+
 from importlib.metadata import version as _version
+from typing import TYPE_CHECKING
 
 __version__ = _version("calkit-python")
 
 from .core import *  # noqa: F403, I001
 
+if TYPE_CHECKING:
+    from calkit import (  # noqa: F401
+        calc,
+        check,
+        cloud,
+        conda,
+        config,
+        datasets,
+        detect,
+        docker,
+        dvc,
+        environments,
+        fs,
+        git,
+        github,
+        gui,
+        invenio,
+        julia,
+        jupyter,
+        licenses,
+        magics,
+        matlab,
+        models,
+        notebooks,
+        office,
+        ops,
+        overleaf,
+        pipeline,
+        releases,
+        server,
+        templates,
+    )
+
+    from .notebooks import declare_notebook  # noqa: F401
+
 # Lazy-load submodules to speed up imports
 _SUBMODULES = {
     "git",

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/check.py

@@ -174,14 +174,13 @@ def check_reproducibility(
     wdir: str = ".", log_func: Callable | None = None
 ) -> ReproCheck:
     """Check the reproducibility of a project."""
-    import git
     from git.exc import InvalidGitRepositoryError
 
     res = dict()
     if log_func is None:
         log_func = print
     try:
-        git.Repo(wdir)
+        calkit.git.get_repo(wdir)
         res["is_git_repo"] = True
     except InvalidGitRepositoryError:
         res["is_git_repo"] = False

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/check.py

@@ -1292,9 +1292,7 @@ def check_env_vars(
                 dotenv_path=".env", key_to_set=name, value_to_set=value
             )
     # Ensure that .env is ignored by git
-    import git
-
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     if not repo.ignored(".env"):
         typer.echo("Adding .env to .gitignore")
         with open(".gitignore", "a") as f:

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/config.py

@@ -92,7 +92,6 @@ def setup_remote(
     """Setup the Calkit cloud as the default DVC remote and store a token in
     the local config.
     """
-    import git
     from git.exc import InvalidGitRepositoryError
 
     from calkit.dvc import configure_remote, set_remote_auth
@@ -113,7 +112,7 @@ def setup_remote(
     except (ValueError, RuntimeError) as e:
         raise_error(f"Failed to set up DVC remote: {e}")
     if not no_commit:
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
         repo.git.add(".dvc/config")
         if ".dvc/config" in calkit.git.get_staged_files():
             typer.echo("Committing changes to DVC config")

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/core.py

@@ -46,10 +46,25 @@ def complete_stage_names(
         ryaml = ruamel.yaml.YAML()
         with open("calkit.yaml") as f:
             info = ryaml.load(f) or {}
+        candidates: list[str] = []
         stages = info.get("pipeline", {}).get("stages", {})
+        candidates.extend(stages.keys())
+        for sp_cfg in info.get("subprojects", []):
+            if not isinstance(sp_cfg, dict) or not sp_cfg.get("path"):
+                continue
+            sp_path = sp_cfg["path"]
+            sp_calkit = os.path.join(sp_path, "calkit.yaml")
+            sp_name = os.path.basename(sp_path.rstrip("/"))
+            candidates.append(sp_name)
+            if os.path.isfile(sp_calkit):
+                with open(sp_calkit) as f:
+                    sp_info = ryaml.load(f) or {}
+                sp_stages = sp_info.get("pipeline", {}).get("stages", {})
+                for stage in sp_stages:
+                    candidates.append(f"{sp_name}:{stage}")
         return [
             CompletionItem(name)
-            for name in stages
+            for name in candidates
             if name.startswith(incomplete)
         ]
     except Exception:

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/import_.py

@@ -72,7 +72,6 @@ def import_dataset(
 
     Currently only supports datasets kept in DVC, not Git.
     """
-    import git
     import requests
     from tqdm import tqdm
 
@@ -95,7 +94,7 @@ def import_dataset(
         raise_error("A dataset already exists in this project at this path")
     elif overwrite and ds_dest_path in ds_paths:
         datasets = [ds for ds in datasets if ds["path"] != ds_dest_path]
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     # Obtain, save, and commit the .dvc file for the dataset, or if this is
     # kept in Git, just download the files and commit them here
     typer.echo("Fetching import info")
@@ -261,8 +260,6 @@ def import_environment(
     ] = False,
 ) -> None:
     """Import an environment from another project."""
-    import git
-
     raise_error(
         "This command is not yet implemented; "
         "please thumbs-up this issue if you'd like to see "
@@ -317,7 +314,7 @@ def import_environment(
     ck_info["environments"] = environments
     with open("calkit.yaml", "w") as f:
         calkit.ryaml.dump(ck_info, f)
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     repo.git.add("calkit.yaml")
     if not no_commit and calkit.git.get_staged_files():
         repo.git.commit(["-m", f"Import environment {src}"])
@@ -371,7 +368,6 @@ def import_from_zenodo(
     ] = False,
 ):
     """Import files from a Zenodo record."""
-    import git
     import requests
 
     from calkit.dvc import run_dvc_command
@@ -440,7 +436,7 @@ def import_from_zenodo(
             for chunk in resp.iter_content(chunk_size=8192):
                 f.write(chunk)
     # Decide if this should be stored in Git or DVC
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     commit_paths = []
     if storage is None:
         if calkit.get_size(dest_dir) > calkit.DVC_SIZE_THRESH_BYTES:
@@ -508,7 +504,7 @@ def import_from_zenodo(
         json.dump(imports, f, indent=2)
     # Commit if desired
     if not no_commit:
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
         repo.git.add(commit_paths)
         if repo.git.diff(commit_paths + ["--staged"]):
             repo.git.commit(

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/list.py

@@ -151,10 +151,8 @@ def list_stages(
 @list_app.command(name="remotes")
 def list_remotes():
     """List Git and DVC remotes."""
-    import git
-
     try:
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
         for remote in repo.remotes:
             typer.echo(f"(Git) {remote.name}: {remote.url}")
     except Exception as e:

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/main/core.py

@@ -125,8 +125,6 @@ def init(
     ] = False,
 ):
     """Initialize the current working directory."""
-    import git
-
     subprocess.run(["git", "init"])
     result = calkit.dvc.run_dvc_command(
         ["init"] + (["--force"] if force else [])
@@ -138,7 +136,7 @@ def init(
     if result != 0:
         raise_error("Failed to configure DVC autostage")
     # Commit the newly created .dvc directory
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     repo.git.add(".dvc")
     repo.git.commit("-m", "Initialize DVC")
     # TODO: Initialize `calkit.yaml`
@@ -316,6 +314,17 @@ def get_status(
             ),
         ),
     ] = None,
+    no_check_envs: Annotated[
+        bool,
+        typer.Option(
+            "--no-env-check",
+            help=(
+                "Skip environment checks. "
+                "Note that this may produce an inaccurate pipeline status "
+                "if materialized environments have changed."
+            ),
+        ),
+    ] = False,
     as_json: Annotated[
         bool, typer.Option("--json", help="Output status as JSON.")
     ] = False,
@@ -325,10 +334,12 @@ def get_status(
     from git.exc import InvalidGitRepositoryError
 
     ck_info = calkit.load_calkit_info()
-    # If there's anything in ck_info and this isn't a Git repo, initialize one
+    # If there's anything in ck_info and this isn't a Git repo, initialize one.
+    # Use search_parent_directories so a subproject folder inside a parent repo
+    # is discovered correctly rather than getting a new git init.
     if ck_info:
         try:
-            git.Repo()
+            calkit.git.get_repo()
         except InvalidGitRepositoryError:
             git.Repo.init()
     valid_categories = ["project", "git", "dvc", "pipeline"]
@@ -349,7 +360,7 @@ def get_status(
         pipeline_status = calkit.pipeline.get_status(
             ck_info=ck_info,
             targets=targets,
-            check_environments=True,
+            check_environments=not no_check_envs,
             clean_notebooks=True,
             compile_to_dvc=True,
         )
@@ -373,7 +384,7 @@ def get_status(
             )
         if "git" in categories:
             try:
-                repo = git.Repo()
+                repo = calkit.git.get_repo()
                 changed_files = calkit.git.get_changed_files(repo=repo)
                 staged_files = calkit.git.get_staged_files(repo=repo)
                 untracked_files = calkit.git.get_untracked_files(repo=repo)
@@ -577,7 +588,6 @@ def add(
     adding any .dvc files to Git when adding to DVC.
     """
     import dvc.repo
-    import git
     from dvc.exceptions import NotDvcRepoError
     from git.exc import InvalidGitRepositoryError
 
@@ -598,7 +608,7 @@ def add(
     if to is not None and to not in ["git", "dvc", "dvc-zip"]:
         raise_error(f"Invalid option for 'to': {to}")
     try:
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
     except InvalidGitRepositoryError:
         # Prompt user if they want to run git init here
         warn("Current directory is not a Git repo")
@@ -610,7 +620,7 @@ def add(
             subprocess.check_call(["git", "init"])
         else:
             raise_error("Not currently in a Git repo; run `calkit init` first")
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
     try:
         dvc_repo = calkit.dvc.get_dvc_repo()
     except NotDvcRepoError:
@@ -1043,10 +1053,32 @@ def pull(
             if calkit.dvc.detect_calkit_remote_type(name, url) == "http":
                 typer.echo(f"Checking authentication for DVC remote: {name}")
                 calkit.dvc.set_remote_auth(remote_name=name)
+    if (
+        not no_recursive
+        and "--recursive" not in dvc_args
+        and "-R" not in dvc_args
+    ):
+        dvc_args.append("--recursive")
     result = calkit.dvc.run_dvc_command(["pull"] + dvc_args)
     if result != 0:
         raise_error("DVC pull failed")
     calkit.dvc.zip.sync_all(direction="to-workspace")
+    if not no_recursive:
+        # Pull DVC in isolated subprojects (those with their own .dvc folder)
+        ck_info = calkit.load_calkit_info()
+        for sp in ck_info.get("subprojects", []):
+            if not isinstance(sp, dict) or not sp.get("path"):
+                continue
+            sp_path = sp["path"]
+            if not os.path.isdir(os.path.join(sp_path, ".dvc")):
+                continue
+            typer.echo(f"DVC pulling subproject: {sp_path}")
+            sp_result = calkit.dvc.run_dvc_command(
+                ["pull"] + dvc_args, cwd=sp_path
+            )
+            if sp_result != 0:
+                raise_error(f"DVC pull failed for subproject: {sp_path}")
+            calkit.dvc.zip.sync_all(direction="to-workspace", wdir=sp_path)
 
 
 @app.command(name="push")
@@ -1116,9 +1148,7 @@ def ignore(
     ] = False,
 ):
     """Ignore a file, i.e., keep it out of version control."""
-    import git
-
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     # Ensure path makes it into .gitignore as a POSIX path
     path = Path(path).as_posix()
     if repo.ignored(path):
@@ -1129,7 +1159,7 @@ def ignore(
     with open(".gitignore", "a") as f:
         f.write(txt)
     if not no_commit:
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
         repo.git.reset()
         repo.git.add(".gitignore")
         if calkit.git.get_staged_files():
@@ -1384,13 +1414,18 @@ def run(
             help="Sync with Overleaf before and after running.",
         ),
     ] = False,
+    no_push: Annotated[
+        bool,
+        typer.Option(
+            "--no-push", help="Do not push to Git and DVC after saving."
+        ),
+    ] = False,
 ) -> dict:
     """Check dependencies and run the pipeline."""
     import dvc.log
     import dvc.repo
     import dvc.repo.reproduce
     import dvc.ui
-    import git
     from dvc.cli import main as dvc_cli_main
     from git.exc import InvalidGitRepositoryError
 
@@ -1404,9 +1439,11 @@ def run(
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     ck_info = calkit.load_calkit_info()
-    # Ensure Git is initialized so DVC can be used
+    # Ensure Git is initialized so DVC can be used.
+    # Use search_parent_directories so running from a subproject folder
+    # discovers the parent repo instead of triggering a new git init.
     try:
-        git.Repo()
+        calkit.git.get_repo()
     except InvalidGitRepositoryError:
         if not quiet:
             typer.echo("Initializing Git repo")
@@ -1456,12 +1493,45 @@ def run(
         failed = not result.get("success", False)
         if failed:
             warn(f"Failed to check environment '{env_name}'")
+    # Check environments for each subproject so DVC can detect changes
+    subprojects = ck_info.get("subprojects", [])
+    if subprojects:
+        prev_cwd = os.getcwd()
+        for subproject in subprojects:
+            if not isinstance(subproject, dict) or not subproject.get("path"):
+                continue
+            sp = Path(subproject["path"]).as_posix()
+            if not os.path.isdir(sp):
+                continue
+            if not quiet:
+                typer.echo(f"Checking environments for subproject: {sp}")
+            os.chdir(sp)
+            try:
+                sp_ck_info = calkit.load_calkit_info()
+                sp_env_results = calkit.environments.check_all_in_pipeline(
+                    ck_info=sp_ck_info, force=force
+                )
+                for env_name, sp_result in sp_env_results.items():
+                    if verbose:
+                        typer.echo(f"{sp}/{env_name}: {sp_result}")
+                    if not sp_result.get("success", False):
+                        warn(
+                            f"Failed to check environment '{env_name}' "
+                            f"in subproject '{sp}'"
+                        )
+            except Exception as e:
+                warn(
+                    f"Failed to check environments for subproject '{sp}': "
+                    f"{e.__class__.__name__}: {e}"
+                )
+            finally:
+                os.chdir(prev_cwd)
     # If specified, perform initial Overleaf sync
     if sync_overleaf:
         overleaf_sync(no_commit=False, no_push=True, verbose=verbose)
-    # Compile the DVC pipeline
+    # Compile the DVC pipeline (and subproject pipelines)
     dvc_stages = None
-    if ck_info.get("pipeline", {}):
+    if ck_info.get("pipeline", {}) or ck_info.get("subprojects"):
         if not quiet:
             typer.echo("Compiling DVC pipeline")
         try:
@@ -1521,7 +1591,7 @@ def run(
             raise_error("No stages found to run")
     if save_logs:
         # Get status of Git repo before running
-        repo = git.Repo()
+        repo = calkit.git.get_repo()
         git_rev = repo.head.commit.hexsha
         try:
             git_branch = repo.active_branch.name
@@ -1538,7 +1608,9 @@ def run(
         dvc_data_status_before.pop("git", None)  # Remove git status
     if targets is None:
         targets = []
-    args = deepcopy(targets)
+    args, isolated_sp_targets = calkit.pipeline.translate_run_targets(
+        deepcopy(targets), ck_info=ck_info
+    )
     # Extract any boolean args
     for name in [
         "quiet",
@@ -1560,6 +1632,47 @@ def run(
     ]:
         if locals()[name.replace("-", "_")]:
             args.append("--" + name)
+    # Inline subprojects (no .dvc/ dir) share the parent DVC project.
+    # Tell DVC to discover all pipelines so their dvc.yaml files are included,
+    # unless the user already specified targets that include dvc.yaml paths
+    # (translated from subproject shorthand) or --all-pipelines was passed.
+    inline_subprojects = [
+        sp
+        for sp in subprojects
+        if isinstance(sp, dict)
+        and sp.get("path")
+        and not os.path.isdir(os.path.join(sp["path"], ".dvc"))
+    ]
+    translated_targets_have_dvc_yaml = any(
+        "dvc.yaml" in t for t in args if not t.startswith("--")
+    )
+    if (
+        inline_subprojects
+        and not targets
+        and not all_pipelines
+        and not translated_targets_have_dvc_yaml
+    ):
+        args.append("--all-pipelines")
+    failed = False
+    # Run isolated subproject stage targets directly inside their directories.
+    if isolated_sp_targets:
+        for sp_path, stage_name in isolated_sp_targets:
+            sp_args = [stage_name] if stage_name else []
+            if dry:
+                sp_args.append("--dry")
+            if force:
+                sp_args.append("--force")
+            original_dir = os.getcwd()
+            try:
+                os.chdir(sp_path)
+                sp_res = dvc_cli_main(["repro"] + sp_args)
+            finally:
+                os.chdir(original_dir)
+            if sp_res != 0:
+                failed = True
+        if not args or all(a.startswith("--") for a in args):
+            # Only isolated subproject stage targets were given; skip parent run
+            return {}
     if pipeline is not None:
         args += ["--pipeline", pipeline]
     if downstream is not None:
@@ -1593,7 +1706,7 @@ def run(
     # Disable other misc DVC output
     dvc.ui.ui.write = lambda *args, **kwargs: None
     res = dvc_cli_main(["repro"] + args)
-    failed = res != 0
+    failed = failed or res != 0
     # Parse log to get timing and which stages ran
     with open(log_fpath, "r") as f:
         log_content = f.read()
@@ -1687,7 +1800,7 @@ def run(
             save_message = "Run pipeline"
         if not quiet:
             typer.echo("Saving the project after successful run")
-        save(save_all=True, message=save_message)
+        save(save_all=True, message=save_message, no_push=no_push)
     # If specified, perform final Overleaf sync
     if sync_overleaf:
         overleaf_sync(
@@ -2272,8 +2385,6 @@ def run_procedure(
             return bool(value)
         return value
 
-    import git
-
     from calkit.models import Procedure
 
     ck_info = calkit.load_calkit_info(process_includes="procedures")
@@ -2285,7 +2396,7 @@ def run_procedure(
         proc = Procedure.model_validate(procs[name])
     except Exception as e:
         raise_error(f"Procedure '{name}' is invalid: {e}")
-    git_repo = git.Repo()
+    git_repo = calkit.git.get_repo()
     # Check to make sure the working tree is clean, so we know we ran the
     # committed version of the procedure
     git_status = git_repo.git.status()
@@ -2431,10 +2542,8 @@ def set_env_var(
     value: Annotated[str, typer.Argument(help="Value of the variable.")],
 ):
     """Set an environmental variable for the project in its '.env' file."""
-    import git
-
     # Ensure that .env is ignored by git
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     if not repo.ignored(".env"):
         typer.echo("Adding .env to .gitignore")
         with open(".gitignore", "a") as f:
@@ -2493,9 +2602,7 @@ def upgrade(
 @app.command(name="switch-branch")
 def switch_branch(name: Annotated[str, typer.Argument(help="Branch name.")]):
     """Switch to a different branch."""
-    import git
-
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
     if name not in repo.heads:
         typer.echo(f"Branch '{name}' does not exist; creating")
         cmd = ["-b", name]
@@ -2624,9 +2731,7 @@ def map_paths(
     Currently this is done with copying. Outputs are ensured to be ignored by
     Git.
     """
-    import git
-
-    repo = git.Repo()
+    repo = calkit.git.get_repo()
 
     def validate_and_split(mapping: str) -> tuple[str, str]:
         if "->" not in mapping:

{calkit_python-0.38.4 → calkit_python-0.39.1}/calkit/cli/main/xr.py

@@ -297,7 +297,7 @@ def execute_and_record(
         language = "latex"
         pdf_path = first_arg.removesuffix(".tex") + ".pdf"
         try:
-            repo = git.Repo(".")
+            repo = calkit.git.get_repo()
             if repo.git.ls_files(pdf_path):
                 stage["pdf_storage"] = "git"
             else:
@@ -484,7 +484,7 @@ def execute_and_record(
             )
     # Initialize git repo and get DVC paths for storage determination
     try:
-        repo = git.Repo(".")
+        repo = calkit.git.get_repo()
     except InvalidGitRepositoryError:
         repo = None
     dvc_paths = []