calkit-python 0.41.23__tar.gz → 0.41.24__tar.gz

{calkit_python-0.41.23 → calkit_python-0.41.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.23
+Version: 0.41.24
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.23 → calkit_python-0.41.24}/calkit/cli/main/core.py

@@ -59,6 +59,7 @@ from calkit.cli.notebooks import notebooks_app
 from calkit.cli.office import office_app
 from calkit.cli.overleaf import overleaf_app
 from calkit.cli.scheduler import scheduler_app
+from calkit.cli.sync import sync_app
 from calkit.cli.update import update_app
 
 app = typer.Typer(
@@ -89,6 +90,7 @@ app.add_typer(
     help="Work with a job scheduler (SLURM or PBS).",
 )
 app.add_typer(dev_app, name="dev", help="Developer tools.", hidden=True)
+app.add_typer(sync_app, name="sync", help="Sync with external systems.")
 
 
 def _to_shell_cmd(cmd: list[str]) -> str:
@@ -1157,6 +1159,12 @@ def save(
 @app.command(name="pull")
 def pull(
     no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
+    no_dvc: Annotated[
+        bool, typer.Option("--no-dvc", help="Do not pull from DVC.")
+    ] = False,
+    no_git: Annotated[
+        bool, typer.Option("--no-git", help="Do not pull from Git.")
+    ] = False,
     git_args: Annotated[
         list[str],
         typer.Option("--git-arg", help="Additional Git args."),
@@ -1181,58 +1189,62 @@ def pull(
     ] = False,
 ):
     """Pull with both Git and DVC."""
-    typer.echo("Git pulling")
     if force:
         if "-f" not in git_args and "--force" not in git_args:
             git_args.append("-f")
         if "-f" not in dvc_args and "--force" not in dvc_args:
             dvc_args.append("-f")
-    try:
-        git_cmd = ["git", "pull"]
-        if not no_recursive and "--recurse-submodules" not in git_args:
-            git_cmd.append("--recurse-submodules")
-        subprocess.check_call(git_cmd + git_args)
-    except subprocess.CalledProcessError:
-        raise_error("Git pull failed")
-    typer.echo("DVC pulling")
-    if not no_check_auth:
-        # Check that our dvc remotes all have our DVC token set for them
-        remotes = calkit.dvc.get_remotes()
-        for name, url in remotes.items():
-            if calkit.dvc.detect_calkit_remote_type(name, url) == "http":
-                typer.echo(f"Checking authentication for DVC remote: {name}")
-                calkit.dvc.set_remote_auth(remote_name=name)
-    if (
-        not no_recursive
-        and "--recursive" not in dvc_args
-        and "-R" not in dvc_args
-    ):
-        dvc_args.append("--recursive")
-    result = calkit.dvc.run_dvc_command(
-        ["pull"] + dvc_args,
-        lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
-    )
-    if result != 0:
-        raise_error("DVC pull failed")
-    calkit.dvc.zip.sync_all(direction="to-workspace")
-    if not no_recursive:
-        # Pull DVC in isolated subprojects (those with their own .dvc folder)
-        ck_info = calkit.load_calkit_info()
-        for sp in ck_info.get("subprojects", []):
-            if not isinstance(sp, dict) or not sp.get("path"):
-                continue
-            sp_path = sp["path"]
-            if not os.path.isdir(os.path.join(sp_path, ".dvc")):
-                continue
-            typer.echo(f"DVC pulling subproject: {sp_path}")
-            sp_result = calkit.dvc.run_dvc_command(
-                ["pull"] + dvc_args,
-                cwd=sp_path,
-                lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
-            )
-            if sp_result != 0:
-                raise_error(f"DVC pull failed for subproject: {sp_path}")
-            calkit.dvc.zip.sync_all(direction="to-workspace", wdir=sp_path)
+    if not no_git:
+        typer.echo("Git pulling")
+        try:
+            git_cmd = ["git", "pull"]
+            if not no_recursive and "--recurse-submodules" not in git_args:
+                git_cmd.append("--recurse-submodules")
+            subprocess.check_call(git_cmd + git_args)
+        except subprocess.CalledProcessError:
+            raise_error("Git pull failed")
+    if not no_dvc:
+        typer.echo("DVC pulling")
+        if not no_check_auth:
+            # Check that our dvc remotes all have our DVC token set for them
+            remotes = calkit.dvc.get_remotes()
+            for name, url in remotes.items():
+                if calkit.dvc.detect_calkit_remote_type(name, url) == "http":
+                    typer.echo(
+                        f"Checking authentication for DVC remote: {name}"
+                    )
+                    calkit.dvc.set_remote_auth(remote_name=name)
+        if (
+            not no_recursive
+            and "--recursive" not in dvc_args
+            and "-R" not in dvc_args
+        ):
+            dvc_args.append("--recursive")
+        result = calkit.dvc.run_dvc_command(
+            ["pull"] + dvc_args,
+            lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
+        )
+        if result != 0:
+            raise_error("DVC pull failed")
+        calkit.dvc.zip.sync_all(direction="to-workspace")
+        if not no_recursive:
+            # Pull DVC in isolated subprojects (those with their own .dvc folder)
+            ck_info = calkit.load_calkit_info()
+            for sp in ck_info.get("subprojects", []):
+                if not isinstance(sp, dict) or not sp.get("path"):
+                    continue
+                sp_path = sp["path"]
+                if not os.path.isdir(os.path.join(sp_path, ".dvc")):
+                    continue
+                typer.echo(f"DVC pulling subproject: {sp_path}")
+                sp_result = calkit.dvc.run_dvc_command(
+                    ["pull"] + dvc_args,
+                    cwd=sp_path,
+                    lock_timeout=calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT,
+                )
+                if sp_result != 0:
+                    raise_error(f"DVC pull failed in subproject: {sp_path}")
+                calkit.dvc.zip.sync_all(direction="to-workspace", wdir=sp_path)
 
 
 @app.command(name="push")
@@ -1284,16 +1296,6 @@ def push(
             raise_error("Git push failed")
 
 
-@app.command(name="sync")
-def sync(
-    no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
-):
-    """Sync the project repo by pulling and then pushing."""
-    # TODO: Walk users through merge conflicts if they arise
-    pull(no_check_auth=no_check_auth)
-    push(no_check_auth=no_check_auth)
-
-
 @app.command(name="ignore")
 def ignore(
     path: Annotated[str, typer.Argument(help="Path to ignore.")],
@@ -1685,6 +1687,29 @@ def _concurrent_scheduler_prepass(
                 )
 
 
+def _get_subproject_targets_for_run(
+    subproject_path: str,
+    targets: list[str] | None,
+    include_dvc_yaml_targets: bool = False,
+) -> tuple[bool, list[str] | None]:
+    """Return whether and which subproject stages a run target selects."""
+    if not targets:
+        return True, None
+    sp = Path(subproject_path).as_posix()
+    target_prefixes = {sp, Path(sp).name}
+    if include_dvc_yaml_targets:
+        target_prefixes.add(f"{sp}/dvc.yaml")
+    selected_stages = []
+    for target in targets:
+        target_prefix, separator, stage_name = target.partition(":")
+        if target_prefix not in target_prefixes:
+            continue
+        if not separator or not stage_name:
+            return True, None
+        selected_stages.append(stage_name)
+    return bool(selected_stages), selected_stages or None
+
+
 @app.command(name="run")
 def run(
     targets: Annotated[
@@ -1942,6 +1967,18 @@ def run(
             sp = Path(subproject["path"]).as_posix()
             if not os.path.isdir(sp):
                 continue
+            if single_item:
+                sp_selected, sp_targets = _get_subproject_targets_for_run(
+                    subproject_path=sp,
+                    targets=targets,
+                    include_dvc_yaml_targets=not os.path.isdir(
+                        os.path.join(sp, ".dvc")
+                    ),
+                )
+                if not sp_selected:
+                    continue
+            else:
+                sp_targets = None
             os.chdir(sp)
             try:
                 sp_ck_info = calkit.load_calkit_info()
@@ -1950,7 +1987,7 @@ def run(
                         f"📦 Checking environments for subproject: {sp}"
                     )
                 sp_env_results = calkit.environments.check_all_in_pipeline(
-                    ck_info=sp_ck_info, force=force
+                    ck_info=sp_ck_info, targets=sp_targets, force=force
                 )
                 for env_name, sp_result in sp_env_results.items():
                     if verbose:
@@ -2181,7 +2218,7 @@ def run(
     file_handler.setFormatter(formatter)
     dvc.log.logger.addHandler(file_handler)
     # Remove newline logging in dvc.repo.reproduce
-    dvc.repo.reproduce.logger.setLevel(logging.ERROR)
+    dvc.repo.reproduce.logger.setLevel(logging.WARNING)
     # Disable other misc DVC output
     dvc.ui.ui.write = lambda *args, **kwargs: None
     # Tell `calkit scheduler batch` to resubmit completed jobs under --force;

{calkit_python-0.41.23 → calkit_python-0.41.24}/calkit/cli/overleaf.py

@@ -13,6 +13,7 @@ from typing_extensions import Annotated
 
 import calkit
 from calkit.cli import AliasGroup, raise_error, warn
+from calkit.cli.sync import sync_app
 
 overleaf_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
@@ -362,6 +363,7 @@ def import_publication(
     sync(paths=[dest_dir], no_commit=no_commit, push_only=push_only)
 
 
+@sync_app.command(name="overleaf")
 @overleaf_app.command(name="sync")
 def sync(
     paths: Annotated[

calkit_python-0.41.24/calkit/cli/sync.py

@@ -0,0 +1,80 @@
+"""CLI for syncing."""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import typer
+from typing_extensions import Annotated
+
+import calkit
+from calkit.cli import AliasGroup, raise_error
+
+sync_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
+
+
+@sync_app.command(name="git")
+def sync_git(
+    no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
+) -> None:
+    """Sync the Git repository by pulling and then pushing."""
+    from calkit.cli.main.core import pull, push
+
+    try:
+        repo = calkit.git.get_repo()
+    except Exception:
+        raise_error("No Git repository found. Run 'git init' first.")
+    if not repo.remotes:
+        raise_error(
+            "No Git remotes configured. Add a remote with "
+            "'git remote add <name> <url>'."
+        )
+    pull(no_dvc=True, no_check_auth=no_check_auth)
+    push(no_dvc=True, no_check_auth=no_check_auth)
+
+
+@sync_app.command(name="dvc")
+def sync_dvc(
+    no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
+) -> None:
+    """Sync the DVC repository by pulling and then pushing."""
+    from calkit.cli.main.core import pull, push
+
+    try:
+        calkit.dvc.get_dvc_repo()
+    except Exception:
+        raise_error("No DVC repository found. Run 'calkit init' first.")
+    if not calkit.dvc.get_remotes():
+        raise_error(
+            "No DVC remotes configured. Add a remote with "
+            "'dvc remote add <name> <url>'."
+        )
+    pull(no_git=True, no_check_auth=no_check_auth)
+    push(no_git=True, no_check_auth=no_check_auth)
+
+
+@sync_app.command(name="all")
+def sync_all() -> None:
+    """Sync all registered systems."""
+    from calkit.cli.overleaf import sync as overleaf_sync
+
+    # Run each known target in a stable order, reporting and collecting any
+    # failures. Each target is responsible for raising a clear error if it is
+    # not configured, so users calling 'calkit sync <target>' directly get a
+    # helpful message.
+    sync_funcs: list[tuple[str, Callable[[], None]]] = [
+        ("git", sync_git),
+        ("dvc", sync_dvc),
+        ("overleaf", overleaf_sync),
+    ]
+    failures = []
+    for target_name, sync_func in sync_funcs:
+        typer.echo(f"Syncing {target_name}...")
+        try:
+            sync_func()
+        except Exception as e:
+            typer.echo(f"Failed to sync {target_name}: {e}", err=True)
+            failures.append(target_name)
+    # Exit with an error if any target failed so callers can react.
+    if failures:
+        raise typer.Exit(1)

{calkit_python-0.41.23 → calkit_python-0.41.24}/calkit/tests/cli/main/test_core.py

@@ -22,6 +22,7 @@ import calkit.cli.main
 from calkit.cli.core import complete_stage_names
 from calkit.cli.main.core import (
     _get_running_pipeline_status,
+    _get_subproject_targets_for_run,
     _prune_run_logs,
     _run_dvc_repro,
     _stage_run_info_from_log_content,
@@ -45,6 +46,52 @@ skipif_windows_mock_scheduler = pytest.mark.skipif(
 )
 
 
+@pytest.mark.parametrize(
+    ("subproject_path", "targets", "include_dvc_yaml_targets", "expected"),
+    [
+        ("sub1", None, False, (True, None)),
+        ("sub1", ["parent-stage"], False, (False, None)),
+        ("sub1", ["sub1"], False, (True, None)),
+        ("nested/sub1", ["sub1:build"], False, (True, ["build"])),
+        (
+            "nested/sub1",
+            ["nested/sub1:build"],
+            False,
+            (True, ["build"]),
+        ),
+        (
+            "nested/sub1",
+            ["nested/sub1/dvc.yaml"],
+            True,
+            (True, None),
+        ),
+        (
+            "nested/sub1",
+            ["nested/sub1/dvc.yaml:build"],
+            True,
+            (True, ["build"]),
+        ),
+        (
+            "nested/sub1",
+            ["nested/sub1/dvc.yaml:build"],
+            False,
+            (False, None),
+        ),
+    ],
+)
+def test_get_subproject_targets_for_run(
+    subproject_path, targets, include_dvc_yaml_targets, expected
+):
+    assert (
+        _get_subproject_targets_for_run(
+            subproject_path,
+            targets,
+            include_dvc_yaml_targets=include_dvc_yaml_targets,
+        )
+        == expected
+    )
+
+
 def _repo_test_file(name: str) -> Path:
     """Find a file in the repository-level ``test`` directory."""
     for parent in Path(__file__).resolve().parents:

{calkit_python-0.41.23 → calkit_python-0.41.24}/calkit/tests/cli/main/test_subprojects.py

@@ -115,6 +115,83 @@ def test_inline_subproject(tmp_dir):
         assert "v2" in f.read()
 
 
+def test_single_item_skips_unrelated_subproject_preparation(tmp_dir):
+    subprocess.check_call(["calkit", "init"])
+    os.makedirs("sub1")
+    write_ck_info(
+        "sub1/calkit.yaml",
+        {
+            "pipeline": {
+                "stages": {
+                    "subproject-only": make_stage(
+                        'echo "subproject" > subproject.txt'
+                    )
+                }
+            }
+        },
+    )
+    write_ck_info(
+        "calkit.yaml",
+        {
+            "subprojects": [{"path": "sub1"}],
+            "pipeline": {
+                "stages": {
+                    "parent-only": make_stage('echo "parent" > parent.txt')
+                }
+            },
+        },
+    )
+    result = subprocess.run(
+        ["calkit", "run", "--single-item", "parent-only"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert os.path.isfile("parent.txt")
+    assert "Checking environments for subproject" not in result.stdout
+    selected_subproject = subprocess.run(
+        ["calkit", "run", "--single-item", "sub1:subproject-only"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    assert os.path.isfile("sub1/subproject.txt")
+    assert "Checking environments for subproject: sub1" in (
+        selected_subproject.stdout
+    )
+
+    # A DVC-style inline target should still prepare the selected subproject.
+    write_ck_info(
+        "sub1/calkit.yaml",
+        {
+            "pipeline": {
+                "stages": {
+                    "subproject-only": make_stage(
+                        'echo "dvc target" > subproject.txt'
+                    )
+                }
+            }
+        },
+    )
+    calkit.pipeline.to_dvc(write=True, manage_gitignore=False)
+    selected_inline_dvc_target = subprocess.run(
+        [
+            "calkit",
+            "run",
+            "--single-item",
+            "sub1/dvc.yaml:subproject-only",
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    with open("sub1/subproject.txt") as f:
+        assert "dvc target" in f.read()
+    assert "Checking environments for subproject: sub1" in (
+        selected_inline_dvc_target.stdout
+    )
+
+
 def init_isolated_subproject(path, stages):
     os.makedirs(path, exist_ok=True)
     subprocess.check_call(["git", "init"], cwd=path)

calkit_python-0.41.24/calkit/tests/cli/test_sync.py

@@ -0,0 +1,116 @@
+"""Tests for the sync CLI commands."""
+
+from unittest.mock import MagicMock, patch
+
+from typer.testing import CliRunner
+
+from calkit.cli.main import app
+
+runner = CliRunner()
+
+
+def test_sync_help():
+    result = runner.invoke(app, ["sync", "--help"])
+    assert result.exit_code == 0
+    assert "Sync with external systems" in result.output
+    assert "git" in result.output
+    assert "dvc" in result.output
+    assert "overleaf" in result.output
+    assert "all" in result.output
+
+
+def test_sync_git_calls_pull_and_push():
+    mock_repo = MagicMock()
+    mock_repo.remotes = ["origin"]
+    with patch("calkit.git.get_repo", return_value=mock_repo):
+        with patch("calkit.cli.main.core.pull") as mock_pull:
+            with patch("calkit.cli.main.core.push") as mock_push:
+                result = runner.invoke(app, ["sync", "git"])
+                assert result.exit_code == 0
+                mock_pull.assert_called_once_with(
+                    no_dvc=True, no_check_auth=False
+                )
+                mock_push.assert_called_once_with(
+                    no_dvc=True, no_check_auth=False
+                )
+
+
+def test_sync_git_errors_when_not_initialized():
+    with patch("calkit.git.get_repo", side_effect=Exception("not a repo")):
+        result = runner.invoke(app, ["sync", "git"])
+        assert result.exit_code != 0
+        assert "No Git repository found" in result.output
+
+
+def test_sync_dvc_calls_pull_and_push():
+    with patch("calkit.dvc.get_dvc_repo"):
+        with patch("calkit.dvc.get_remotes", return_value={"origin": "url"}):
+            with patch("calkit.cli.main.core.pull") as mock_pull:
+                with patch("calkit.cli.main.core.push") as mock_push:
+                    result = runner.invoke(app, ["sync", "dvc"])
+                    assert result.exit_code == 0
+                    mock_pull.assert_called_once_with(
+                        no_git=True, no_check_auth=False
+                    )
+                    mock_push.assert_called_once_with(
+                        no_git=True, no_check_auth=False
+                    )
+
+
+def test_sync_dvc_errors_when_not_initialized():
+    with patch(
+        "calkit.dvc.get_dvc_repo", side_effect=Exception("not a dvc repo")
+    ):
+        result = runner.invoke(app, ["sync", "dvc"])
+        assert result.exit_code != 0
+        assert "No DVC repository found" in result.output
+
+
+def test_sync_all_runs_all_targets():
+    def mock_sync_git():
+        print("Mock syncing git")
+
+    def mock_sync_dvc():
+        print("Mock syncing dvc")
+
+    def mock_sync_overleaf():
+        print("Mock syncing overleaf")
+
+    with patch("calkit.cli.sync.sync_git", mock_sync_git):
+        with patch("calkit.cli.sync.sync_dvc", mock_sync_dvc):
+            with patch("calkit.cli.overleaf.sync", mock_sync_overleaf):
+                result = runner.invoke(app, ["sync", "all"])
+                assert result.exit_code == 0
+                assert "Syncing git..." in result.output
+                assert "Mock syncing git" in result.output
+                assert "Syncing dvc..." in result.output
+                assert "Mock syncing dvc" in result.output
+                assert "Syncing overleaf..." in result.output
+                assert "Mock syncing overleaf" in result.output
+
+
+def test_sync_all_reports_target_failures():
+    def mock_sync_git():
+        print("Mock syncing git")
+
+    def mock_sync_dvc():
+        raise RuntimeError("dvc is broken")
+
+    with patch("calkit.cli.sync.sync_git", mock_sync_git):
+        with patch("calkit.cli.sync.sync_dvc", mock_sync_dvc):
+            with patch("calkit.cli.overleaf.sync"):
+                result = runner.invoke(app, ["sync", "all"])
+                assert result.exit_code == 1
+                assert "Syncing git..." in result.output
+                assert "Mock syncing git" in result.output
+                assert "Syncing dvc..." in result.output
+                assert "Failed to sync dvc: dvc is broken" in result.output
+
+
+def test_sync_overleaf_is_accessible():
+    # Since we can't easily mock the entire overleaf sync environment here
+    # without duplicating test_overleaf.py, we just test that the command
+    # is registered and displays its help correctly.
+    result = runner.invoke(app, ["sync", "overleaf", "--help"])
+    assert result.exit_code == 0
+    assert "Sync folders with Overleaf" in result.output