calkit-python 0.41.21__tar.gz → 0.41.23__tar.gz

{calkit_python-0.41.21 → calkit_python-0.41.23}/AGENTS.md

@@ -34,6 +34,12 @@ a feature in one function over many different test functions.
 
 Do not write docstrings in test functions.
 
+Do not put blank lines inside function bodies; separate logical sections with
+comments instead. This only applies to blank lines we write, not ones the
+formatter inserts, e.g., ruff-format adds one after an in-function import
+block. Don't hoist imports to module scope just to avoid those; imports are
+kept inside functions to keep CLI startup fast.
+
 For prose, only use one space after punctuation.
 
 Don't overzealously split up functions just because they're long.

{calkit_python-0.41.21 → calkit_python-0.41.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.21
+Version: 0.41.23
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.21 → calkit_python-0.41.23}/calkit/cli/main/core.py

@@ -1419,6 +1419,39 @@ def _stage_run_info_from_log_content(log_content: str) -> dict:
     return res
 
 
+def _run_dvc_repro(argv: list[str]) -> int | None:
+    """Run ``dvc repro`` via the DVC CLI, tolerating teardown failures.
+
+    Returns DVC's exit code, or ``None`` if the command ran but DVC's
+    post-command teardown failed to import a module. After ``do_run`` finishes,
+    ``dvc.cli.main`` reports anonymous analytics and cleans up cached repos,
+    importing ``dvc.daemon`` and ``dvc.repo.open_repo`` only at that point. In
+    some broken or mixed installs those submodules can't be imported, so the
+    teardown raises a ``ModuleNotFoundError`` that escapes DVC entirely and
+    crashes the run with a confusing traceback once the pipeline has already
+    finished (see issue #1018). That doesn't affect the pipeline result, so
+    swallow it and signal ``None`` so the caller derives success/failure from
+    the run log instead of a lost exit code.
+
+    Only those two modules are tolerated, and deliberately so. ``dvc.cli.main``
+    runs the command inside a broad ``except Exception`` that turns any failure
+    into an exit code, so nothing from the command itself reaches here; but it
+    also imports ``dvc._debug``/``dvc.config``/``dvc.logger`` *before* that
+    block, and a broken install can fail there too. Swallowing those would leave
+    an empty run log and report a pipeline that never ran as successful, so
+    anything but a teardown module must propagate and fail loudly.
+    """
+    from dvc.cli import main as dvc_cli_main
+
+    try:
+        return int(dvc_cli_main(argv))
+    except ModuleNotFoundError as e:
+        if e.name not in ("dvc.daemon", "dvc.repo.open_repo"):
+            raise
+        warn(f"DVC post-run teardown failed and was ignored ({e})")
+        return None
+
+
 def _prune_run_logs(
     logs_dir: str, keep: int = 10, protect: str | None = None
 ) -> None:
@@ -1828,7 +1861,6 @@ def run(
     import dvc.repo
     import dvc.repo.reproduce
     import dvc.ui
-    from dvc.cli import main as dvc_cli_main
     from git.exc import InvalidGitRepositoryError
 
     import calkit.dvc.zip
@@ -2080,13 +2112,21 @@ def run(
                 with calkit.dvc.dvc_lock_timeout(
                     calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT
                 ):
-                    sp_res = dvc_cli_main(["repro"] + sp_args)
+                    sp_res = _run_dvc_repro(["repro"] + sp_args)
             finally:
                 os.chdir(original_dir)
-            if sp_res != 0:
+            # ``None`` means the stage ran but DVC's teardown failed; treat that
+            # as success since the exit code is unrecoverable here.
+            if sp_res is not None and sp_res != 0:
                 failed = True
         if not args or all(a.startswith("--") for a in args):
-            # Only isolated subproject stage targets were given; skip parent run
+            # Only isolated subproject stage targets were given; skip the parent
+            # run, but still report failure the way the parent path does below,
+            # else a failing subproject stage would exit zero.
+            os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
+            if failed:
+                raise_error("Pipeline failed")
+            calkit.echo("Pipeline completed successfully ✅")
             return {}
     if pipeline is not None:
         args += ["--pipeline", pipeline]
@@ -2157,14 +2197,21 @@ def run(
         # the instant it finishes) can all hold it momentarily. Without this,
         # such a collision aborts the whole run with "Unable to acquire lock".
         with calkit.dvc.dvc_lock_timeout(calkit.dvc.DEFAULT_RUN_LOCK_TIMEOUT):
-            res = dvc_cli_main(["repro"] + args)
+            res = _run_dvc_repro(["repro"] + args)
     finally:
         os.environ.pop("CALKIT_FORCE", None)
-    failed = failed or res != 0
     # Parse log to get timing and which stages ran
     with open(log_fpath, "r") as f:
         log_content = f.read()
         stage_run_info = _stage_run_info_from_log_content(log_content)
+    if res is None:
+        # DVC's exit code was lost to a teardown failure; fall back to the log,
+        # which records any stage that failed to reproduce.
+        failed = failed or any(
+            info.get("status") == "failed" for info in stage_run_info.values()
+        )
+    else:
+        failed = failed or res != 0
     # Zip dvc-zip outputs for stages that actually ran
     if stage_run_info:
         from calkit.models.io import PathOutput

{calkit_python-0.41.21 → calkit_python-0.41.23}/calkit/cli/overleaf.py

@@ -16,6 +16,20 @@ from calkit.cli import AliasGroup, raise_error, warn
 
 overleaf_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
+# Shown when a pull from Overleaf fails, most commonly due to an invalid or
+# expired token. Calkit authenticates only with the token in its config, but a
+# stale token cached by the OS credential manager from an earlier version can
+# still interfere, so we point users there as a fallback.
+PULL_FAILED_MESSAGE = (
+    "Failed to pull from Overleaf; "
+    "check that your Overleaf token is valid\n"
+    "Run 'calkit config get overleaf_token' and ensure that it matches one in "
+    "your Overleaf account settings (https://overleaf.com/user/settings).\n"
+    "If it looks correct but pulling still fails, delete any saved "
+    "'git.overleaf.com' credential from your OS credential manager (Windows "
+    "Credential Manager, macOS Keychain, or your Linux keyring) and try again."
+)
+
 
 def _extract_title_from_tex(tex_file_path: str) -> str | None:
     """Extract the title from a LaTeX file."""
@@ -169,8 +183,6 @@ def import_publication(
     ] = False,
 ):
     """Import a publication from an Overleaf project."""
-    import git
-
     from calkit.cli.main import ignore as git_ignore
     from calkit.cli.new import new_latex_stage
 
@@ -202,16 +214,13 @@ def import_publication(
     overleaf_dir = os.path.join(".calkit", "overleaf")
     os.makedirs(overleaf_dir, exist_ok=True)
     git_ignore(overleaf_dir, no_commit=no_commit)
-    overleaf_project_dir = os.path.join(overleaf_dir, overleaf_project_id)
-    git_clone_url = calkit.overleaf.get_git_remote_url(
-        project_id=overleaf_project_id, token=overleaf_token
-    )
+    overleaf_project_dir = calkit.overleaf.get_project_dir(overleaf_project_id)
     if os.path.isdir(overleaf_project_dir):
         warn("This Overleaf project has already been cloned; removing")
         shutil.rmtree(overleaf_project_dir)
     # Clone the Overleaf project
     typer.echo("Cloning Overleaf project")
-    git.Repo.clone_from(git_clone_url, overleaf_project_dir)
+    calkit.overleaf.clone(overleaf_project_id, overleaf_token)
     # Detect target path if not specified
     if target_path is None:
         ol_contents = os.listdir(
@@ -428,8 +437,6 @@ def sync(
     ] = False,
 ):
     """Sync folders with Overleaf."""
-    import git
-
     # TODO: We should probably ensure the pipeline isn't stale
     # Read all synced folders, fixing legacy schema if applicable
     overleaf_info = calkit.overleaf.get_sync_info(fix_legacy=True)
@@ -504,32 +511,15 @@ def sync(
                     "or use --auto-commit/-a to automatically commit them."
                 )
         # Ensure we've cloned the Overleaf project
-        overleaf_project_dir = os.path.join(
-            ".calkit", "overleaf", overleaf_project_id
+        overleaf_repo = calkit.overleaf.get_repo(
+            overleaf_project_id, str(overleaf_token)
         )
-        overleaf_remote_url = calkit.overleaf.get_git_remote_url(
-            project_id=overleaf_project_id, token=str(overleaf_token)
-        )
-        if not os.path.isdir(overleaf_project_dir):
-            overleaf_repo = git.Repo.clone_from(
-                overleaf_remote_url, to_path=overleaf_project_dir
-            )
-        else:
-            overleaf_repo = calkit.git.get_repo(overleaf_project_dir)
         # Pull the latest version in the Overleaf project
         typer.echo("Pulling the latest version from Overleaf")
-        # Ensure that our current Overleaf remote URL is correct
-        overleaf_repo.git.remote("set-url", "origin", overleaf_remote_url)
         try:
             overleaf_repo.git.pull()
         except Exception:
-            raise_error(
-                "Failed to pull from Overleaf; "
-                "check that your Overleaf token is valid\n"
-                "Run 'calkit config get overleaf_token' and ensure that "
-                "it matches one in your Overleaf account settings "
-                "(https://overleaf.com/user/settings)"
-            )
+            raise_error(PULL_FAILED_MESSAGE)
         if resolve:
             last_sync_commit = resolving_info["last_overleaf_commit"]
         else:
@@ -608,8 +598,6 @@ def get_status(
     ] = None,
 ):
     """Check the status of folders synced with Overleaf in a project."""
-    import git
-
     # Read all synced folders, fixing legacy schema if applicable
     overleaf_info = calkit.overleaf.get_sync_info(fix_legacy=False)
     if not overleaf_info:
@@ -642,32 +630,18 @@ def get_status(
         )
         wdir = path_in_project
         # Ensure we've cloned the Overleaf project
-        overleaf_project_dir = os.path.join(
-            ".calkit", "overleaf", overleaf_project_id
+        overleaf_project_dir = calkit.overleaf.get_project_dir(
+            overleaf_project_id
         )
-        overleaf_remote_url = calkit.overleaf.get_git_remote_url(
-            project_id=overleaf_project_id, token=str(overleaf_token)
+        overleaf_repo = calkit.overleaf.get_repo(
+            overleaf_project_id, str(overleaf_token)
         )
-        if not os.path.isdir(overleaf_project_dir):
-            overleaf_repo = git.Repo.clone_from(
-                overleaf_remote_url, to_path=overleaf_project_dir
-            )
-        else:
-            overleaf_repo = calkit.git.get_repo(overleaf_project_dir)
         # Pull the latest version in the Overleaf project
         typer.echo("Pulling the latest version from Overleaf")
-        # Ensure that our current Overleaf remote URL is correct
-        overleaf_repo.git.remote("set-url", "origin", overleaf_remote_url)
         try:
             overleaf_repo.git.pull()
         except Exception:
-            raise_error(
-                "Failed to pull from Overleaf; "
-                "check that your Overleaf token is valid\n"
-                "Run 'calkit config get overleaf_token' and ensure that "
-                "it matches one in your Overleaf account settings "
-                "(https://overleaf.com/user/settings)"
-            )
+            raise_error(PULL_FAILED_MESSAGE)
         last_sync_commit = sync_data.get("last_sync_commit")
         if last_sync_commit:
             commits_since = calkit.overleaf.get_commits_since_last_sync(

{calkit_python-0.41.21 → calkit_python-0.41.23}/calkit/overleaf.py

@@ -30,6 +30,69 @@ def get_git_remote_url(project_id: str, token: str) -> str:
     return f"https://git:{token}@git.overleaf.com/{project_id}"
 
 
+# Git config options that force authentication to use only the token embedded
+# in the Overleaf remote URL, never a credential saved in the operating
+# system's credential store (macOS Keychain, Windows Credential Manager, etc.).
+# An expired token can linger in that store and shadow a freshly set one, which
+# otherwise makes it impossible to recover a project after a token expires.
+_CREDENTIAL_CLONE_OPTIONS = [
+    "-c credential.helper=",
+    "-c credential.interactive=false",
+]
+
+
+def _disable_credential_store(repo: git.Repo) -> None:
+    """Configure a cloned Overleaf repo so git authenticates only with the
+    token embedded in its remote URL, ignoring the OS credential store.
+
+    Setting an empty ``credential.helper`` in the repo's local config resets
+    the list of helpers inherited from the system and global config, so no
+    stale token can be read from (or written to) the credential store.
+    """
+    repo.git.config("credential.helper", "")
+    repo.git.config("credential.interactive", "false")
+
+
+def get_project_dir(project_id: str) -> str:
+    """Return the local directory into which an Overleaf project is cloned,
+    relative to the Calkit project working directory.
+    """
+    return os.path.join(".calkit", "overleaf", project_id)
+
+
+def clone(project_id: str, token: str) -> git.Repo:
+    """Clone an Overleaf project, authenticating only with ``token``."""
+    repo = git.Repo.clone_from(
+        get_git_remote_url(project_id=project_id, token=token),
+        get_project_dir(project_id),
+        multi_options=_CREDENTIAL_CLONE_OPTIONS,
+        allow_unsafe_options=True,
+    )
+    _disable_credential_store(repo)
+    return repo
+
+
+def get_repo(project_id: str, token: str) -> git.Repo:
+    """Return the Overleaf repo for ``project_id``, cloning it if it does not
+    yet exist.
+
+    When the repo already exists, its remote URL is refreshed so a freshly set
+    token takes effect, and credential handling is reset so a stale token from
+    the OS credential store is never used.
+    """
+    dest = get_project_dir(project_id)
+    if not os.path.isdir(dest):
+        return clone(project_id, token)
+    repo = calkit.git.get_repo(dest)
+    repo.git.remote(
+        "set-url",
+        "origin",
+        get_git_remote_url(project_id=project_id, token=token),
+    )
+    _disable_credential_store(repo)
+    return repo
+
+
 def project_id_to_url(project_id: str) -> str:
     return f"https://www.overleaf.com/project/{project_id}"
 

{calkit_python-0.41.21 → calkit_python-0.41.23}/calkit/tests/cli/main/test_core.py

@@ -8,6 +8,7 @@ import sys
 from datetime import datetime
 from pathlib import Path
 from pprint import pprint
+from unittest.mock import Mock
 
 import dvc.repo
 import git
@@ -22,6 +23,7 @@ from calkit.cli.core import complete_stage_names
 from calkit.cli.main.core import (
     _get_running_pipeline_status,
     _prune_run_logs,
+    _run_dvc_repro,
     _stage_run_info_from_log_content,
     _stage_target_from_cmd,
     _to_shell_cmd,
@@ -915,6 +917,83 @@ def test_run(tmp_dir):
     assert "stage_run_info" in res
 
 
+def test_run_dvc_repro_tolerates_teardown_failure(monkeypatch, capsys):
+    import dvc.cli
+
+    # Normal invocation returns DVC's exit code unchanged
+    monkeypatch.setattr(dvc.cli, "main", Mock(return_value=7))
+    assert _run_dvc_repro(["repro", "my-stage"]) == 7
+    # DVC lazily imports ``dvc.daemon`` and ``dvc.repo.open_repo`` for
+    # analytics/cleanup only after the command has already run. On a broken
+    # install those imports fail (issue #1018), crashing the run once the
+    # pipeline has finished, so tolerate them and report ``None`` instead.
+    for missing in ["dvc.daemon", "dvc.repo.open_repo"]:
+        monkeypatch.setattr(
+            dvc.cli,
+            "main",
+            Mock(
+                side_effect=ModuleNotFoundError(
+                    f"No module named {missing!r}", name=missing
+                )
+            ),
+        )
+        assert _run_dvc_repro(["repro"]) is None
+        assert "teardown failed" in capsys.readouterr().out
+    # A module DVC imports *before* running the command (dvc._debug, dvc.config,
+    # dvc.logger) failing means the pipeline never ran at all. Swallowing that
+    # would leave an empty log and report success, so it must propagate.
+    monkeypatch.setattr(
+        dvc.cli,
+        "main",
+        Mock(
+            side_effect=ModuleNotFoundError(
+                "No module named 'dvc._debug'", name="dvc._debug"
+            )
+        ),
+    )
+    with pytest.raises(ModuleNotFoundError, match="dvc._debug"):
+        _run_dvc_repro(["repro"])
+    # Non-import errors are unexpected too -- DVC turns real command failures
+    # into an exit code -- so they must propagate rather than be masked.
+    monkeypatch.setattr(
+        dvc.cli, "main", Mock(side_effect=RuntimeError("boom"))
+    )
+    with pytest.raises(RuntimeError, match="boom"):
+        _run_dvc_repro(["repro"])
+
+
+def test_run_isolated_subproject_stage_exit_code(tmp_dir):
+    subprocess.check_call(["calkit", "init"])
+    # An isolated subproject is its own Git/DVC repo, so its stage targets run
+    # directly inside it rather than through the parent pipeline
+    os.makedirs("isolated-sp", exist_ok=True)
+    subprocess.check_call(["git", "init"], cwd="isolated-sp")
+    subprocess.check_call(["calkit", "init"], cwd="isolated-sp")
+    ck_info = calkit.load_calkit_info()
+    ck_info["subprojects"] = [{"path": "isolated-sp"}]
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    # A failing subproject stage must exit nonzero; the isolated-target path
+    # returns before the parent pipeline runs, and used to skip raising
+    with open("isolated-sp/dvc.yaml", "w") as f:
+        yaml.dump(
+            {
+                "stages": {
+                    "stage-b": {"cmd": 'python -c "import sys; sys.exit(1)"'}
+                }
+            },
+            f,
+        )
+    result = subprocess.run(["calkit", "run", "isolated-sp:stage-b"])
+    assert result.returncode != 0
+    # A passing one still exits zero
+    with open("isolated-sp/dvc.yaml", "w") as f:
+        yaml.dump(
+            {"stages": {"stage-b": {"cmd": "python -c \"print('ok')\""}}}, f
+        )
+    subprocess.check_call(["calkit", "run", "isolated-sp:stage-b"])
+
+
 def test_run_ignore_errors(tmp_dir):
     subprocess.check_call(["calkit", "init"])
     # Create a pipeline with a failing stage and an independent stage

{calkit_python-0.41.21 → calkit_python-0.41.23}/calkit/tests/cli/test_overleaf.py

@@ -409,3 +409,35 @@ def test_overleaf_sync_trailing_slash_or_space(tmp_dir):
         text=True,
     )
     assert res2.returncode == 0, f"Trailing space sync failed: {res2.stderr}"
+
+
+def test_get_repo_disables_credential_store(tmp_dir):
+    # Set up a repo to act as the Overleaf remote (in the test env the remote
+    # URL is a local directory)
+    pid = str(uuid.uuid4())
+    _make_temp_overleaf_project(pid)
+    dest = calkit.overleaf.get_project_dir(pid)
+    # A fresh clone should reset the credential helper for that repo so only
+    # the token in its remote URL is used, never a stale one from the OS
+    # credential store
+    repo = calkit.overleaf.clone(pid, "sometoken")
+    assert os.path.isdir(os.path.join(dest, ".git"))
+    assert repo.git.config(["--local", "--get", "credential.helper"]) == ""
+    assert (
+        repo.git.config(["--local", "--get", "credential.interactive"])
+        == "false"
+    )
+    # Opening an already-cloned project should refresh the remote URL so an
+    # updated token takes effect, and keep the credential store disabled
+    repo2 = calkit.overleaf.get_repo(pid, "newtoken")
+    assert repo2.git.remote(
+        ["get-url", "origin"]
+    ) == calkit.overleaf.get_git_remote_url(pid, "newtoken")
+    assert repo2.git.config(["--local", "--get", "credential.helper"]) == ""
+    # A project that has not yet been cloned should be cloned
+    pid2 = str(uuid.uuid4())
+    _make_temp_overleaf_project(pid2)
+    calkit.overleaf.get_repo(pid2, "sometoken")
+    assert os.path.isdir(
+        os.path.join(calkit.overleaf.get_project_dir(pid2), ".git")
+    )