calkit-python 0.49.3__py3-none-any.whl → 0.49.5__py3-none-any.whl

calkit/agent_skills/resolve-merge-conflicts/SKILL.md

@@ -0,0 +1,72 @@
+---
+name: resolve-merge-conflicts
+description: Resolve Git merge conflicts in a Calkit project, including
+  dvc.lock, stage run records, and pipeline outputs. Use when the user
+  invokes `/calkit:resolve-merge-conflicts`, asks to merge a branch, or a
+  merge, pull, or rebase stopped with conflicts.
+---
+
+# Resolve merge conflicts
+
+Some conflicted files record what the pipeline ran rather than what anyone
+wrote, so they're resolved per stage by Calkit, not by editing text.
+
+## Division of labor
+
+Done by Calkit, never by hand:
+
+- `dvc.lock` and `.calkit/stage-runs.json`, in the project and any
+  subproject. Don't edit them or take one side whole with
+  `git checkout --ours/--theirs`: that mixes stages, or drops the records of
+  stages that only ran on the other side.
+- Git-tracked pipeline outputs, which are checked out from the side each
+  stage's `dvc.lock` entry came from.
+- `dvc.yaml`, which is generated from `calkit.yaml`. Once `calkit.yaml` is
+  resolved, take either side (`git checkout --ours dvc.yaml`) and run
+  `calkit status`, which rewrites it.
+
+Done by you, with judgment: everything else, e.g., source code,
+`calkit.yaml`, and manuscripts. When the two sides disagree about intent,
+not just text, ask the user.
+
+## Starting a merge
+
+Prefer `calkit merge BRANCH` to `git merge`. It sets up the merge driver,
+resolves what it can, and checks out DVC-tracked outputs afterwards. Most
+stages merge without a conflict: one changed on one side only takes that
+side.
+
+## Stages changed on both sides
+
+Each needs one choice, which decides its `dvc.lock` entry, run record, and
+Git-tracked outputs together:
+
+- `ours` or `theirs`: keep that side's result.
+- `rerun`: keep neither, so `calkit run` reruns it.
+
+You have no terminal for the prompt, so pass choices as options:
+
+```sh
+calkit fix merge-conflicts --theirs fit --rerun figures
+calkit fix merge-conflicts --prefer rerun
+```
+
+Name subproject stages as `calkit check stage-runs` does, e.g.,
+`sub/dvc.yaml:stage`. Run it without options first to list the stages that
+need a choice.
+
+To choose, compare each side's inputs with the merged ones. If the merged
+inputs match one side, keep that side. If they match neither, e.g., both
+sides edited the code, the stage will rerun either way, so choose `rerun`.
+If a stage is expensive to run, or its result is shared with others, ask the
+user. During a rebase, "ours" is the branch being rebased onto, and
+"theirs" is the commit being replayed.
+
+## Finishing
+
+1. Resolve and `git add` the remaining files.
+2. For a merge, run `calkit merge --continue`, with any stage choices still
+   needed. For a rebase or cherry-pick, run `calkit fix merge-conflicts`,
+   then `git rebase --continue` or `git cherry-pick --continue`.
+3. Run `calkit status`, then `calkit run` if stages are stale.
+4. Run `calkit check stage-runs` to confirm every stage matches its run.

calkit/cli/__init__.py

@@ -19,6 +19,16 @@ def run() -> None:
     # otherwise print the *current* CLI's help instead of re-execing
     # under the requested version.
     _maybe_exec_with_version()
+    if sys.platform == "win32":
+        import io
+
+        from calkit.core import is_mintty
+
+        # mintty shows UTF-8, so emoji needn't be replaced, and output is
+        # line buffered as it would be to a terminal
+        for stream in (sys.stdout, sys.stderr):
+            if isinstance(stream, io.TextIOWrapper) and is_mintty(stream):
+                stream.reconfigure(encoding="utf-8", line_buffering=True)
     from .main import app
 
     # Completion lists commands, so needs them all

calkit/cli/core.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import os
 import re
 import subprocess
+import sys
 from enum import Enum
 from typing import Any, NoReturn
 
@@ -152,8 +153,24 @@ def echo_json(obj: Any) -> None:
     typer.echo(json.dumps(obj, default=str))
 
 
+def _echo_above_spinner(text: str, err: bool) -> bool:
+    """Print through a spinner's stand-in for the stream, if there is one.
+
+    Click writes to a Windows console's handle directly, which would put the
+    text on the spinner's line rather than above it.
+    """
+    file_proxy = sys.modules.get("rich.file_proxy")
+    stream = sys.stderr if err else sys.stdout
+    if file_proxy is None or not isinstance(stream, file_proxy.FileProxy):
+        return False
+    print(text, file=stream, flush=True)
+    return True
+
+
 def raise_error(txt: str) -> NoReturn:
-    typer.echo(typer.style("Error: " + str(txt), fg="red"), err=True)
+    text = typer.style("Error: " + str(txt), fg="red")
+    if not _echo_above_spinner(text, err=True):
+        typer.echo(text, err=True)
     raise typer.Exit(1)
 
 
@@ -164,7 +181,6 @@ def warn(txt: str, prefix: str = "Warning: ", err: bool = False):
     # defined.
     from calkit.core import encode_safe
 
-    typer.echo(
-        typer.style(prefix + encode_safe(str(txt), err=err), fg="yellow"),
-        err=err,
-    )
+    text = typer.style(prefix + encode_safe(str(txt), err=err), fg="yellow")
+    if not _echo_above_spinner(text, err=err):
+        typer.echo(text, err=err)

calkit/cli/fix.py

@@ -0,0 +1,196 @@
+"""CLI for fixing things."""
+
+from __future__ import annotations
+
+import posixpath
+import sys
+
+import click
+import typer
+from typing_extensions import Annotated
+
+from calkit.cli import AliasGroup, raise_error, warn
+
+fix_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
+
+OursOpt = Annotated[
+    list[str],
+    typer.Option(
+        "--ours",
+        "-o",
+        help="Keep our side of this stage (repeatable).",
+        show_default=False,
+    ),
+]
+TheirsOpt = Annotated[
+    list[str],
+    typer.Option(
+        "--theirs",
+        "-t",
+        help="Keep their side of this stage (repeatable).",
+        show_default=False,
+    ),
+]
+RerunOpt = Annotated[
+    list[str],
+    typer.Option(
+        "--rerun",
+        "-r",
+        help="Keep neither side of this stage, so it reruns (repeatable).",
+        show_default=False,
+    ),
+]
+PreferOpt = Annotated[
+    str | None,
+    typer.Option(
+        "--prefer",
+        help="Choice for every other stage changed on both sides "
+        "(ours, theirs, or rerun).",
+    ),
+]
+
+
+def resolve_merge_conflicts(
+    ours: list[str] | None = None,
+    theirs: list[str] | None = None,
+    rerun: list[str] | None = None,
+    prefer: str | None = None,
+) -> dict:
+    """Resolve conflicts in the files Calkit merges, asking for choices.
+
+    Returns what ``calkit.merge.resolve_conflicts`` does.
+    """
+    import calkit.git
+    import calkit.merge
+
+    if prefer is not None and prefer not in calkit.merge.CHOICES:
+        raise_error(
+            f"--prefer must be one of {', '.join(calkit.merge.CHOICES)}"
+        )
+    choices: dict[str, str] = {}
+    for choice, names in [
+        (calkit.merge.OURS, ours),
+        (calkit.merge.THEIRS, theirs),
+        (calkit.merge.RERUN, rerun),
+    ]:
+        for name in names or []:
+            if name in choices and choices[name] != choice:
+                raise_error(f"Stage '{name}' was given more than one choice")
+            choices[name] = choice
+    repo = calkit.git.get_repo()
+    operation = calkit.merge.get_operation(repo)
+    if operation is None:
+        raise_error("No merge, rebase, or cherry-pick is in progress")
+    assert operation is not None
+    labels = {}
+    for side, ref in [("ours", "HEAD"), ("theirs", operation[1])]:
+        labels[side] = repo.git.log("-1", "--format=%h %s", ref)
+
+    def choose(name: str, info: dict) -> str:
+        if not sys.stdin.isatty():
+            raise_error(
+                f"Stage '{name}' changed on both sides; choose with --ours, "
+                "--theirs, --rerun, or --prefer"
+            )
+        typer.echo(f"\nStage '{name}' changed on both sides:")
+        for side in ("ours", "theirs"):
+            typer.echo(f"  {side} ({labels[side]}): {info[side]}")
+        if info["changed_deps"]:
+            typer.echo(
+                "  Inputs that differ: " + ", ".join(info["changed_deps"])
+            )
+        answer = typer.prompt(
+            "Keep [o]urs, [t]heirs, or [r]erun it",
+            type=click.Choice(["o", "t", "r"]),
+            show_choices=False,
+        )
+        return {"o": "ours", "t": "theirs", "r": "rerun"}[answer]
+
+    try:
+        result = calkit.merge.resolve_conflicts(
+            repo, choices=choices, prefer=prefer, choose=choose
+        )
+    except calkit.merge.MergeError as e:
+        raise_error(str(e))
+    unknown = sorted(set(choices) - set(result["choices"]))
+    for name in unknown:
+        warn(
+            f"Stage '{name}' wasn't changed on both sides; ignoring its choice"
+        )
+    for path in result["resolved"]:
+        typer.echo(f"Resolved {path}")
+    rerun_names = [
+        n for n, c in result["choices"].items() if c == calkit.merge.RERUN
+    ]
+    if rerun_names:
+        typer.echo(
+            "Run 'calkit run' to rerun " + ", ".join(sorted(rerun_names))
+        )
+    if result["unmerged"]:
+        typer.echo("Still in conflict:")
+        for path in result["unmerged"]:
+            typer.echo(f"  {path}")
+    return result
+
+
+def checkout_dvc_outputs() -> None:
+    """Check out DVC-tracked outputs to match dvc.lock."""
+    import calkit.dvc
+    import calkit.dvc.zip
+
+    if calkit.dvc.run_dvc_command(["checkout"]) != 0:
+        warn(
+            "Some DVC-tracked outputs couldn't be checked out; run "
+            "'calkit dvc pull' to fetch them"
+        )
+    calkit.dvc.zip.sync_all(direction="to-workspace")
+
+
+@fix_app.command(name="merge-conflicts")
+def fix_merge_conflicts(
+    ours: OursOpt = [],
+    theirs: TheirsOpt = [],
+    rerun: RerunOpt = [],
+    prefer: PreferOpt = None,
+) -> None:
+    """Resolve merge conflicts in dvc.lock and stage run records.
+
+    Each stage is merged whole. A stage changed on one side takes that side,
+    and one changed on both needs a choice, which also decides its run record
+    and Git-tracked outputs. Stages are named as in 'calkit check
+    stage-runs', e.g., 'sub/dvc.yaml:stage' for a subproject's.
+    """
+    result = resolve_merge_conflicts(
+        ours=ours, theirs=theirs, rerun=rerun, prefer=prefer
+    )
+    operation = result["operation"]
+    unmerged = result["unmerged"]
+    finish = {
+        "merge": "calkit merge --continue",
+        "squash": "calkit merge --continue",
+        "rebase": "git rebase --continue",
+        "cherry-pick": "git cherry-pick --continue",
+    }[operation]
+    # DVC can't read the pipeline while a dvc.yaml is in conflict, and
+    # 'calkit merge --continue' checks outputs out after committing anyway
+    steps = [finish]
+    if any(posixpath.basename(p) == "dvc.yaml" for p in unmerged):
+        if operation not in ("merge", "squash"):
+            steps.insert(0, "calkit dvc checkout")
+    else:
+        checkout_dvc_outputs()
+    then = " and ".join(f"'{step}'" for step in steps)
+    if unmerged:
+        raise_error(f"Resolve the rest and 'git add' them, then run {then}")
+    typer.echo(f"Run {then} to finish")
+
+
+@fix_app.command(name="storage")
+def fix_storage() -> None:
+    """Move paths to where they're declared to be stored (Git or DVC).
+
+    Same as 'calkit check storage --fix'.
+    """
+    from calkit.cli.check import check_storage
+
+    check_storage(fix=True, as_json=False)

calkit/cli/main/core.py

@@ -26,6 +26,7 @@ import typer
 from typing_extensions import Annotated, Optional
 
 import calkit
+import calkit.merge
 from calkit import (
     AUTO_IGNORE_PATHS,
     AUTO_IGNORE_PREFIXES,
@@ -54,6 +55,13 @@ from calkit.cli.config import config_app
 from calkit.cli.delete import delete_app
 from calkit.cli.describe import describe_app
 from calkit.cli.dev import dev_app
+from calkit.cli.fix import (
+    OursOpt,
+    PreferOpt,
+    RerunOpt,
+    TheirsOpt,
+    fix_app,
+)
 from calkit.cli.hub import hub_app
 from calkit.cli.import_ import import_app
 from calkit.cli.latex import latex_app
@@ -89,6 +97,7 @@ app.add_typer(import_app, name="import", help="Import objects.")
 app.add_typer(office_app, name="office", help="Work with Microsoft Office.")
 app.add_typer(update_app, name="update", help="Update objects.")
 app.add_typer(check_app, name="check", help="Check things.")
+app.add_typer(fix_app, name="fix", help="Fix things.")
 app.add_typer(latex_app, name="latex|tex", help="Work with LaTeX.")
 app.add_typer(overleaf_app, name="overleaf|ol", help="Interact with Overleaf.")
 app.add_typer(hub_app, name="hub|cloud", help="Interact with a Calkit hub.")
@@ -413,6 +422,8 @@ def clone(
         location = url.split("/")[-1].removesuffix(".git")
     typer.echo(f"Moving into repo dir: {location}")
     os.chdir(location)
+    # So Git merges dvc.lock and stage run records stage by stage
+    calkit.merge.configure_driver()
     # Setup auth for any Calkit remotes
     if not no_config_remote:
         remotes = calkit.dvc.get_remotes()
@@ -542,6 +553,18 @@ def get_status(
         if isinstance(spinner, Status):
             spinner.update(text)
 
+    def start_docker() -> None:
+        # Asked with the spinner out of the way, as in 'calkit run'
+        import calkit.docker
+
+        if isinstance(spinner, Status):
+            spinner.stop()
+        try:
+            calkit.docker.ensure_ready(interactive=True)
+        finally:
+            if isinstance(spinner, Status):
+                spinner.start()
+
     def get_dvc_data_status() -> dict:
         # Reuse what the pipeline status computed under the same DVC lock
         if (
@@ -624,15 +647,23 @@ def get_status(
     # while, e.g., when environments are checked again, so a spinner shows
     # until then. It writes to its own copy of the terminal, since what the
     # environment checks' tools print is discarded.
+    # Whether there's someone to ask is worked out before the spinner starts
+    can_ask = not as_json and calkit.core.is_tty()
     spinner: AbstractContextManager = nullcontext()
-    if not as_json and sys.stderr.isatty():
+    mintty = calkit.core.is_mintty(sys.stderr)
+    if not as_json and (sys.stderr.isatty() or mintty):
         import logging
 
         from rich.console import Console
 
-        spinner = Console(file=os.fdopen(os.dup(2), "w")).status(
-            "Checking status"
-        )
+        # mintty takes ANSI and UTF-8, though it isn't a console
+        spinner = Console(
+            file=os.fdopen(
+                os.dup(2), "w", encoding="utf-8" if mintty else None
+            ),
+            force_terminal=mintty or None,
+            legacy_windows=False if mintty else None,
+        ).status("Checking status")
         # DVC's progress bars would each be printed above the spinner, and
         # they're shown only if this is at or below their level
         logging.getLogger("dvc.progress").setLevel(logging.CRITICAL + 1)
@@ -665,6 +696,7 @@ def get_status(
                 with_storage_problems="dvc" in categories,
                 quiet_environment_checks=True,
                 progress=step,
+                start_docker=start_docker if can_ask else None,
             )
             if pipeline_status.failed_environment_checks:
                 warn(
@@ -834,6 +866,19 @@ def get_status(
         if targets:
             git_cmd += ["--"] + targets
         run_cmd(git_cmd)
+        try:
+            repo = calkit.git.get_repo()
+            unmerged = calkit.merge.get_unmerged_paths(repo)
+            operation = calkit.merge.get_operation(repo)
+        except Exception:
+            unmerged, operation = [], None
+        if operation is not None and any(
+            calkit.merge.is_merged_file(p) for p in unmerged
+        ):
+            typer.echo(
+                "Run 'calkit fix merge-conflicts' to resolve the conflicts "
+                "in dvc.lock and stage run records"
+            )
         typer.echo()
     if "dvc" in categories:
         print_sep("DVC")
@@ -1798,12 +1843,23 @@ def pull(
             dvc_args.append("-f")
     if not no_git:
         typer.echo("Git pulling")
+        # So Git merges dvc.lock and stage run records stage by stage
+        calkit.merge.configure_driver()
         try:
             git_cmd = ["git", "pull"]
             if not no_recursive and "--recurse-submodules" not in git_args:
                 git_cmd.append("--recurse-submodules")
             subprocess.check_call(git_cmd + git_args)
         except subprocess.CalledProcessError:
+            repo = calkit.git.get_repo()
+            if calkit.merge.get_operation(repo) is not None and any(
+                calkit.merge.is_merged_file(p)
+                for p in calkit.merge.get_unmerged_paths(repo)
+            ):
+                typer.echo(
+                    "Run 'calkit fix merge-conflicts' to resolve the "
+                    "conflicts in dvc.lock and stage run records"
+                )
             raise_error("Git pull failed")
     if not no_dvc:
         typer.echo("DVC pulling")
@@ -3112,6 +3168,8 @@ def run(
             subprocess.check_call(["git", "init"])
         except subprocess.CalledProcessError:
             raise_error("Failed to initialize Git repo")
+    # So Git merges dvc.lock and stage run records stage by stage
+    calkit.merge.configure_driver()
     # Set env vars
     calkit.set_env_vars(ck_info=ck_info)
     # Stages that can't run here are skipped, keeping their outputs, and
@@ -3128,12 +3186,18 @@ def run(
     # someone to ask is worked out first
     can_ask = is_tty()
     spinner: Any = None
-    if not (quiet or verbose) and sys.stderr.isatty():
+    mintty = calkit.core.is_mintty(sys.stderr)
+    if not (quiet or verbose) and (sys.stderr.isatty() or mintty):
         from rich.console import Console
 
-        spinner = Console(file=os.fdopen(os.dup(2), "w")).status(
-            "Getting ready to run"
-        )
+        # mintty takes ANSI and UTF-8, though it isn't a console
+        spinner = Console(
+            file=os.fdopen(
+                os.dup(2), "w", encoding="utf-8" if mintty else None
+            ),
+            force_terminal=mintty or None,
+            legacy_windows=False if mintty else None,
+        ).status("Getting ready to run")
     with spinner or contextlib.nullcontext():
         step("Getting system information", "💻")
         # Only the stages this run can reach are checked, so nothing else
@@ -4781,7 +4845,7 @@ def install_app(
         ok = _install.prompt_and_install(name, interactive=is_tty())
     if not ok:
         raise_error(f"Failed to install '{name}'")
-    typer.echo(f"✅ Installed '{name}'")
+    calkit.echo(f"✅ Installed '{name}'")
 
 
 @app.command(name="xproc|runproc", help="Execute a procedure.")
@@ -5046,6 +5110,133 @@ def switch_branch(
     calkit.dvc.zip.sync_all(direction="to-workspace")
 
 
+@app.command(name="merge")
+def merge(
+    branch: Annotated[
+        str | None,
+        typer.Argument(help="Branch or commit to merge into the current one."),
+    ] = None,
+    cont: Annotated[
+        bool,
+        typer.Option(
+            "--continue", help="Finish a merge stopped by conflicts."
+        ),
+    ] = False,
+    abort: Annotated[
+        bool, typer.Option("--abort", help="Abort the merge in progress.")
+    ] = False,
+    squash: Annotated[
+        bool,
+        typer.Option(
+            "--squash", help="Combine the branch's changes into one commit."
+        ),
+    ] = False,
+    message: Annotated[
+        str | None,
+        typer.Option(
+            "--message",
+            "-m",
+            help="Commit message (default: Git's for the merge or squash).",
+        ),
+    ] = None,
+    ours: OursOpt = [],
+    theirs: TheirsOpt = [],
+    rerun: RerunOpt = [],
+    prefer: PreferOpt = None,
+    git_args: Annotated[
+        list[str],
+        typer.Option("--git-arg", help="Additional Git merge args."),
+    ] = [],
+) -> None:
+    """Merge a branch, resolving dvc.lock and stage run records per stage.
+
+    A stage changed on both sides needs a choice: ours, theirs, or rerun,
+    which also decides its run record and Git-tracked outputs. DVC-tracked
+    outputs are checked out to match afterwards. Other conflicts are left
+    to resolve before 'calkit merge --continue'.
+    """
+    from calkit.cli.fix import checkout_dvc_outputs, resolve_merge_conflicts
+
+    if sum([branch is not None, cont, abort]) != 1:
+        raise_error("Give a branch, --continue, or --abort")
+    if squash and branch is None:
+        raise_error("--squash needs a branch")
+    repo = calkit.git.get_repo()
+
+    def commit() -> None:
+        cmd = ["git", "commit"] + (
+            ["-m", message] if message else ["--no-edit"]
+        )
+        if subprocess.call(cmd) != 0:
+            raise_error("Git commit failed")
+        checkout_dvc_outputs()
+
+    if abort:
+        op = calkit.merge.get_operation(repo)
+        # A squash merge has no merge to abort, only changes to reset
+        if op is not None and op[0] == "squash":
+            cmd = ["git", "reset", "--merge"]
+        else:
+            cmd = ["git", "merge", "--abort"]
+        if subprocess.call(cmd) != 0:
+            raise_error("Git merge abort failed")
+        checkout_dvc_outputs()
+        return
+    if branch is not None:
+        calkit.merge.configure_driver()
+        cmd = ["git", "merge", "--no-edit"]
+        if squash:
+            cmd.append("--squash")
+        elif message:
+            cmd += ["-m", message]
+        if subprocess.call(cmd + git_args + [branch]) == 0:
+            # A squash merge stops short of committing
+            staged = subprocess.call(["git", "diff", "--cached", "--quiet"])
+            if squash and staged:
+                commit()
+            else:
+                checkout_dvc_outputs()
+            return
+    op = calkit.merge.get_operation(repo)
+    if op is None or op[0] not in ("merge", "squash"):
+        raise_error(
+            "Git merge failed"
+            if branch is not None
+            else "No merge is in progress"
+        )
+    if any(
+        calkit.merge.is_merged_file(p)
+        for p in calkit.merge.get_unmerged_paths(repo)
+    ):
+        result = resolve_merge_conflicts(
+            ours=ours, theirs=theirs, rerun=rerun, prefer=prefer
+        )
+        unmerged = result["unmerged"]
+    else:
+        unmerged = calkit.merge.get_unmerged_paths(repo)
+        if unmerged:
+            typer.echo("Still in conflict:")
+            for path in unmerged:
+                typer.echo(f"  {path}")
+    if unmerged:
+        raise_error(
+            "Resolve the conflicts, 'git add' them, then run "
+            "'calkit merge --continue'"
+        )
+    commit()
+
+
+@app.command(name="merge-driver", hidden=True)
+def merge_driver(
+    base: Annotated[str, typer.Argument()],
+    ours: Annotated[str, typer.Argument()],
+    theirs: Annotated[str, typer.Argument()],
+    path: Annotated[str, typer.Argument()],
+) -> None:
+    """Merge a file whose content is grouped by stage, as Git's merge driver."""
+    raise typer.Exit(calkit.merge.run_driver(base, ours, theirs, path))
+
+
 @app.command(name="stash")
 def stash(
     pop: Annotated[