calkit-python 0.49.4__py3-none-any.whl → 0.50.0__py3-none-any.whl

calkit/agent_skills/resolve-merge-conflicts/SKILL.md

@@ -0,0 +1,72 @@
+---
+name: resolve-merge-conflicts
+description: Resolve Git merge conflicts in a Calkit project, including
+  dvc.lock, stage run records, and pipeline outputs. Use when the user
+  invokes `/calkit:resolve-merge-conflicts`, asks to merge a branch, or a
+  merge, pull, or rebase stopped with conflicts.
+---
+
+# Resolve merge conflicts
+
+Some conflicted files record what the pipeline ran rather than what anyone
+wrote, so they're resolved per stage by Calkit, not by editing text.
+
+## Division of labor
+
+Done by Calkit, never by hand:
+
+- `dvc.lock` and `.calkit/stage-runs.json`, in the project and any
+  subproject. Don't edit them or take one side whole with
+  `git checkout --ours/--theirs`: that mixes stages, or drops the records of
+  stages that only ran on the other side.
+- Git-tracked pipeline outputs, which are checked out from the side each
+  stage's `dvc.lock` entry came from.
+- `dvc.yaml`, which is generated from `calkit.yaml`. Once `calkit.yaml` is
+  resolved, take either side (`git checkout --ours dvc.yaml`) and run
+  `calkit status`, which rewrites it.
+
+Done by you, with judgment: everything else, e.g., source code,
+`calkit.yaml`, and manuscripts. When the two sides disagree about intent,
+not just text, ask the user.
+
+## Starting a merge
+
+Prefer `calkit merge BRANCH` to `git merge`. It sets up the merge driver,
+resolves what it can, and checks out DVC-tracked outputs afterwards. Most
+stages merge without a conflict: one changed on one side only takes that
+side.
+
+## Stages changed on both sides
+
+Each needs one choice, which decides its `dvc.lock` entry, run record, and
+Git-tracked outputs together:
+
+- `ours` or `theirs`: keep that side's result.
+- `rerun`: keep neither, so `calkit run` reruns it.
+
+You have no terminal for the prompt, so pass choices as options:
+
+```sh
+calkit fix merge-conflicts --theirs fit --rerun figures
+calkit fix merge-conflicts --prefer rerun
+```
+
+Name subproject stages as `calkit check stage-runs` does, e.g.,
+`sub/dvc.yaml:stage`. Run it without options first to list the stages that
+need a choice.
+
+To choose, compare each side's inputs with the merged ones. If the merged
+inputs match one side, keep that side. If they match neither, e.g., both
+sides edited the code, the stage will rerun either way, so choose `rerun`.
+If a stage is expensive to run, or its result is shared with others, ask the
+user. During a rebase, "ours" is the branch being rebased onto, and
+"theirs" is the commit being replayed.
+
+## Finishing
+
+1. Resolve and `git add` the remaining files.
+2. For a merge, run `calkit merge --continue`, with any stage choices still
+   needed. For a rebase or cherry-pick, run `calkit fix merge-conflicts`,
+   then `git rebase --continue` or `git cherry-pick --continue`.
+3. Run `calkit status`, then `calkit run` if stages are stale.
+4. Run `calkit check stage-runs` to confirm every stage matches its run.

calkit/cli/check.py

@@ -2466,7 +2466,9 @@ def check_storage(
     stale: set[str] | None = set()
     if any(p["stage"] for p in to_record):
         status = calkit.pipeline.get_status(
-            check_environments=False, clean_notebooks=False
+            check_environments=False,
+            clean_notebooks=False,
+            preprocess_stages=False,
         )
         if not status.errors:
             stale = {n.split("@")[0] for n in status.stale_stage_names}

calkit/cli/fix.py

@@ -0,0 +1,196 @@
+"""CLI for fixing things."""
+
+from __future__ import annotations
+
+import posixpath
+import sys
+
+import click
+import typer
+from typing_extensions import Annotated
+
+from calkit.cli import AliasGroup, raise_error, warn
+
+fix_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
+
+OursOpt = Annotated[
+    list[str],
+    typer.Option(
+        "--ours",
+        "-o",
+        help="Keep our side of this stage (repeatable).",
+        show_default=False,
+    ),
+]
+TheirsOpt = Annotated[
+    list[str],
+    typer.Option(
+        "--theirs",
+        "-t",
+        help="Keep their side of this stage (repeatable).",
+        show_default=False,
+    ),
+]
+RerunOpt = Annotated[
+    list[str],
+    typer.Option(
+        "--rerun",
+        "-r",
+        help="Keep neither side of this stage, so it reruns (repeatable).",
+        show_default=False,
+    ),
+]
+PreferOpt = Annotated[
+    str | None,
+    typer.Option(
+        "--prefer",
+        help="Choice for every other stage changed on both sides "
+        "(ours, theirs, or rerun).",
+    ),
+]
+
+
+def resolve_merge_conflicts(
+    ours: list[str] | None = None,
+    theirs: list[str] | None = None,
+    rerun: list[str] | None = None,
+    prefer: str | None = None,
+) -> dict:
+    """Resolve conflicts in the files Calkit merges, asking for choices.
+
+    Returns what ``calkit.merge.resolve_conflicts`` does.
+    """
+    import calkit.git
+    import calkit.merge
+
+    if prefer is not None and prefer not in calkit.merge.CHOICES:
+        raise_error(
+            f"--prefer must be one of {', '.join(calkit.merge.CHOICES)}"
+        )
+    choices: dict[str, str] = {}
+    for choice, names in [
+        (calkit.merge.OURS, ours),
+        (calkit.merge.THEIRS, theirs),
+        (calkit.merge.RERUN, rerun),
+    ]:
+        for name in names or []:
+            if name in choices and choices[name] != choice:
+                raise_error(f"Stage '{name}' was given more than one choice")
+            choices[name] = choice
+    repo = calkit.git.get_repo()
+    operation = calkit.merge.get_operation(repo)
+    if operation is None:
+        raise_error("No merge, rebase, or cherry-pick is in progress")
+    assert operation is not None
+    labels = {}
+    for side, ref in [("ours", "HEAD"), ("theirs", operation[1])]:
+        labels[side] = repo.git.log("-1", "--format=%h %s", ref)
+
+    def choose(name: str, info: dict) -> str:
+        if not sys.stdin.isatty():
+            raise_error(
+                f"Stage '{name}' changed on both sides; choose with --ours, "
+                "--theirs, --rerun, or --prefer"
+            )
+        typer.echo(f"\nStage '{name}' changed on both sides:")
+        for side in ("ours", "theirs"):
+            typer.echo(f"  {side} ({labels[side]}): {info[side]}")
+        if info["changed_deps"]:
+            typer.echo(
+                "  Inputs that differ: " + ", ".join(info["changed_deps"])
+            )
+        answer = typer.prompt(
+            "Keep [o]urs, [t]heirs, or [r]erun it",
+            type=click.Choice(["o", "t", "r"]),
+            show_choices=False,
+        )
+        return {"o": "ours", "t": "theirs", "r": "rerun"}[answer]
+
+    try:
+        result = calkit.merge.resolve_conflicts(
+            repo, choices=choices, prefer=prefer, choose=choose
+        )
+    except calkit.merge.MergeError as e:
+        raise_error(str(e))
+    unknown = sorted(set(choices) - set(result["choices"]))
+    for name in unknown:
+        warn(
+            f"Stage '{name}' wasn't changed on both sides; ignoring its choice"
+        )
+    for path in result["resolved"]:
+        typer.echo(f"Resolved {path}")
+    rerun_names = [
+        n for n, c in result["choices"].items() if c == calkit.merge.RERUN
+    ]
+    if rerun_names:
+        typer.echo(
+            "Run 'calkit run' to rerun " + ", ".join(sorted(rerun_names))
+        )
+    if result["unmerged"]:
+        typer.echo("Still in conflict:")
+        for path in result["unmerged"]:
+            typer.echo(f"  {path}")
+    return result
+
+
+def checkout_dvc_outputs() -> None:
+    """Check out DVC-tracked outputs to match dvc.lock."""
+    import calkit.dvc
+    import calkit.dvc.zip
+
+    if calkit.dvc.run_dvc_command(["checkout"]) != 0:
+        warn(
+            "Some DVC-tracked outputs couldn't be checked out; run "
+            "'calkit dvc pull' to fetch them"
+        )
+    calkit.dvc.zip.sync_all(direction="to-workspace")
+
+
+@fix_app.command(name="merge-conflicts")
+def fix_merge_conflicts(
+    ours: OursOpt = [],
+    theirs: TheirsOpt = [],
+    rerun: RerunOpt = [],
+    prefer: PreferOpt = None,
+) -> None:
+    """Resolve merge conflicts in dvc.lock and stage run records.
+
+    Each stage is merged whole. A stage changed on one side takes that side,
+    and one changed on both needs a choice, which also decides its run record
+    and Git-tracked outputs. Stages are named as in 'calkit check
+    stage-runs', e.g., 'sub/dvc.yaml:stage' for a subproject's.
+    """
+    result = resolve_merge_conflicts(
+        ours=ours, theirs=theirs, rerun=rerun, prefer=prefer
+    )
+    operation = result["operation"]
+    unmerged = result["unmerged"]
+    finish = {
+        "merge": "calkit merge --continue",
+        "squash": "calkit merge --continue",
+        "rebase": "git rebase --continue",
+        "cherry-pick": "git cherry-pick --continue",
+    }[operation]
+    # DVC can't read the pipeline while a dvc.yaml is in conflict, and
+    # 'calkit merge --continue' checks outputs out after committing anyway
+    steps = [finish]
+    if any(posixpath.basename(p) == "dvc.yaml" for p in unmerged):
+        if operation not in ("merge", "squash"):
+            steps.insert(0, "calkit dvc checkout")
+    else:
+        checkout_dvc_outputs()
+    then = " and ".join(f"'{step}'" for step in steps)
+    if unmerged:
+        raise_error(f"Resolve the rest and 'git add' them, then run {then}")
+    typer.echo(f"Run {then} to finish")
+
+
+@fix_app.command(name="storage")
+def fix_storage() -> None:
+    """Move paths to where they're declared to be stored (Git or DVC).
+
+    Same as 'calkit check storage --fix'.
+    """
+    from calkit.cli.check import check_storage
+
+    check_storage(fix=True, as_json=False)

calkit/cli/latex.py

@@ -561,6 +561,17 @@ def _run_latexmk(
             os.remove(fdb_path)
 
 
+@latex_app.command(name="map-components")
+def map_components(
+    tex_file: Annotated[
+        str, typer.Argument(help="The .tex file whose components to map.")
+    ],
+) -> None:
+    """Map in components a LaTeX document includes with `\\ck*` commands."""
+    for problem in calkit.latex.map_ck_components(tex_file):
+        warn(f"{problem}; it will print as ??")
+
+
 @latex_app.command(name="build")
 def build(
     tex_file: Annotated[str, typer.Argument(help="The .tex file to compile.")],
@@ -657,6 +668,10 @@ def build(
     tex_dir = os.path.dirname(tex_file) or "."
     stem = Path(tex_file).stem
     pdf_dir = output_dir if output_dir is not None else tex_dir
+    # Components the document includes with calkit.sty's commands, e.g.,
+    # \ckfigure, have to be beside it before TeX can read them
+    for problem in calkit.latex.map_ck_components(tex_file):
+        warn(f"{problem}; it will print as ??")
     if not os.path.isfile(os.path.join(pdf_dir, stem + ".pdf")):
         fdb_dir = aux_dir if aux_dir is not None else tex_dir
         fdb_fpath = os.path.join(fdb_dir, stem + ".fdb_latexmk")
@@ -766,8 +781,9 @@ _IFDEFINED_MARKUP_RE = re.compile(
 )
 # A one-argument macro's definition, up to its body's opening brace
 _ONE_ARG_MACRO_DEF_RE = re.compile(
-    r"\\(?:(?:re|provide)?newcommand\*?\s*(?:\{\s*\\([A-Za-z@]+)\s*\}"
-    r"|\\([A-Za-z@]+))\s*\[1\]|def\s*\\([A-Za-z@]+)#1)\s*(?=\{)"
+    r"\\(?:(?:(?:re)?newcommand|providecommand)\*?\s*"
+    r"(?:\{\s*\\([A-Za-z@]+)\s*\}|\\([A-Za-z@]+))\s*\[1\]"
+    r"|def\s*\\([A-Za-z@]+)#1)\s*(?=\{)"
 )
 get_diff_path = calkit.latex.get_diff_path
 _default_base_ref = calkit.latex.default_base_ref
@@ -969,6 +985,7 @@ def diff(
     generates, are fetched as they were at that revision, so each side of
     the comparison shows its own.
     """
+    import calkit.resources
 
     def stage_path(stage: dict, path: str) -> str:
         """A stage's path in the project's frame rather than its wdir's."""
@@ -1335,7 +1352,7 @@ def diff(
         which can't say what's inside, so those directories beside the
         document are included whole. Neither finds another stage's output
         DVC doesn't cache, so the pipeline's inputs for the document are
-        added too.
+        added too, as are the components the document includes.
         """
         doc_dir = Path(root, os.path.dirname(tex_file))
         pointed = [
@@ -1344,7 +1361,17 @@ def diff(
             if p.is_file()
         ]
         detected = calkit.latex.detect_inputs(tex_file, wdir=root)
-        return detected + pointed + pipeline_inputs
+        # What the revision's own calkit.sty commands include, which the
+        # stage may not depend on now, at each path TeX would try
+        _, components = calkit.latex.read_ck_usage(
+            os.path.join(root, tex_file)
+        )
+        included = [
+            path
+            for c in components
+            for path in calkit.latex.component_candidates(c)[0]
+        ]
+        return detected + pointed + included + pipeline_inputs
 
     def point_changed_figures_at_base(base_root: str, head_root: str) -> None:
         """Make the older side's changed figures refer to its own copies.
@@ -1531,6 +1558,22 @@ def diff(
         for root in prepared:
             break_verbatim_params(root)
             expand_wrapper_macros(root)
+            # Each side's own values, figures, and tables, read from its
+            # revision, or the working tree for its copy of the sources
+            calkit.latex.expand_ck_components(
+                root, "." if root == working_copy else root, tex_file
+            )
+        # \cknum in the marked-up document needs calkit.sty beside it
+        build_style = os.path.join(
+            head_root, os.path.dirname(tex_file), calkit.latex.STYLE_FNAME
+        )
+        if not os.path.isfile(build_style):
+            with open(build_style, "w", encoding="utf-8") as f:
+                f.write(
+                    calkit.resources.read_text(
+                        "latex", calkit.latex.STYLE_FNAME
+                    )
+                )
         point_changed_figures_at_base(checkouts["base"], head_root)
         _build_diff(
             base_tex_fpath=sides["base"],
@@ -1711,6 +1754,9 @@ def _build_diff(
         # chose otherwise
         if not any(a.startswith("--graphics-markup") for a in latexdiff_args):
             latexdiff_cmd.append("--graphics-markup=both")
+        # A value from calkit.sty is marked like text, so a changed one shows
+        # struck out beside its new value, rather than only as it is now
+        latexdiff_cmd.append("--append-safecmd=cknum")
         # User pass-through args come last so they can override Calkit's
         # defaults
         latexdiff_cmd += latexdiff_args
@@ -2013,7 +2059,10 @@ def to_docx(
         try:
             calkit.docx.pdf_to_docx(pdf_path, output)
         except RuntimeError as e:
-            raise_error(str(e))
+            raise_error(
+                f"{e}\nTo convert from the source instead, use "
+                "--engine libreoffice"
+            )
     elif engine == "libreoffice":
         import shlex
 
@@ -2032,6 +2081,8 @@ def to_docx(
         job = Path(source).stem + "-calkit-docx"
         build = os.path.join(calkit.latex.DOCX_BUILD_DIR, Path(source).stem)
         shutil.rmtree(build, ignore_errors=True)
+        # Under .calkit/local, which keeps itself out of Git
+        calkit.ensure_local_dir()
         os.makedirs(build)
         # The PDF build's bibliography, since TeX4ht doesn't run BibTeX
         bbl = Path(source).with_suffix(".bbl")
@@ -2169,6 +2220,32 @@ def to_docx(
     blks = calkit.latex.blocks(lines)
     paras = doc.paragraphs
     matched = calkit.latex.align([p.text for p in paras], blks)
+    # Word's PDF import drops a hyphen ending a line even when the word has
+    # it, e.g., "two-dimensional" as "twodimensional", so it goes back where
+    # the paragraph's source spells the word with it and never without
+    if engine == "word":
+        for para, blk in zip(paras, matched):
+            if blk is None or para.element is None:
+                continue
+            words = re.findall(
+                r"(?<![\w-])[^\W\d_]\w*(?:-\w+)+(?![\w-])", blk.text
+            )
+            for word in dict.fromkeys(words):
+                pattern = re.compile(
+                    r"(?<![\w-])"
+                    + "(-?)".join(re.escape(w) for w in word.split("-"))
+                    + r"(?![\w-])",
+                    re.IGNORECASE,
+                )
+                if any("" in m.groups() for m in pattern.finditer(blk.text)):
+                    continue
+                text = doc.linear_text(para.element)
+                # From the end, so earlier offsets still hold
+                for m in reversed(list(pattern.finditer(text))):
+                    for k in reversed(range(1, len(m.groups()) + 1)):
+                        if not m.group(k):
+                            doc.insert_text(para.element, m.start(k), "-")
+        paras = doc.paragraphs
     # Bookmark each anchored paragraph; a block rendering as several
     # paragraphs gets numbered suffixes
     original: dict[str, str] = {}
@@ -2251,6 +2328,26 @@ def to_docx(
         # Word lays some displays out in tables of its own, which go too
         # if all else they hold is the prose on either side, moved out.
         emu_limit = int(1.5 * 914400)
+        # A figure without a caption, in a float or not, could be any tall
+        # picture, so then none go
+        captionless, depth, captioned = False, 0, False
+        for ln in lines:
+            code = calkit.latex._strip_comments(ln.text)
+            for m in re.finditer(
+                r"\\(begin|end)\{(figure|wrapfigure|sidewaysfigure)\*?\}"
+                r"|\\(caption|includegraphics)\b",
+                code,
+            ):
+                if m.group(1) == "begin":
+                    depth, captioned = depth + 1, captioned and depth > 0
+                elif m.group(1) == "end":
+                    depth = max(0, depth - 1)
+                    if depth == 0 and not captioned:
+                        captionless = True
+                elif m.group(3) == "caption":
+                    captioned = True
+                elif depth == 0:
+                    captionless = True
         w_body = doc.doc.find(calkit.docx._tag(calkit.docx.W, "body"))
         assert w_body is not None
         w_p = calkit.docx._tag(calkit.docx.W, "p")
@@ -2263,6 +2360,11 @@ def to_docx(
             )
 
         anchored = {p.element for p, b in zip(paras, matched) if b is not None}
+        captions = {
+            p.element
+            for p, b in zip(paras, matched)
+            if b is not None and b.floating
+        }
 
         def fragment(el: Any, context: str) -> bool:
             # Never prose: anything anchored to the source, e.g., a caption
@@ -2276,7 +2378,7 @@ def to_docx(
             prose_like = (
                 len(words) >= 2
                 and calkit.latex.similarity(text, context) >= 0.5
-                and not re.search(eq_num + r"\s*$", text)
+                and not calkit.latex.equation_fragment(text)
             )
             if el in anchored or len(words) > 3 or prose_like:
                 return False
@@ -2287,7 +2389,29 @@ def to_docx(
                     "wordprocessingDrawing}extent"
                 )
             )
-            return not tall
+            if not tall:
+                return True
+            if captionless:
+                return False
+            # A tall picture is a figure where there's a caption beside it,
+            # past any other pictures, else a display drawn as one
+            children = list(w_body)
+            i = children.index(unit_of(el))
+            for step in (-1, 1):
+                k = i + step
+                while (
+                    0 <= k < len(children) and not text_of(children[k]).strip()
+                ):
+                    k += step
+                if not 0 <= k < len(children):
+                    continue
+                near = children[k]
+                if any(q in captions for q in near.iter(w_p)) or re.match(
+                    r"\s*(Figure|Fig\.|Table|Scheme|Chart)\s+[A-Z]?\.?\d",
+                    text_of(near),
+                ):
+                    return False
+            return True
 
         def unit_of(el: Any) -> Any:
             while doc._parents.get(el) is not w_body:
@@ -2690,6 +2814,7 @@ def to_docx(
         .isoformat(timespec="seconds")
         .replace(":", "-")
     )
+    calkit.ensure_local_dir()
     dirs = [calkit.latex.LOCAL_DOCX_EXPORTS_DIR]
     if log:
         dirs.append(calkit.latex.DOCX_EXPORTS_DIR)
@@ -3084,6 +3209,7 @@ def merge_docx(
         .isoformat(timespec="seconds")
         .replace(":", "-")
     )
+    calkit.ensure_local_dir()
     dirs = [calkit.latex.LOCAL_DOCX_MERGES_DIR]
     if log:
         dirs.append(calkit.latex.DOCX_MERGES_DIR)