calkit-python 0.46.0__tar.gz → 0.47.1__tar.gz

{calkit_python-0.46.0 → calkit_python-0.47.1}/.pre-commit-config.yaml

@@ -16,11 +16,12 @@ repos:
       - id: check-toml
       # The example projects' results files are stage outputs: a stage
       # writes them, and a formatted copy is a changed output that makes
-      # the stage stale, so they are left exactly as written.
+      # the stage stale, so they are left exactly as written. Likewise
+      # .calkit/env-locks, which calkit writes and compares byte for byte.
       - id: end-of-file-fixer
-        exclude: ^hub/frontend/src/client/|(^|/)renv/activate\.R$|^examples/.*/results/
+        exclude: ^hub/frontend/src/client/|(^|/)renv/activate\.R$|^examples/.*/results/|(^|/)\.calkit/env-locks/
       - id: trailing-whitespace
-        exclude: ^hub/frontend/src/client/|(^|/)renv/activate\.R$|^examples/.*/results/
+        exclude: ^hub/frontend/src/client/|(^|/)renv/activate\.R$|^examples/.*/results/|(^|/)\.calkit/env-locks/
       - id: check-yaml
         args: ["--unsafe"]
       - id: check-json

{calkit_python-0.46.0 → calkit_python-0.47.1}/AGENTS.md

@@ -65,3 +65,5 @@ respectively.
 
 Never create `helpers` or `utils` modules or packages--tight coupling and
 low cohesion architecture.
+
+Don't be verbose in comments. One sentence is usually fine.

{calkit_python-0.46.0 → calkit_python-0.47.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: calkit-python
-Version: 0.46.0
+Version: 0.47.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -26,6 +26,7 @@ Requires-Dist: arithmeval
 Requires-Dist: bibtexparser
 Requires-Dist: checksumdir
 Requires-Dist: docx2pdf
+Requires-Dist: dulwich>=1.2.5
 Requires-Dist: dvc[all]==3.67.1
 Requires-Dist: fastapi
 Requires-Dist: fsspec>=2026.4.0

{calkit_python-0.46.0 → calkit_python-0.47.1}/calkit/__init__.py

@@ -16,6 +16,7 @@ if TYPE_CHECKING:
         dependencies,
         detect,
         docker,
+        docx,
         dvc,
         environments,
         fs,
@@ -27,8 +28,10 @@ if TYPE_CHECKING:
         invenio,
         julia,
         jupyter,
+        latex,
         licenses,
         magics,
+        markdown,
         matlab,
         models,
         notebooks,
@@ -38,53 +41,60 @@ if TYPE_CHECKING:
         pipeline,
         procedures,
         provenance,
+        questions,
         releases,
         reproducibility,
         resources,
         schema,
         server,
         templates,
+        workspace,
     )
 
     from .notebooks import declare_notebook  # noqa: F401
 
 # Lazy-load submodules to speed up imports
 _SUBMODULES = {
-    "git",
-    "dvc",
-    "hub",
-    "fs",
-    "jupyter",
-    "config",
-    "models",
-    "office",
-    "templates",
-    "conda",
     "calc",
-    "procedures",
-    "provenance",
-    "reproducibility",
-    "github",
-    "invenio",
-    "releases",
-    "resources",
-    "licenses",
-    "overleaf",
-    "julia",
-    "notebooks",
-    "environments",
-    "pipeline",
-    "matlab",
+    "conda",
+    "config",
     "datasets",
     "dependencies",
     "detect",
     "docker",
+    "docx",
+    "dvc",
+    "environments",
+    "fs",
+    "git",
+    "github",
     "gui",
+    "hub",
     "install",
+    "invenio",
+    "julia",
+    "jupyter",
+    "latex",
+    "licenses",
     "magics",
+    "markdown",
+    "matlab",
+    "models",
+    "notebooks",
+    "office",
     "ops",
+    "overleaf",
+    "pipeline",
+    "procedures",
+    "provenance",
+    "questions",
+    "releases",
+    "reproducibility",
+    "resources",
     "schema",
     "server",
+    "templates",
+    "workspace",
 }
 
 

{calkit_python-0.46.0 → calkit_python-0.47.1}/calkit/cli/check.py

@@ -1517,7 +1517,9 @@ def check_docker_env(
             deps_md5s=deps_md5s,
             run_config=run_config,
         )
-        with open(arch_lock_fpath, "w") as f:
+        # newline="\n" so the file is byte-identical on every platform,
+        # rather than a Windows checkout flipping it to CRLF.
+        with open(arch_lock_fpath, "w", newline="\n") as f:
             json.dump(arch_lock, f, indent=4)
 
     current_arch = get_docker_arch()
@@ -2058,20 +2060,32 @@ def check_questions(
     json_output: Annotated[
         bool, typer.Option("--json", help="Output the report as JSON.")
     ] = False,
+    no_pipeline: Annotated[
+        bool,
+        typer.Option(
+            "--no-pipeline",
+            help="Skip asking DVC which stages are out of date, which is "
+            "the slowest part of the check.",
+        ),
+    ] = False,
 ) -> None:
-    """Check that answered questions are consistent with their evidence.
-
-    A question is stale if any of its evidence changed after the commit
-    that last edited the question, in Git history for Git-tracked outputs
-    or in dvc.lock for DVC-tracked ones. Evidence paths must exist, value
-    keys must resolve, every placeholder in the text must render, and a
-    publication label must still be present in the LaTeX source. Exits
-    with an error if any answered question is stale or broken.
+    """Check that answered questions are backed by current evidence.
+
+    Reports, worst first: evidence that isn't there (never run, never
+    pushed, or pinned to a Git ref that doesn't exist); broken references
+    (a key that doesn't resolve, a placeholder that names no evidence, a
+    label missing from the LaTeX); evidence the pipeline would rebuild;
+    and evidence from a frozen stage, or downstream of one, which nothing
+    will ever report out of date unless the citation pins a git_ref.
+
+    Evidence pinned with a git_ref is checked at that ref rather than in
+    the working tree. Exits with an error if any answered question is
+    missing evidence, broken, or out of date with the pipeline.
     """
     from calkit.questions import check_questions as _check_questions
     from calkit.questions import format_status
 
-    status = _check_questions(wdir=wdir)
+    status = _check_questions(wdir=wdir, check_pipeline=not no_pipeline)
     if json_output:
         calkit.echo(json.dumps(status.model_dump(mode="json"), indent=2))
     else:

{calkit_python-0.46.0 → calkit_python-0.47.1}/calkit/cli/latex.py

@@ -613,3 +613,430 @@ def _build_diff(
     finally:
         if not keep_tex and os.path.isfile(diff_tex):
             os.remove(diff_tex)
+
+
+@latex_app.command(name="to-docx")
+def to_docx(
+    pdf_path: Annotated[str, typer.Argument(help="Compiled PDF to export.")],
+    source: Annotated[
+        str | None,
+        typer.Option(
+            "--source",
+            help=(
+                "Main .tex file. Defaults to the pipeline stage's target, "
+                "else the .tex next to the PDF."
+            ),
+        ),
+    ] = None,
+    output: Annotated[
+        str | None,
+        typer.Option(
+            "--output",
+            "-o",
+            help="Where to write the .docx. Defaults to <pdf>-for-review.docx.",
+        ),
+    ] = None,
+    comment_only: Annotated[
+        bool,
+        typer.Option("--comment-only", help="Lock the document to comments."),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Overwrite an existing export."),
+    ] = False,
+) -> None:
+    """Export a Word copy of a LaTeX document for review.
+
+    Uses Word's own PDF import, so the copy looks like the PDF, then records
+    inside the file which source line each paragraph came from and the text
+    as sent, so ``merge-docx`` can bring edits and comments back.
+    """
+    import datetime
+    import sys
+    import uuid
+    from typing import Any
+
+    import calkit.docx
+    import calkit.git
+    import calkit.pipeline
+    from calkit.models.docx import LatexDocxExport
+    from calkit.models.pipeline import LatexStage
+
+    ck_info = calkit.load_calkit_info()
+    stages = ck_info.get("pipeline", {}).get("stages", {})
+    stage_name = calkit.pipeline.get_stage_for_output(pdf_path, ck_info)
+    # A latex stage's PDF is implied by its target, not listed as an output
+    if stage_name is None:
+        for name, stage in stages.items():
+            if isinstance(stage, dict) and stage.get("kind") == "latex":
+                pdf = LatexStage.model_validate(stage).pdf_path
+                if pdf == Path(pdf_path).as_posix():
+                    stage_name = name
+                    break
+    if source is None and stage_name is not None:
+        source = stages[stage_name].get("target_path")
+    if source is None:
+        source = Path(pdf_path).with_suffix(".tex").as_posix()
+    source = Path(source).as_posix()
+    if not os.path.isfile(source):
+        raise_error(f"Source {source} does not exist; pass --source")
+    if stage_name is not None:
+        typer.echo(f"Running stage {stage_name} to make sure the PDF is fresh")
+        subprocess.run(
+            [sys.executable, "-m", "calkit", "run", stage_name], check=True
+        )
+    if not os.path.isfile(pdf_path):
+        raise_error(f"{pdf_path} does not exist")
+    if output is None:
+        output = (
+            Path(pdf_path)
+            .with_name(Path(pdf_path).stem + "-for-review.docx")
+            .as_posix()
+        )
+    if os.path.exists(output) and not force:
+        raise_error(f"{output} already exists; use --force to overwrite it")
+    typer.echo("Converting the PDF with Word")
+    try:
+        calkit.docx.pdf_to_docx(pdf_path, output)
+    except RuntimeError as e:
+        raise_error(str(e))
+    doc = calkit.docx.Document(output)
+    lines = calkit.latex.flatten(source)
+    blks = calkit.latex.blocks(lines)
+    paras = doc.paragraphs
+    matched = calkit.latex.align([p.text for p in paras], blks)
+    # Bookmark each anchored paragraph; a block rendering as several
+    # paragraphs gets numbered suffixes
+    original: dict[str, str] = {}
+    para_for_block: dict[int, Any] = {}
+    seen: dict[str, int] = {}
+    for i, (para, blk) in enumerate(zip(paras, matched)):
+        if blk is None or para.element is None:
+            continue
+        name = calkit.latex.make_bookmark_name(blk.path, blk.lineno)
+        seen[name] = seen.get(name, 0) + 1
+        if seen[name] > 1:
+            name = f"{name}_{seen[name]}"
+        doc.add_bookmark(para.element, name, 9000 + i)
+        original[name] = para.text
+        para_for_block.setdefault(id(blk), para.element)
+    # Existing comment blocks in the source go out as Word comments
+    threads, anchors, highlights, resolved = [], [], [], []
+    for path in sorted({ln.path for ln in lines}):
+        file_lines = Path(path).read_text(encoding="utf-8").split("\n")
+        for tc in calkit.latex.parse_comments(file_lines):
+            after = tc.lineno + tc.nlines
+            blk = next(
+                (b for b in blks if b.path == path and b.lineno >= after), None
+            )
+            if blk is not None and id(blk) in para_for_block:
+                threads.append(tc.messages())
+                anchors.append(para_for_block[id(blk)])
+                highlights.append(
+                    (tc.highlight, tc.highlight_occ) if tc.highlight else None
+                )
+                resolved.append(tc.resolved)
+    doc.add_comments(threads, anchors, highlights, resolved)
+    if comment_only:
+        doc.protect("comments")
+    else:
+        doc.track_changes()
+    doc.show_all_markup()
+    rev, dirty = None, False
+    try:
+        repo = calkit.git.get_repo()
+        rev = repo.head.commit.hexsha
+        dirty = any(repo.is_dirty(path=p) for p in {ln.path for ln in lines})
+    except Exception:
+        pass
+    export_id = str(uuid.uuid4())
+    doc.write_original(
+        calkit.docx.Original(
+            export_id, rev, source, original, doc.media_hashes
+        )
+    )
+    doc.set_identifier(f"calkit-latex-export:{export_id}:{rev or ''}:{source}")
+    doc.save()
+    record = LatexDocxExport(
+        id=export_id,
+        created=datetime.datetime.now(datetime.timezone.utc),
+        source=source,
+        pdf=Path(pdf_path).as_posix(),
+        docx=Path(output).as_posix(),
+        rev=rev,
+        dirty=dirty,
+        permission="comment" if comment_only else "suggest",
+        paragraphs=len(original),
+        unanchored=sum(
+            1 for p, m in zip(paras, matched) if m is None and p.text
+        ),
+        comments_exported=len(threads),
+        files={
+            p: "md5:" + calkit.get_md5(p)
+            for p in sorted(
+                {ln.path for ln in lines}
+                | {Path(pdf_path).as_posix(), Path(output).as_posix()}
+            )
+        },
+    )
+    os.makedirs(calkit.latex.DOCX_EXPORTS_DIR, exist_ok=True)
+    with open(
+        os.path.join(calkit.latex.DOCX_EXPORTS_DIR, f"{export_id}.json"),
+        "w",
+        encoding="utf-8",
+        newline="\n",
+    ) as f:
+        f.write(record.model_dump_json(indent=2))
+    typer.echo(
+        f"Wrote {output} ({record.paragraphs} paragraphs anchored, "
+        f"{record.unanchored} not, {len(threads)} comments)"
+    )
+
+
+@latex_app.command(name="merge-docx")
+def merge_docx(
+    docx_path: Annotated[str, typer.Argument(help="Reviewed .docx to merge.")],
+    no_comments: Annotated[
+        bool,
+        typer.Option(
+            "--no-comments", help="Don't write comments to the .tex."
+        ),
+    ] = False,
+) -> None:
+    """Merge a reviewed Word document back into the LaTeX source.
+
+    Accepted changes are applied. Tracked changes not yet accepted or
+    rejected, and edits that no longer fit the source, are left alone with
+    a warning: deal with them in Word and merge again. Comments become
+    comment blocks above the paragraph; threads resolved in Word are
+    marked resolved.
+    """
+    import datetime
+
+    import calkit.docx
+    import calkit.git
+    from calkit.cli.core import warn
+    from calkit.models.docx import LatexDocxMerge, LatexDocxMergeChange
+
+    doc = calkit.docx.Document(docx_path)
+    original = doc.read_original()
+    if original is None:
+        raise_error(
+            f"{docx_path} was not exported by Calkit, or its metadata was "
+            "stripped by another application"
+        )
+    if not os.path.isfile(original.source):
+        raise_error(f"Source {original.source} does not exist")
+    # Figures come from the pipeline, so a picture swapped or edited in
+    # Word can't be merged
+    media = doc.media_hashes
+    changed = sorted(
+        name
+        for name in set(media) | set(original.media)
+        if media.get(name) != original.media.get(name)
+    )
+    if changed:
+        warn(
+            "Figures were changed in Word and can't be merged; edit the "
+            "pipeline instead: " + ", ".join(changed)
+        )
+    lines = calkit.latex.flatten(original.source)
+    blks = calkit.latex.blocks(lines)
+    path_for_hash = {
+        calkit.latex.make_bookmark_name(p, 0).split("_")[1]: p
+        for p in {ln.path for ln in lines}
+    }
+    edits: dict[str, list[tuple[int, int, list[str]]]] = {}
+    changes: list[LatexDocxMergeChange] = []
+    for para in doc.paragraphs:
+        if para.bookmark is None or para.bookmark not in original.paragraphs:
+            continue
+        sent = original.paragraphs[para.bookmark]
+        if para.text == sent:
+            continue
+        parts = para.bookmark.split("_")
+        path, lineno = path_for_hash.get(parts[1], ""), int(parts[2])
+        blk = calkit.latex.find_block(
+            blks, path, lineno, sent
+        ) or calkit.latex.find_block(blks, path, lineno, para.text)
+        if blk is None:
+            warn(f"Can't place an edit from {path}:{lineno}: {para.text[:60]}")
+            changes.append(
+                LatexDocxMergeChange(
+                    path=path, lineno=lineno, status="unplaced"
+                )
+            )
+            continue
+        loc = f"{blk.path}:{blk.lineno}"
+        if para.pending:
+            warn(f"Tracked change at {loc} not yet accepted or rejected")
+            changes.append(
+                LatexDocxMergeChange(
+                    path=blk.path,
+                    lineno=blk.lineno,
+                    status="pending",
+                    author=", ".join(para.authors) or None,
+                )
+            )
+            continue
+        sent_tex = calkit.latex.from_word_text(sent)
+        new_tex = calkit.latex.from_word_text(para.text)
+        if calkit.latex.already_applied(blk, sent_tex, new_tex):
+            changes.append(
+                LatexDocxMergeChange(
+                    path=blk.path, lineno=blk.lineno, status="already-applied"
+                )
+            )
+            continue
+        new_lines = calkit.latex.apply_edit(blk, sent_tex, new_tex)
+        if new_lines is None:
+            warn(
+                f"Edit at {loc} touches markup; apply it by hand: {para.text[:60]}"
+            )
+            changes.append(
+                LatexDocxMergeChange(
+                    path=blk.path, lineno=blk.lineno, status="unplaced"
+                )
+            )
+            continue
+        edits.setdefault(blk.path, []).append(
+            (blk.lineno, len(blk.lines), new_lines)
+        )
+        changes.append(
+            LatexDocxMergeChange(
+                path=blk.path, lineno=blk.lineno, status="applied"
+            )
+        )
+        typer.echo(f"Applied edit at {loc}")
+    files = {
+        p: Path(p).read_text(encoding="utf-8").split("\n")
+        for p in {ln.path for ln in lines}
+    }
+    for path, updates in edits.items():
+        for lineno, count, new_lines in sorted(updates, reverse=True):
+            files[path][lineno - 1 : lineno - 1 + count] = new_lines
+    added = updated = 0
+    if not no_comments:
+        # Threads keyed by root, anchored through the root's bookmark
+        comments = doc.comments
+        by_id = {c.para_id: c for c in comments}
+        roots = [c for c in comments if not c.parent_id]
+        # Resolve every thread to a block first, then edit each file from
+        # the bottom up so earlier line numbers stay valid
+        placed: list[tuple[calkit.latex.Block, calkit.latex.TexComment]] = []
+        for root in roots:
+            thread = [root] + [
+                c
+                for c in comments
+                if c.parent_id and by_id.get(c.parent_id) is root
+            ]
+            tc = calkit.latex.TexComment(
+                [
+                    calkit.latex.Entry(
+                        c.author, c.text, date=calkit.latex.word_date(c.date)
+                    )
+                    for c in thread
+                ],
+                highlight=root.highlight,
+                highlight_occ=root.highlight_occ,
+                resolved=root.done,
+            )
+            if root.bookmark is None:
+                warn(
+                    f"Comment by {root.author} has no anchor: {root.text[:60]}"
+                )
+                continue
+            parts = root.bookmark.split("_")
+            path, lineno = path_for_hash.get(parts[1], ""), int(parts[2])
+            blk = calkit.latex.find_block(
+                blks, path, lineno, original.paragraphs.get(root.bookmark, "")
+            )
+            if blk is None:
+                warn(
+                    f"Can't place a comment by {root.author}: {root.text[:60]}"
+                )
+                continue
+            placed.append((blk, tc))
+        for blk, tc in sorted(
+            placed, key=lambda x: (x[0].path, x[0].lineno), reverse=True
+        ):
+            content = files[blk.path]
+            at = blk.lineno
+            existing = next(
+                (
+                    e
+                    for e in calkit.latex.parse_comments(content)
+                    if e.author == tc.author and e.text == tc.text
+                ),
+                None,
+            )
+            if existing is not None:
+                if (
+                    existing.messages() == tc.messages()
+                    and existing.resolved == tc.resolved
+                ):
+                    continue
+                # Word knows neither emails nor what the source already
+                # recorded, so carry those over for unchanged messages
+                known = {(e.author, e.text): e for e in existing.entries}
+                for e in tc.entries:
+                    old_e = known.get((e.author, e.text))
+                    if old_e is not None:
+                        e.email = old_e.email
+                        e.date = old_e.date or e.date
+                del content[
+                    existing.lineno - 1 : existing.lineno - 1 + existing.nlines
+                ]
+                if existing.lineno < at:
+                    at -= existing.nlines
+            content[at - 1 : at - 1] = tc.render()
+            added += existing is None
+            updated += existing is not None
+    for path, content in files.items():
+        new = "\n".join(content)
+        if new != Path(path).read_text(encoding="utf-8"):
+            Path(path).write_text(new, encoding="utf-8")
+    rev = None
+    try:
+        rev = calkit.git.get_repo().head.commit.hexsha
+    except Exception:
+        pass
+    seen = {a for p in doc.paragraphs for a in p.authors}
+    seen |= {c.author for c in doc.comments}
+    record = LatexDocxMerge(
+        export_id=original.id,
+        created=datetime.datetime.now(datetime.timezone.utc),
+        docx=Path(docx_path).as_posix(),
+        rev=rev,
+        authors=sorted(seen),
+        last_modified_by=doc.last_modified_by,
+        changes=changes,
+        comments_added=added,
+        comments_updated=updated,
+        files={
+            p: "md5:" + calkit.get_md5(p)
+            for p in sorted(
+                {ln.path for ln in lines} | {Path(docx_path).as_posix()}
+            )
+        },
+    )
+    os.makedirs(calkit.latex.DOCX_MERGES_DIR, exist_ok=True)
+    stamp = record.created.strftime("%Y%m%dT%H%M%S.%fZ")
+    with open(
+        os.path.join(
+            calkit.latex.DOCX_MERGES_DIR, f"{original.id}-{stamp}.json"
+        ),
+        "w",
+        encoding="utf-8",
+        newline="\n",
+    ) as f:
+        f.write(record.model_dump_json(indent=2))
+    counts = {
+        s: sum(1 for c in changes if c.status == s)
+        for s in ("applied", "already-applied", "pending", "unplaced")
+    }
+    typer.echo(
+        f"Applied {counts['applied']} edits ({counts['already-applied']} "
+        f"already there, {counts['pending']} pending, {counts['unplaced']} "
+        f"unplaced); {added} comments added, {updated} updated"
+    )