calkit-python 0.46.0__py3-none-any.whl → 0.47.0__py3-none-any.whl

calkit/__init__.py

@@ -16,6 +16,7 @@ if TYPE_CHECKING:
         dependencies,
         detect,
         docker,
+        docx,
         dvc,
         environments,
         fs,
@@ -27,8 +28,10 @@ if TYPE_CHECKING:
         invenio,
         julia,
         jupyter,
+        latex,
         licenses,
         magics,
+        markdown,
         matlab,
         models,
         notebooks,
@@ -38,53 +41,60 @@ if TYPE_CHECKING:
         pipeline,
         procedures,
         provenance,
+        questions,
         releases,
         reproducibility,
         resources,
         schema,
         server,
         templates,
+        workspace,
     )
 
     from .notebooks import declare_notebook  # noqa: F401
 
 # Lazy-load submodules to speed up imports
 _SUBMODULES = {
-    "git",
-    "dvc",
-    "hub",
-    "fs",
-    "jupyter",
-    "config",
-    "models",
-    "office",
-    "templates",
-    "conda",
     "calc",
-    "procedures",
-    "provenance",
-    "reproducibility",
-    "github",
-    "invenio",
-    "releases",
-    "resources",
-    "licenses",
-    "overleaf",
-    "julia",
-    "notebooks",
-    "environments",
-    "pipeline",
-    "matlab",
+    "conda",
+    "config",
     "datasets",
     "dependencies",
     "detect",
     "docker",
+    "docx",
+    "dvc",
+    "environments",
+    "fs",
+    "git",
+    "github",
     "gui",
+    "hub",
     "install",
+    "invenio",
+    "julia",
+    "jupyter",
+    "latex",
+    "licenses",
     "magics",
+    "markdown",
+    "matlab",
+    "models",
+    "notebooks",
+    "office",
     "ops",
+    "overleaf",
+    "pipeline",
+    "procedures",
+    "provenance",
+    "questions",
+    "releases",
+    "reproducibility",
+    "resources",
     "schema",
     "server",
+    "templates",
+    "workspace",
 }
 
 

calkit/cli/check.py

@@ -1517,7 +1517,9 @@ def check_docker_env(
             deps_md5s=deps_md5s,
             run_config=run_config,
         )
-        with open(arch_lock_fpath, "w") as f:
+        # newline="\n" so the file is byte-identical on every platform,
+        # rather than a Windows checkout flipping it to CRLF.
+        with open(arch_lock_fpath, "w", newline="\n") as f:
             json.dump(arch_lock, f, indent=4)
 
     current_arch = get_docker_arch()

calkit/cli/latex.py

@@ -613,3 +613,430 @@ def _build_diff(
     finally:
         if not keep_tex and os.path.isfile(diff_tex):
             os.remove(diff_tex)
+
+
+@latex_app.command(name="to-docx")
+def to_docx(
+    pdf_path: Annotated[str, typer.Argument(help="Compiled PDF to export.")],
+    source: Annotated[
+        str | None,
+        typer.Option(
+            "--source",
+            help=(
+                "Main .tex file. Defaults to the pipeline stage's target, "
+                "else the .tex next to the PDF."
+            ),
+        ),
+    ] = None,
+    output: Annotated[
+        str | None,
+        typer.Option(
+            "--output",
+            "-o",
+            help="Where to write the .docx. Defaults to <pdf>-for-review.docx.",
+        ),
+    ] = None,
+    comment_only: Annotated[
+        bool,
+        typer.Option("--comment-only", help="Lock the document to comments."),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Overwrite an existing export."),
+    ] = False,
+) -> None:
+    """Export a Word copy of a LaTeX document for review.
+
+    Uses Word's own PDF import, so the copy looks like the PDF, then records
+    inside the file which source line each paragraph came from and the text
+    as sent, so ``merge-docx`` can bring edits and comments back.
+    """
+    import datetime
+    import sys
+    import uuid
+    from typing import Any
+
+    import calkit.docx
+    import calkit.git
+    import calkit.pipeline
+    from calkit.models.docx import LatexDocxExport
+    from calkit.models.pipeline import LatexStage
+
+    ck_info = calkit.load_calkit_info()
+    stages = ck_info.get("pipeline", {}).get("stages", {})
+    stage_name = calkit.pipeline.get_stage_for_output(pdf_path, ck_info)
+    # A latex stage's PDF is implied by its target, not listed as an output
+    if stage_name is None:
+        for name, stage in stages.items():
+            if isinstance(stage, dict) and stage.get("kind") == "latex":
+                pdf = LatexStage.model_validate(stage).pdf_path
+                if pdf == Path(pdf_path).as_posix():
+                    stage_name = name
+                    break
+    if source is None and stage_name is not None:
+        source = stages[stage_name].get("target_path")
+    if source is None:
+        source = Path(pdf_path).with_suffix(".tex").as_posix()
+    source = Path(source).as_posix()
+    if not os.path.isfile(source):
+        raise_error(f"Source {source} does not exist; pass --source")
+    if stage_name is not None:
+        typer.echo(f"Running stage {stage_name} to make sure the PDF is fresh")
+        subprocess.run(
+            [sys.executable, "-m", "calkit", "run", stage_name], check=True
+        )
+    if not os.path.isfile(pdf_path):
+        raise_error(f"{pdf_path} does not exist")
+    if output is None:
+        output = (
+            Path(pdf_path)
+            .with_name(Path(pdf_path).stem + "-for-review.docx")
+            .as_posix()
+        )
+    if os.path.exists(output) and not force:
+        raise_error(f"{output} already exists; use --force to overwrite it")
+    typer.echo("Converting the PDF with Word")
+    try:
+        calkit.docx.pdf_to_docx(pdf_path, output)
+    except RuntimeError as e:
+        raise_error(str(e))
+    doc = calkit.docx.Document(output)
+    lines = calkit.latex.flatten(source)
+    blks = calkit.latex.blocks(lines)
+    paras = doc.paragraphs
+    matched = calkit.latex.align([p.text for p in paras], blks)
+    # Bookmark each anchored paragraph; a block rendering as several
+    # paragraphs gets numbered suffixes
+    original: dict[str, str] = {}
+    para_for_block: dict[int, Any] = {}
+    seen: dict[str, int] = {}
+    for i, (para, blk) in enumerate(zip(paras, matched)):
+        if blk is None or para.element is None:
+            continue
+        name = calkit.latex.make_bookmark_name(blk.path, blk.lineno)
+        seen[name] = seen.get(name, 0) + 1
+        if seen[name] > 1:
+            name = f"{name}_{seen[name]}"
+        doc.add_bookmark(para.element, name, 9000 + i)
+        original[name] = para.text
+        para_for_block.setdefault(id(blk), para.element)
+    # Existing comment blocks in the source go out as Word comments
+    threads, anchors, highlights, resolved = [], [], [], []
+    for path in sorted({ln.path for ln in lines}):
+        file_lines = Path(path).read_text(encoding="utf-8").split("\n")
+        for tc in calkit.latex.parse_comments(file_lines):
+            after = tc.lineno + tc.nlines
+            blk = next(
+                (b for b in blks if b.path == path and b.lineno >= after), None
+            )
+            if blk is not None and id(blk) in para_for_block:
+                threads.append(tc.messages())
+                anchors.append(para_for_block[id(blk)])
+                highlights.append(
+                    (tc.highlight, tc.highlight_occ) if tc.highlight else None
+                )
+                resolved.append(tc.resolved)
+    doc.add_comments(threads, anchors, highlights, resolved)
+    if comment_only:
+        doc.protect("comments")
+    else:
+        doc.track_changes()
+    doc.show_all_markup()
+    rev, dirty = None, False
+    try:
+        repo = calkit.git.get_repo()
+        rev = repo.head.commit.hexsha
+        dirty = any(repo.is_dirty(path=p) for p in {ln.path for ln in lines})
+    except Exception:
+        pass
+    export_id = str(uuid.uuid4())
+    doc.write_original(
+        calkit.docx.Original(
+            export_id, rev, source, original, doc.media_hashes
+        )
+    )
+    doc.set_identifier(f"calkit-latex-export:{export_id}:{rev or ''}:{source}")
+    doc.save()
+    record = LatexDocxExport(
+        id=export_id,
+        created=datetime.datetime.now(datetime.timezone.utc),
+        source=source,
+        pdf=Path(pdf_path).as_posix(),
+        docx=Path(output).as_posix(),
+        rev=rev,
+        dirty=dirty,
+        permission="comment" if comment_only else "suggest",
+        paragraphs=len(original),
+        unanchored=sum(
+            1 for p, m in zip(paras, matched) if m is None and p.text
+        ),
+        comments_exported=len(threads),
+        files={
+            p: "md5:" + calkit.get_md5(p)
+            for p in sorted(
+                {ln.path for ln in lines}
+                | {Path(pdf_path).as_posix(), Path(output).as_posix()}
+            )
+        },
+    )
+    os.makedirs(calkit.latex.DOCX_EXPORTS_DIR, exist_ok=True)
+    with open(
+        os.path.join(calkit.latex.DOCX_EXPORTS_DIR, f"{export_id}.json"),
+        "w",
+        encoding="utf-8",
+        newline="\n",
+    ) as f:
+        f.write(record.model_dump_json(indent=2))
+    typer.echo(
+        f"Wrote {output} ({record.paragraphs} paragraphs anchored, "
+        f"{record.unanchored} not, {len(threads)} comments)"
+    )
+
+
+@latex_app.command(name="merge-docx")
+def merge_docx(
+    docx_path: Annotated[str, typer.Argument(help="Reviewed .docx to merge.")],
+    no_comments: Annotated[
+        bool,
+        typer.Option(
+            "--no-comments", help="Don't write comments to the .tex."
+        ),
+    ] = False,
+) -> None:
+    """Merge a reviewed Word document back into the LaTeX source.
+
+    Accepted changes are applied. Tracked changes not yet accepted or
+    rejected, and edits that no longer fit the source, are left alone with
+    a warning: deal with them in Word and merge again. Comments become
+    comment blocks above the paragraph; threads resolved in Word are
+    marked resolved.
+    """
+    import datetime
+
+    import calkit.docx
+    import calkit.git
+    from calkit.cli.core import warn
+    from calkit.models.docx import LatexDocxMerge, LatexDocxMergeChange
+
+    doc = calkit.docx.Document(docx_path)
+    original = doc.read_original()
+    if original is None:
+        raise_error(
+            f"{docx_path} was not exported by Calkit, or its metadata was "
+            "stripped by another application"
+        )
+    if not os.path.isfile(original.source):
+        raise_error(f"Source {original.source} does not exist")
+    # Figures come from the pipeline, so a picture swapped or edited in
+    # Word can't be merged
+    media = doc.media_hashes
+    changed = sorted(
+        name
+        for name in set(media) | set(original.media)
+        if media.get(name) != original.media.get(name)
+    )
+    if changed:
+        warn(
+            "Figures were changed in Word and can't be merged; edit the "
+            "pipeline instead: " + ", ".join(changed)
+        )
+    lines = calkit.latex.flatten(original.source)
+    blks = calkit.latex.blocks(lines)
+    path_for_hash = {
+        calkit.latex.make_bookmark_name(p, 0).split("_")[1]: p
+        for p in {ln.path for ln in lines}
+    }
+    edits: dict[str, list[tuple[int, int, list[str]]]] = {}
+    changes: list[LatexDocxMergeChange] = []
+    for para in doc.paragraphs:
+        if para.bookmark is None or para.bookmark not in original.paragraphs:
+            continue
+        sent = original.paragraphs[para.bookmark]
+        if para.text == sent:
+            continue
+        parts = para.bookmark.split("_")
+        path, lineno = path_for_hash.get(parts[1], ""), int(parts[2])
+        blk = calkit.latex.find_block(
+            blks, path, lineno, sent
+        ) or calkit.latex.find_block(blks, path, lineno, para.text)
+        if blk is None:
+            warn(f"Can't place an edit from {path}:{lineno}: {para.text[:60]}")
+            changes.append(
+                LatexDocxMergeChange(
+                    path=path, lineno=lineno, status="unplaced"
+                )
+            )
+            continue
+        loc = f"{blk.path}:{blk.lineno}"
+        if para.pending:
+            warn(f"Tracked change at {loc} not yet accepted or rejected")
+            changes.append(
+                LatexDocxMergeChange(
+                    path=blk.path,
+                    lineno=blk.lineno,
+                    status="pending",
+                    author=", ".join(para.authors) or None,
+                )
+            )
+            continue
+        sent_tex = calkit.latex.from_word_text(sent)
+        new_tex = calkit.latex.from_word_text(para.text)
+        if calkit.latex.already_applied(blk, sent_tex, new_tex):
+            changes.append(
+                LatexDocxMergeChange(
+                    path=blk.path, lineno=blk.lineno, status="already-applied"
+                )
+            )
+            continue
+        new_lines = calkit.latex.apply_edit(blk, sent_tex, new_tex)
+        if new_lines is None:
+            warn(
+                f"Edit at {loc} touches markup; apply it by hand: {para.text[:60]}"
+            )
+            changes.append(
+                LatexDocxMergeChange(
+                    path=blk.path, lineno=blk.lineno, status="unplaced"
+                )
+            )
+            continue
+        edits.setdefault(blk.path, []).append(
+            (blk.lineno, len(blk.lines), new_lines)
+        )
+        changes.append(
+            LatexDocxMergeChange(
+                path=blk.path, lineno=blk.lineno, status="applied"
+            )
+        )
+        typer.echo(f"Applied edit at {loc}")
+    files = {
+        p: Path(p).read_text(encoding="utf-8").split("\n")
+        for p in {ln.path for ln in lines}
+    }
+    for path, updates in edits.items():
+        for lineno, count, new_lines in sorted(updates, reverse=True):
+            files[path][lineno - 1 : lineno - 1 + count] = new_lines
+    added = updated = 0
+    if not no_comments:
+        # Threads keyed by root, anchored through the root's bookmark
+        comments = doc.comments
+        by_id = {c.para_id: c for c in comments}
+        roots = [c for c in comments if not c.parent_id]
+        # Resolve every thread to a block first, then edit each file from
+        # the bottom up so earlier line numbers stay valid
+        placed: list[tuple[calkit.latex.Block, calkit.latex.TexComment]] = []
+        for root in roots:
+            thread = [root] + [
+                c
+                for c in comments
+                if c.parent_id and by_id.get(c.parent_id) is root
+            ]
+            tc = calkit.latex.TexComment(
+                [
+                    calkit.latex.Entry(
+                        c.author, c.text, date=calkit.latex.word_date(c.date)
+                    )
+                    for c in thread
+                ],
+                highlight=root.highlight,
+                highlight_occ=root.highlight_occ,
+                resolved=root.done,
+            )
+            if root.bookmark is None:
+                warn(
+                    f"Comment by {root.author} has no anchor: {root.text[:60]}"
+                )
+                continue
+            parts = root.bookmark.split("_")
+            path, lineno = path_for_hash.get(parts[1], ""), int(parts[2])
+            blk = calkit.latex.find_block(
+                blks, path, lineno, original.paragraphs.get(root.bookmark, "")
+            )
+            if blk is None:
+                warn(
+                    f"Can't place a comment by {root.author}: {root.text[:60]}"
+                )
+                continue
+            placed.append((blk, tc))
+        for blk, tc in sorted(
+            placed, key=lambda x: (x[0].path, x[0].lineno), reverse=True
+        ):
+            content = files[blk.path]
+            at = blk.lineno
+            existing = next(
+                (
+                    e
+                    for e in calkit.latex.parse_comments(content)
+                    if e.author == tc.author and e.text == tc.text
+                ),
+                None,
+            )
+            if existing is not None:
+                if (
+                    existing.messages() == tc.messages()
+                    and existing.resolved == tc.resolved
+                ):
+                    continue
+                # Word knows neither emails nor what the source already
+                # recorded, so carry those over for unchanged messages
+                known = {(e.author, e.text): e for e in existing.entries}
+                for e in tc.entries:
+                    old_e = known.get((e.author, e.text))
+                    if old_e is not None:
+                        e.email = old_e.email
+                        e.date = old_e.date or e.date
+                del content[
+                    existing.lineno - 1 : existing.lineno - 1 + existing.nlines
+                ]
+                if existing.lineno < at:
+                    at -= existing.nlines
+            content[at - 1 : at - 1] = tc.render()
+            added += existing is None
+            updated += existing is not None
+    for path, content in files.items():
+        new = "\n".join(content)
+        if new != Path(path).read_text(encoding="utf-8"):
+            Path(path).write_text(new, encoding="utf-8")
+    rev = None
+    try:
+        rev = calkit.git.get_repo().head.commit.hexsha
+    except Exception:
+        pass
+    seen = {a for p in doc.paragraphs for a in p.authors}
+    seen |= {c.author for c in doc.comments}
+    record = LatexDocxMerge(
+        export_id=original.id,
+        created=datetime.datetime.now(datetime.timezone.utc),
+        docx=Path(docx_path).as_posix(),
+        rev=rev,
+        authors=sorted(seen),
+        last_modified_by=doc.last_modified_by,
+        changes=changes,
+        comments_added=added,
+        comments_updated=updated,
+        files={
+            p: "md5:" + calkit.get_md5(p)
+            for p in sorted(
+                {ln.path for ln in lines} | {Path(docx_path).as_posix()}
+            )
+        },
+    )
+    os.makedirs(calkit.latex.DOCX_MERGES_DIR, exist_ok=True)
+    stamp = record.created.strftime("%Y%m%dT%H%M%S.%fZ")
+    with open(
+        os.path.join(
+            calkit.latex.DOCX_MERGES_DIR, f"{original.id}-{stamp}.json"
+        ),
+        "w",
+        encoding="utf-8",
+        newline="\n",
+    ) as f:
+        f.write(record.model_dump_json(indent=2))
+    counts = {
+        s: sum(1 for c in changes if c.status == s)
+        for s in ("applied", "already-applied", "pending", "unplaced")
+    }
+    typer.echo(
+        f"Applied {counts['applied']} edits ({counts['already-applied']} "
+        f"already there, {counts['pending']} pending, {counts['unplaced']} "
+        f"unplaced); {added} comments added, {updated} updated"
+    )

calkit/cli/new.py

@@ -65,22 +65,37 @@ def _person_from_options(
     return person.model_dump(exclude_none=True)
 
 
-def _split_template_subdir(
-    template: str, git_url: str
-) -> tuple[str, str | None]:
-    """Split a directory out of an 'owner/repo/path' template.
+def _parse_template(
+    template: str, hub_url: str
+) -> tuple[str, str | None, str | None]:
+    """Resolve a template to its name, Git URL, and directory in the repo.
 
-    One repo can hold several self-contained example projects, so a
-    template may name a directory within it. Only the shorthand form is
-    split; a full URL is left alone, since its path is the repo's.
+    'owner/project[/dir]', or that under the hub URL, names a project on
+    the hub, whose Git URL is looked up later, so it comes back as None.
+    An HTTPS or SSH URL works on any host, e.g.,
+    'https://github.com/owner/repo/dir'. One repo can hold several
+    self-contained example projects, so a template may name a directory
+    within it; the first two path segments are the repo. Other URLs,
+    e.g., 'file://', are used as is.
     """
-    if "://" in template or "github.com" in template:
-        return git_url, None
-    parts = template.strip("/").split("/")
-    if len(parts) <= 2:
-        return git_url, None
-    subdir = "/".join(parts[2:])
-    return git_url.rsplit("/" + subdir, 1)[0], subdir
+    import re
+
+    hub_url = hub_url.rstrip("/")
+    for prefix in (hub_url + "/", hub_url.split("://", 1)[-1] + "/"):
+        if template.startswith(prefix):
+            template = template.removeprefix(prefix)
+            break
+    m = re.match(r"^(https?://[^/]+/|[\w.-]+@[\w.-]+:)(.+)$", template)
+    if m is None and "://" in template:
+        return template, template, None
+    base, path = m.groups() if m else (None, template)
+    parts = path.strip("/").split("/")
+    if len(parts) < 2:
+        raise_error(f"Template '{template}' should be 'owner/project'")
+    parts[1] = parts[1].removesuffix(".git")
+    subdir = "/".join(parts[2:]) or None
+    url = base + "/".join(parts[:2]) if base else None
+    return "/".join(parts), url, subdir
 
 
 @new_app.command(name="project", cls=_NewProjectCommand)
@@ -137,10 +152,12 @@ def new_project(
         str | None,
         typer.Option(
             "--template",
+            "--from",
             "-t",
             help=(
-                "Template from which to derive the project, e.g., "
-                "'calkit/example-basic'."
+                "Template from which to derive the project: a hub project "
+                "as 'owner/project' or its hub URL, or a Git URL on any "
+                "host, e.g., 'https://github.com/owner/repo/dir'."
             ),
         ),
     ] = None,
@@ -412,24 +429,22 @@ def new_project(
         return
     # If using a template, clone it first
     if template:
-        # TODO: If the template is not a Git repo URL, make a request to the
-        # the hub to get it?
-        # For now, assume consistency between hub projects and
-        # GitHub repo URLs
-        if "github.com" in template:
-            template_git_url = template
-            template_name = template.split("github.com")[-1][1:].removesuffix(
-                ".git"
-            )
-        else:
-            template_name = template
-            template_git_url = f"https://github.com/{template}"
-        # A template can name a directory inside a repo, e.g.
-        # 'calkit/calkit/examples/markdown', so one repo can hold several
-        # self-contained examples.
-        template_git_url, template_subdir = _split_template_subdir(
-            template, template_git_url
+        template_name, template_git_url, template_subdir = _parse_template(
+            template, calkit.hub.get_hub_url()
         )
+        if template_git_url is None:
+            project = "/".join(template_name.split("/")[:2])
+            typer.echo(f"Fetching Git repo URL for {project} from the hub")
+            try:
+                template_git_url = calkit.hub.get(f"/projects/{project}")[
+                    "git_repo_url"
+                ]
+            except Exception as e:
+                raise_error(
+                    f"Could not fetch project {project} from the hub ({e}); "
+                    "for a repo not on the hub, pass its URL, e.g., "
+                    f"https://github.com/{template_name}"
+                )
         if template_subdir is None:
             # Now clone it
             subprocess.run(["git", "clone", template_git_url, abs_path])