calkit-python 0.49.5__py3-none-any.whl → 0.50.0__py3-none-any.whl

calkit/cli/check.py

@@ -2466,7 +2466,9 @@ def check_storage(
     stale: set[str] | None = set()
     if any(p["stage"] for p in to_record):
         status = calkit.pipeline.get_status(
-            check_environments=False, clean_notebooks=False
+            check_environments=False,
+            clean_notebooks=False,
+            preprocess_stages=False,
         )
         if not status.errors:
             stale = {n.split("@")[0] for n in status.stale_stage_names}

calkit/cli/latex.py

@@ -561,6 +561,17 @@ def _run_latexmk(
             os.remove(fdb_path)
 
 
+@latex_app.command(name="map-components")
+def map_components(
+    tex_file: Annotated[
+        str, typer.Argument(help="The .tex file whose components to map.")
+    ],
+) -> None:
+    """Map in components a LaTeX document includes with `\\ck*` commands."""
+    for problem in calkit.latex.map_ck_components(tex_file):
+        warn(f"{problem}; it will print as ??")
+
+
 @latex_app.command(name="build")
 def build(
     tex_file: Annotated[str, typer.Argument(help="The .tex file to compile.")],
@@ -657,6 +668,10 @@ def build(
     tex_dir = os.path.dirname(tex_file) or "."
     stem = Path(tex_file).stem
     pdf_dir = output_dir if output_dir is not None else tex_dir
+    # Components the document includes with calkit.sty's commands, e.g.,
+    # \ckfigure, have to be beside it before TeX can read them
+    for problem in calkit.latex.map_ck_components(tex_file):
+        warn(f"{problem}; it will print as ??")
     if not os.path.isfile(os.path.join(pdf_dir, stem + ".pdf")):
         fdb_dir = aux_dir if aux_dir is not None else tex_dir
         fdb_fpath = os.path.join(fdb_dir, stem + ".fdb_latexmk")
@@ -766,8 +781,9 @@ _IFDEFINED_MARKUP_RE = re.compile(
 )
 # A one-argument macro's definition, up to its body's opening brace
 _ONE_ARG_MACRO_DEF_RE = re.compile(
-    r"\\(?:(?:re|provide)?newcommand\*?\s*(?:\{\s*\\([A-Za-z@]+)\s*\}"
-    r"|\\([A-Za-z@]+))\s*\[1\]|def\s*\\([A-Za-z@]+)#1)\s*(?=\{)"
+    r"\\(?:(?:(?:re)?newcommand|providecommand)\*?\s*"
+    r"(?:\{\s*\\([A-Za-z@]+)\s*\}|\\([A-Za-z@]+))\s*\[1\]"
+    r"|def\s*\\([A-Za-z@]+)#1)\s*(?=\{)"
 )
 get_diff_path = calkit.latex.get_diff_path
 _default_base_ref = calkit.latex.default_base_ref
@@ -969,6 +985,7 @@ def diff(
     generates, are fetched as they were at that revision, so each side of
     the comparison shows its own.
     """
+    import calkit.resources
 
     def stage_path(stage: dict, path: str) -> str:
         """A stage's path in the project's frame rather than its wdir's."""
@@ -1335,7 +1352,7 @@ def diff(
         which can't say what's inside, so those directories beside the
         document are included whole. Neither finds another stage's output
         DVC doesn't cache, so the pipeline's inputs for the document are
-        added too.
+        added too, as are the components the document includes.
         """
         doc_dir = Path(root, os.path.dirname(tex_file))
         pointed = [
@@ -1344,7 +1361,17 @@ def diff(
             if p.is_file()
         ]
         detected = calkit.latex.detect_inputs(tex_file, wdir=root)
-        return detected + pointed + pipeline_inputs
+        # What the revision's own calkit.sty commands include, which the
+        # stage may not depend on now, at each path TeX would try
+        _, components = calkit.latex.read_ck_usage(
+            os.path.join(root, tex_file)
+        )
+        included = [
+            path
+            for c in components
+            for path in calkit.latex.component_candidates(c)[0]
+        ]
+        return detected + pointed + included + pipeline_inputs
 
     def point_changed_figures_at_base(base_root: str, head_root: str) -> None:
         """Make the older side's changed figures refer to its own copies.
@@ -1531,6 +1558,22 @@ def diff(
         for root in prepared:
             break_verbatim_params(root)
             expand_wrapper_macros(root)
+            # Each side's own values, figures, and tables, read from its
+            # revision, or the working tree for its copy of the sources
+            calkit.latex.expand_ck_components(
+                root, "." if root == working_copy else root, tex_file
+            )
+        # \cknum in the marked-up document needs calkit.sty beside it
+        build_style = os.path.join(
+            head_root, os.path.dirname(tex_file), calkit.latex.STYLE_FNAME
+        )
+        if not os.path.isfile(build_style):
+            with open(build_style, "w", encoding="utf-8") as f:
+                f.write(
+                    calkit.resources.read_text(
+                        "latex", calkit.latex.STYLE_FNAME
+                    )
+                )
         point_changed_figures_at_base(checkouts["base"], head_root)
         _build_diff(
             base_tex_fpath=sides["base"],
@@ -1711,6 +1754,9 @@ def _build_diff(
         # chose otherwise
         if not any(a.startswith("--graphics-markup") for a in latexdiff_args):
             latexdiff_cmd.append("--graphics-markup=both")
+        # A value from calkit.sty is marked like text, so a changed one shows
+        # struck out beside its new value, rather than only as it is now
+        latexdiff_cmd.append("--append-safecmd=cknum")
         # User pass-through args come last so they can override Calkit's
         # defaults
         latexdiff_cmd += latexdiff_args
@@ -2013,7 +2059,10 @@ def to_docx(
         try:
             calkit.docx.pdf_to_docx(pdf_path, output)
         except RuntimeError as e:
-            raise_error(str(e))
+            raise_error(
+                f"{e}\nTo convert from the source instead, use "
+                "--engine libreoffice"
+            )
     elif engine == "libreoffice":
         import shlex
 
@@ -2032,6 +2081,8 @@ def to_docx(
         job = Path(source).stem + "-calkit-docx"
         build = os.path.join(calkit.latex.DOCX_BUILD_DIR, Path(source).stem)
         shutil.rmtree(build, ignore_errors=True)
+        # Under .calkit/local, which keeps itself out of Git
+        calkit.ensure_local_dir()
         os.makedirs(build)
         # The PDF build's bibliography, since TeX4ht doesn't run BibTeX
         bbl = Path(source).with_suffix(".bbl")
@@ -2169,6 +2220,32 @@ def to_docx(
     blks = calkit.latex.blocks(lines)
     paras = doc.paragraphs
     matched = calkit.latex.align([p.text for p in paras], blks)
+    # Word's PDF import drops a hyphen ending a line even when the word has
+    # it, e.g., "two-dimensional" as "twodimensional", so it goes back where
+    # the paragraph's source spells the word with it and never without
+    if engine == "word":
+        for para, blk in zip(paras, matched):
+            if blk is None or para.element is None:
+                continue
+            words = re.findall(
+                r"(?<![\w-])[^\W\d_]\w*(?:-\w+)+(?![\w-])", blk.text
+            )
+            for word in dict.fromkeys(words):
+                pattern = re.compile(
+                    r"(?<![\w-])"
+                    + "(-?)".join(re.escape(w) for w in word.split("-"))
+                    + r"(?![\w-])",
+                    re.IGNORECASE,
+                )
+                if any("" in m.groups() for m in pattern.finditer(blk.text)):
+                    continue
+                text = doc.linear_text(para.element)
+                # From the end, so earlier offsets still hold
+                for m in reversed(list(pattern.finditer(text))):
+                    for k in reversed(range(1, len(m.groups()) + 1)):
+                        if not m.group(k):
+                            doc.insert_text(para.element, m.start(k), "-")
+        paras = doc.paragraphs
     # Bookmark each anchored paragraph; a block rendering as several
     # paragraphs gets numbered suffixes
     original: dict[str, str] = {}
@@ -2251,6 +2328,26 @@ def to_docx(
         # Word lays some displays out in tables of its own, which go too
         # if all else they hold is the prose on either side, moved out.
         emu_limit = int(1.5 * 914400)
+        # A figure without a caption, in a float or not, could be any tall
+        # picture, so then none go
+        captionless, depth, captioned = False, 0, False
+        for ln in lines:
+            code = calkit.latex._strip_comments(ln.text)
+            for m in re.finditer(
+                r"\\(begin|end)\{(figure|wrapfigure|sidewaysfigure)\*?\}"
+                r"|\\(caption|includegraphics)\b",
+                code,
+            ):
+                if m.group(1) == "begin":
+                    depth, captioned = depth + 1, captioned and depth > 0
+                elif m.group(1) == "end":
+                    depth = max(0, depth - 1)
+                    if depth == 0 and not captioned:
+                        captionless = True
+                elif m.group(3) == "caption":
+                    captioned = True
+                elif depth == 0:
+                    captionless = True
         w_body = doc.doc.find(calkit.docx._tag(calkit.docx.W, "body"))
         assert w_body is not None
         w_p = calkit.docx._tag(calkit.docx.W, "p")
@@ -2263,6 +2360,11 @@ def to_docx(
             )
 
         anchored = {p.element for p, b in zip(paras, matched) if b is not None}
+        captions = {
+            p.element
+            for p, b in zip(paras, matched)
+            if b is not None and b.floating
+        }
 
         def fragment(el: Any, context: str) -> bool:
             # Never prose: anything anchored to the source, e.g., a caption
@@ -2276,7 +2378,7 @@ def to_docx(
             prose_like = (
                 len(words) >= 2
                 and calkit.latex.similarity(text, context) >= 0.5
-                and not re.search(eq_num + r"\s*$", text)
+                and not calkit.latex.equation_fragment(text)
             )
             if el in anchored or len(words) > 3 or prose_like:
                 return False
@@ -2287,7 +2389,29 @@ def to_docx(
                     "wordprocessingDrawing}extent"
                 )
             )
-            return not tall
+            if not tall:
+                return True
+            if captionless:
+                return False
+            # A tall picture is a figure where there's a caption beside it,
+            # past any other pictures, else a display drawn as one
+            children = list(w_body)
+            i = children.index(unit_of(el))
+            for step in (-1, 1):
+                k = i + step
+                while (
+                    0 <= k < len(children) and not text_of(children[k]).strip()
+                ):
+                    k += step
+                if not 0 <= k < len(children):
+                    continue
+                near = children[k]
+                if any(q in captions for q in near.iter(w_p)) or re.match(
+                    r"\s*(Figure|Fig\.|Table|Scheme|Chart)\s+[A-Z]?\.?\d",
+                    text_of(near),
+                ):
+                    return False
+            return True
 
         def unit_of(el: Any) -> Any:
             while doc._parents.get(el) is not w_body:
@@ -2690,6 +2814,7 @@ def to_docx(
         .isoformat(timespec="seconds")
         .replace(":", "-")
     )
+    calkit.ensure_local_dir()
     dirs = [calkit.latex.LOCAL_DOCX_EXPORTS_DIR]
     if log:
         dirs.append(calkit.latex.DOCX_EXPORTS_DIR)
@@ -3084,6 +3209,7 @@ def merge_docx(
         .isoformat(timespec="seconds")
         .replace(":", "-")
     )
+    calkit.ensure_local_dir()
     dirs = [calkit.latex.LOCAL_DOCX_MERGES_DIR]
     if log:
         dirs.append(calkit.latex.DOCX_MERGES_DIR)

calkit/cli/main/core.py

@@ -3343,7 +3343,10 @@ def run(
             step("Compiling DVC pipeline", "🔀")
             try:
                 dvc_stages = calkit.pipeline.to_dvc(
-                    ck_info=ck_info, write=True, for_run=True
+                    ck_info=ck_info,
+                    write=True,
+                    for_run=True,
+                    preprocess_stages=True,
                 )
             except Exception as e:
                 os.environ.pop("CALKIT_PIPELINE_RUNNING", None)

calkit/cli/overleaf.py

@@ -365,6 +365,8 @@ def import_publication(
     os.makedirs(dest_dir, exist_ok=True)
     ck_info = calkit.load_calkit_info()
     pubs = ck_info.get("publications", [])
+    if dest_dir in (ck_info.get("overleaf_sync") or {}):
+        raise_error(f"'{dest_dir}' is already synced with Overleaf")
     # TODO: Don't allow the same Overleaf project ID in multiple publications
     repo = calkit.git.get_repo()
     # Clone the Overleaf project into .calkit/overleaf if it doesn't exist
@@ -400,34 +402,36 @@ def import_publication(
             "please specify with --target"
         )
         return
+    # Determine the PDF output path
+    pdf_path = target_path.removesuffix(".tex") + ".pdf"
+    typer.echo(f"Using PDF path: {pdf_path}")
+    tex_path = pdf_path.removesuffix(".pdf") + ".tex"
+    target_tex_path = Path(dest_dir, tex_path).as_posix()
+    pub_path = Path(dest_dir, pdf_path).as_posix()
+    # A publication already in the project is linked as it is, unless asked
+    # to replace it
+    existing_pub = next((p for p in pubs if p.get("path") == pub_path), None)
+    if existing_pub is not None and overwrite:
+        # Note: This publication will go to the end of the list
+        pubs = [p for p in pubs if p.get("path") != pub_path]
+        existing_pub = None
+    if existing_pub is not None:
+        typer.echo(f"Using the publication already at {pub_path}")
     # Try to extract title from the target LaTeX file if not provided
-    if not title:
-        target_tex_path = os.path.join(
-            dest_dir if push_only else overleaf_project_dir, target_path
+    if existing_pub is None and not title:
+        extracted_title = _extract_title_from_tex(
+            os.path.join(
+                dest_dir if push_only else overleaf_project_dir, target_path
+            )
         )
-        extracted_title = _extract_title_from_tex(target_tex_path)
         if extracted_title:
             typer.echo(f"Detected title: {extracted_title}")
             title = extracted_title
-    if not title:
+    if existing_pub is None and not title:
         raise_error(
             "Title could not be detected from the LaTeX file; "
             "please specify with --title"
         )
-    # Determine the PDF output path
-    pdf_path = target_path.removesuffix(".tex") + ".pdf"  # type: ignore
-    typer.echo(f"Using PDF path: {pdf_path}")
-    tex_path = pdf_path.removesuffix(".pdf") + ".tex"
-    target_tex_path = Path(dest_dir, tex_path).as_posix()
-    pub_path = Path(dest_dir, pdf_path).as_posix()
-    pub_paths = [pub.get("path") for pub in pubs]
-    if not overwrite and pub_path in pub_paths:
-        raise_error(
-            f"A publication already exists in this project at {pub_path}"
-        )
-    elif overwrite and pub_path in pub_paths:
-        # Note: This publication will go to the end of the list
-        pubs = [p for p in pubs if p.get("path") != pub_path]
     # Check that we have a LaTeX environment
     typer.echo("Checking that this project has a LaTeX environment")
     envs = ck_info.get("environments", {})
@@ -457,14 +461,18 @@ def import_publication(
     pipeline = ck_info.get("pipeline", {})
     stages = pipeline.get("stages", {})
     stage_name = None
-    for stage_name_i, stage in stages.items():
-        if (
-            stage.get("kind") == "latex"
-            and stage.get("target_path") == target_tex_path
-        ):
-            stage_name = stage_name_i
-            typer.echo(f"Found build stage '{stage_name}' in pipeline")
-            break
+    if existing_pub is not None and existing_pub.get("stage") in stages:
+        stage_name = existing_pub["stage"]
+        typer.echo(f"Using the publication's build stage '{stage_name}'")
+    else:
+        for stage_name_i, stage in stages.items():
+            if (
+                stage.get("kind") == "latex"
+                and stage.get("target_path") == target_tex_path
+            ):
+                stage_name = stage_name_i
+                typer.echo(f"Found build stage '{stage_name}' in pipeline")
+                break
     if stage_name is None:
         # Create a new stage
         stage_name = calkit.to_kebab_case("build-" + dest_dir)
@@ -482,21 +490,24 @@ def import_publication(
             no_commit=True,
         )
         repo.git.add("calkit.yaml")
-    # Add to publications in calkit.yaml
-    typer.echo("Adding publication to calkit.yaml")
     ck_info = calkit.load_calkit_info()
-    new_pub = dict(
-        path=pub_path,
-        title=title,
-        description=description,
-        kind=kind,
-        stage=stage_name,
-    )
-    pubs.append(new_pub)
-    ck_info["publications"] = pubs
+    if existing_pub is None:
+        typer.echo("Adding publication to calkit.yaml")
+        pubs.append(
+            dict(
+                path=pub_path,
+                title=title,
+                description=description,
+                kind=kind,
+                stage=stage_name,
+            )
+        )
+        ck_info["publications"] = pubs
+    else:
+        for pub in ck_info.get("publications") or []:
+            if pub.get("path") == pub_path and not pub.get("stage"):
+                pub["stage"] = stage_name
     ol_sync = ck_info.get("overleaf_sync", {})
-    if dest_dir in ol_sync:
-        raise_error(f"'{dest_dir}' is already synced with Overleaf")
     ol_sync[dest_dir] = dict(url=src_url)
     if push_paths:
         ol_sync[dest_dir]["push_paths"] = push_paths
@@ -526,6 +537,65 @@ def import_publication(
     )
 
 
+@overleaf_app.command(name="link")
+def link_publication(
+    folder: Annotated[
+        str,
+        typer.Argument(help="Folder in the project, e.g., 'paper'."),
+    ],
+    project_url: Annotated[
+        str,
+        typer.Argument(
+            help=(
+                "Overleaf project URL, e.g., "
+                "https://www.overleaf.com/project/6800005973cb2e35."
+            )
+        ),
+    ],
+    title: Annotated[
+        str | None,
+        typer.Option(
+            "--title",
+            "-t",
+            help="Title, if the publication isn't in calkit.yaml yet.",
+        ),
+    ] = None,
+    target_path: Annotated[
+        str | None,
+        typer.Option("--target", "-T", help="Target TeX file in the folder."),
+    ] = None,
+    push_paths: Annotated[
+        list[str],
+        typer.Option(
+            "--push-path",
+            "-p",
+            help="Paths in the folder to only push to Overleaf, e.g., 'figures'.",
+        ),
+    ] = [],
+    no_commit: Annotated[
+        bool,
+        typer.Option("--no-commit", help="Do not commit changes to repo."),
+    ] = False,
+) -> None:
+    """Link a folder in the project to an Overleaf project, pushing its files
+    there.
+
+    Like 'import --push-only', for a document that starts in the project.
+    """
+    import_publication(
+        src_url=project_url,
+        dest_dir=folder,
+        title=title,
+        target_path=target_path,
+        description=None,
+        kind=None,
+        push_paths=push_paths,
+        no_commit=no_commit,
+        overwrite=False,
+        push_only=True,
+    )
+
+
 def _get_document_stages(folders: list[str]) -> dict[str, str]:
     """LaTeX stages that build a document in these folders, keyed to the
     folder.
@@ -935,6 +1005,9 @@ def sync(
             continue
         if paths is not None and synced_folder not in paths:
             continue
+        # Resolving a conflict only finishes the sync that hit it
+        if resolve and synced_folder != resolving_info["wdir"]:
+            continue
         overleaf_project_id = sync_data.get("project_id")
         if not overleaf_project_id:
             raise_error(
@@ -950,7 +1023,7 @@ def sync(
             f"Syncing {folder} with Overleaf project ID {overleaf_project_id}"
         )
         wdir = synced_folder
-        if resolve and wdir == resolving_info["wdir"]:
+        if resolve:
             repo.git.add(wdir)
             if repo.git.diff(["--staged", wdir]):
                 repo.git.commit(
@@ -959,8 +1032,6 @@ def sync(
             if in_am_session:
                 with calkit.overleaf.staged_changes_set_aside(repo):
                     repo.git.am("--skip")
-        elif resolve:
-            continue
         # Local edits are saved first so they're what's synced and rebuilt
         # from. Untracked files aren't sent to Overleaf, so they're left out.
         if repo.git.diff(wdir) or repo.index.diff("HEAD", wdir):

calkit/cli/update.py

@@ -1679,7 +1679,10 @@ def update_path_storage(
     # calkit.yaml has it now must be known to be up to date
     if outputs and to == "dvc":
         status = calkit.pipeline.get_status(
-            ck_info=ck_info, check_environments=False, clean_notebooks=False
+            ck_info=ck_info,
+            check_environments=False,
+            clean_notebooks=False,
+            preprocess_stages=False,
         )
         if status.errors:
             raise_error(

calkit/config.py

@@ -306,11 +306,67 @@ def set_secret(key: str, value: str) -> None:
     """Sets a secret using keyring, handling byte conversion for Linux."""
     service_name = get_app_name()
     username = _keyring_username(key)
+
+    def update_macos_item() -> bool:
+        # keyring replaces the item, which forgets every app the user chose
+        # to always allow, so they'd be asked again after each token refresh
+        import ctypes
+        import ctypes.util
+
+        import keyring.backends.macOS
+
+        if not isinstance(
+            keyring.get_keyring(), keyring.backends.macOS.Keyring
+        ):
+            return False
+        from keyring.backends.macOS import api
+
+        # Our own handles, so keyring's private ones are left alone
+        found = ctypes.CDLL(ctypes.util.find_library("Foundation"))
+        sec = ctypes.CDLL(ctypes.util.find_library("Security"))
+        found.CFDataCreate.restype = ctypes.c_void_p
+        found.CFDataCreate.argtypes = (
+            ctypes.c_void_p,
+            ctypes.c_char_p,
+            ctypes.c_long,
+        )
+        found.CFRelease.restype = None
+        found.CFRelease.argtypes = (ctypes.c_void_p,)
+        sec.SecItemUpdate.restype = ctypes.c_int32
+        sec.SecItemUpdate.argtypes = (ctypes.c_void_p, ctypes.c_void_p)
+        data = value.encode("utf-8")
+        cf_data = found.CFDataCreate(None, data, len(data))
+        query = api.create_query(
+            kSecClass=api.k_("kSecClassGenericPassword"),
+            kSecAttrService=service_name,
+            kSecAttrAccount=username,
+        )
+        attrs = api.create_query(kSecValueData=ctypes.c_void_p(cf_data))
+        try:
+            status = sec.SecItemUpdate(query, attrs)
+        finally:
+            for ref in (cf_data, query, attrs):
+                if ref:
+                    found.CFRelease(ref)
+        # Only a missing item may be added through keyring, since that
+        # replaces any item there, e.g., after the user denied the update
+        if status == -25300:  # errSecItemNotFound
+            return False
+        if status == -128:  # errSecUserCanceled
+            raise keyring.errors.KeyringLocked(
+                f"Can't store password on keychain: OSStatus {status}"
+            )
+        if status != 0:
+            raise keyring.errors.PasswordSetError(
+                f"Can't store password on keychain: OSStatus {status}"
+            )
+        return True
+
     with _keyring_home():
         if platform.system() == "Linux":
             value_bytes = value.encode("utf-8")
             keyring.set_password(service_name, username, value_bytes)  # type: ignore
-        else:
+        elif platform.system() != "Darwin" or not update_macos_item():
             keyring.set_password(service_name, username, value)
     _secret_cache[(service_name, username)] = value
 
@@ -526,6 +582,11 @@ class Settings(BaseSettings):
                     KeyringOptionalSecret in get_type_args(value.annotation)
                 ) and key in cfg:
                     secret_val = cfg.pop(key)
+                    # Only what changed, so a token refresh leaves the
+                    # others, and who may read them, alone
+                    stored = (get_app_name(), _keyring_username(key))
+                    if _secret_cache.get(stored, ...) == secret_val:
+                        continue
                     if secret_val is not None:
                         set_secret(key, secret_val)
                     else: