calkit-python 0.50.3__py3-none-any.whl → 0.50.4__py3-none-any.whl

calkit/cli/check.py

@@ -16,7 +16,7 @@ import dotenv
 import typer
 
 import calkit
-from calkit.cli import AliasGroup, raise_error, warn
+from calkit.cli import AliasGroup, raise_error, warn, warn_app_control
 from calkit.core import get_md5
 
 
@@ -605,6 +605,7 @@ def check_environment(
         try:
             subprocess.check_call(cmd)
         except subprocess.CalledProcessError:
+            warn_app_control(maybe=True)
             raise_error("Failed to check pixi environment")
     elif env["kind"] == "uv":
         cmd = ["uv", "sync"]
@@ -616,6 +617,7 @@ def check_environment(
         try:
             subprocess.check_call(cmd)
         except subprocess.CalledProcessError:
+            warn_app_control(maybe=True)
             raise_error("Failed to check uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
         if "path" not in env:
@@ -1835,6 +1837,7 @@ def check_conda_env(
             verbose=not quiet,
         )
     except Exception as e:
+        warn_app_control(output=str(e), maybe=True)
         raise_error(f"Failed to check conda environment: {e}")
 
 
@@ -1917,6 +1920,7 @@ def check_venv(
         try:
             subprocess.check_call(create_cmd + [prefix], cwd=wdir)
         except subprocess.CalledProcessError:
+            warn_app_control(maybe=True)
             raise_error(f"Failed to create {kind} at {prefix}")
         # Put a gitignore file in the env dir if one doesn't exist
         gitignore_fpath = os.path.join(prefix_full_path, ".gitignore")
@@ -2117,6 +2121,7 @@ def check_venv(
             pip_install_and_freeze(dep_file_txt)
         except (subprocess.CalledProcessError, OSError):
             if lock_is_current:
+                warn_app_control(maybe=True)
                 raise_error(
                     f"Failed to create {kind} at {prefix} from its lock file "
                     f"({lock_fpath}), which matches the spec ({path}). "
@@ -2132,6 +2137,7 @@ def check_venv(
             try:
                 pip_install_and_freeze(f"-r {path}")
             except subprocess.CalledProcessError:
+                warn_app_control(maybe=True)
                 raise_error(f"Failed to check {kind} from input file {path}")
 
 

calkit/cli/core.py

@@ -184,3 +184,35 @@ def warn(txt: str, prefix: str = "Warning: ", err: bool = False):
     text = typer.style(prefix + encode_safe(str(txt), err=err), fg="yellow")
     if not _echo_above_spinner(text, err=err):
         typer.echo(text, err=err)
+
+
+def warn_app_control(
+    output: str = "", returncode: int | None = None, maybe: bool = False
+) -> None:
+    """Explain a failure Windows App Control may have caused.
+
+    Smart App Control blocks unsigned executables and DLLs it doesn't trust,
+    which environments are full of, and the failure reads as unrelated.
+    Pass ``maybe`` for a failure whose output went straight to the terminal,
+    to warn whenever Smart App Control is on.
+    """
+    from calkit.windows import (
+        SMART_APP_CONTROL_DOCS_URL,
+        app_control_blocked,
+        smart_app_control_on,
+    )
+
+    if app_control_blocked(output=output, returncode=returncode):
+        warn(
+            "Windows blocked a file from running, which Smart App Control "
+            "does to unsigned files it doesn't trust; see "
+            f"{SMART_APP_CONTROL_DOCS_URL}",
+            err=True,
+        )
+    elif maybe and smart_app_control_on():
+        warn(
+            "Smart App Control is on, which can block files in environments "
+            "from running; if the output above says a file was blocked, see "
+            f"{SMART_APP_CONTROL_DOCS_URL}",
+            err=True,
+        )

calkit/cli/export.py

@@ -0,0 +1,139 @@
+"""CLI for exporting data and artifacts from various apps, e.g., a PNG
+from an Inkscape SVG.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING
+
+import typer
+from typing_extensions import Annotated
+
+from calkit.cli import raise_error
+
+if TYPE_CHECKING:
+    from calkit.export import Exporter
+
+export_app = typer.Typer(no_args_is_help=True)
+
+
+def _prepare(exporter: Exporter, source: str, output: str) -> str:
+    """Get ready to export, returning the executable to run."""
+    exporter.check(source, output)
+    if not os.path.isfile(source):
+        raise ValueError(f"'{source}' doesn't exist")
+    os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
+    # What was there is from before, so it can't pass for a new export
+    if os.path.exists(output):
+        os.remove(output)
+    exe = exporter.find_executable()
+    if exe is None:
+        raise ValueError(
+            f"Exporting '{source}' needs {exporter.app} installed"
+        )
+    return exe
+
+
+def inkscape(source: str, output: str, dpi: int | None = None) -> None:
+    """Export a drawing to the format ``output``'s extension names,
+    raising ``ValueError`` if it can't be.
+    """
+    import subprocess
+
+    from calkit.export import INKSCAPE
+
+    exe = _prepare(INKSCAPE, source, output)
+    cmd = [exe, source, f"--export-filename={output}"]
+    if dpi is not None:
+        cmd.append(f"--export-dpi={dpi}")
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if not os.path.isfile(output):
+        detail = (result.stderr or result.stdout or "").strip()
+        raise ValueError(
+            f"Inkscape didn't export '{output}' from '{source}'"
+            + (f": {detail.splitlines()[-1]}" if detail else "")
+        )
+
+
+def gimp(source: str, output: str) -> None:
+    """Flatten a GIMP document into the format ``output``'s extension
+    names, raising ``ValueError`` if it can't be.
+
+    Runs GIMP 3's batch dialect, then GIMP 2's, which differ in their file
+    procedures. Neither reports a failed script in its exit code, so the
+    written file is what says it worked.
+    """
+    import subprocess
+
+    from calkit.export import GIMP
+
+    exe = _prepare(GIMP, source, output)
+    src = os.path.abspath(source).replace("\\", "\\\\").replace('"', '\\"')
+    dst = os.path.abspath(output).replace("\\", "\\\\").replace('"', '\\"')
+    v3 = (
+        f'(let* ((image (car (gimp-file-load RUN-NONINTERACTIVE "{src}"))))'
+        "  (gimp-image-flatten image)"
+        f'  (gimp-file-save RUN-NONINTERACTIVE image "{dst}" "{dst}")'
+        "  (gimp-image-delete image)"
+        "  (gimp-quit 0))"
+    )
+    v2 = (
+        "(let* ((image (car (gimp-file-load RUN-NONINTERACTIVE "
+        f'"{src}" "{src}")))'
+        "       (drawable (car (gimp-image-flatten image))))"
+        f'  (gimp-file-save RUN-NONINTERACTIVE image drawable "{dst}" '
+        f'"{dst}")'
+        "  (gimp-image-delete image)"
+        "  (gimp-quit 0))"
+    )
+    detail = ""
+    for cmd in [
+        [exe, "-i", "--batch-interpreter=plug-in-script-fu-eval", "-b", v3],
+        [exe, "-i", "-b", v2],
+    ]:
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        if os.path.isfile(output):
+            return
+        detail = (result.stderr or result.stdout or "").strip()
+    raise ValueError(
+        f"GIMP didn't export '{output}' from '{source}'"
+        + (f": {detail.splitlines()[-1]}" if detail else "")
+    )
+
+
+@export_app.command(name="inkscape", help="Export a figure from an SVG.")
+def export_inkscape(
+    source: Annotated[str, typer.Argument(help="SVG to export from.")],
+    output: Annotated[
+        str,
+        typer.Option(
+            "--output", "-o", help="File to export to, e.g., a PNG or PDF."
+        ),
+    ],
+    dpi: Annotated[
+        int | None, typer.Option("--dpi", help="Resolution of a PNG.")
+    ] = None,
+) -> None:
+    try:
+        inkscape(source, output, dpi=dpi)
+    except ValueError as e:
+        raise_error(str(e))
+
+
+@export_app.command(
+    name="gimp", help="Export a flattened figure from an XCF file."
+)
+def export_gimp(
+    source: Annotated[str, typer.Argument(help="XCF file to export from.")],
+    output: Annotated[
+        str,
+        typer.Option(
+            "--output", "-o", help="File to export to, e.g., a PNG or TIFF."
+        ),
+    ],
+) -> None:
+    try:
+        gimp(source, output)
+    except ValueError as e:
+        raise_error(str(e))

calkit/cli/latex.py

@@ -660,6 +660,26 @@ def build(
     """
     import time
 
+    def rel_to_tex_dir(path: str) -> str:
+        """Express a current-directory-relative path from the .tex file's
+        directory.
+
+        Pure path arithmetic, since ``os.path.relpath`` resolves names like
+        ``aux`` to devices on Windows.
+        """
+        cwd = os.getcwd()
+        target = Path(os.path.normpath(os.path.join(cwd, path)))
+        start = Path(os.path.normpath(os.path.join(cwd, tex_dir)))
+        # On another drive, there's no relative path
+        if os.path.normcase(target.anchor) != os.path.normcase(start.anchor):
+            return target.as_posix()
+        t = [os.path.normcase(p) for p in target.parts]
+        s = [os.path.normcase(p) for p in start.parts]
+        n = 0
+        while n < min(len(t), len(s)) and t[n] == s[n]:
+            n += 1
+        return "/".join([".."] * (len(s) - n) + list(target.parts[n:])) or "."
+
     # latexmk records a failed run in its file database and then refuses
     # to try again, reporting "Nothing to do" and exiting non-zero with no
     # PDF. Running it again is the first thing anyone does after a
@@ -672,6 +692,33 @@ def build(
     # \ckfigure, have to be beside it before TeX can read them
     for problem in calkit.latex.map_ck_components(tex_file):
         warn(f"{problem}; it will print as ??")
+    # Plain \includegraphics counts as much as \ckfigure, and is fine
+    # wherever the pipeline made the file. Paths are the enclosing
+    # project's, wherever the build runs from, e.g., a stage's wdir
+    root = os.path.abspath(tex_dir)
+    while not os.path.isfile(os.path.join(root, "calkit.yaml")):
+        parent = os.path.dirname(root)
+        if parent == root:
+            root = os.path.abspath(".")
+            break
+        root = parent
+    shown = calkit.latex.find_shown(
+        os.path.relpath(os.path.abspath(tex_file), root), wdir=root
+    )
+    if shown:
+        from calkit.provenance import (
+            find_project_provenance_gaps,
+            how_to_define_origin,
+        )
+
+        for gap in find_project_provenance_gaps(shown, wdir=root):
+            msg = str(gap)
+            how = (
+                "give its URL, DOI, or Git repo in imported_from"
+                if gap.unchecked
+                else how_to_define_origin(gap.path)
+            )
+            warn(f"{msg[0].upper()}{msg[1:]}; {how}")
     if not os.path.isfile(os.path.join(pdf_dir, stem + ".pdf")):
         fdb_dir = aux_dir if aux_dir is not None else tex_dir
         fdb_fpath = os.path.join(fdb_dir, stem + ".fdb_latexmk")
@@ -693,11 +740,9 @@ def build(
     # file's directory; convert the (current-directory-relative) Calkit paths
     # into that frame.
     if output_dir is not None:
-        rel = Path(os.path.relpath(output_dir, tex_dir)).as_posix()
-        latexmk_cmd.append(f"-outdir={rel}")
+        latexmk_cmd.append(f"-outdir={rel_to_tex_dir(output_dir)}")
     if aux_dir is not None:
-        rel = Path(os.path.relpath(aux_dir, tex_dir)).as_posix()
-        latexmk_cmd.append(f"-auxdir={rel}")
+        latexmk_cmd.append(f"-auxdir={rel_to_tex_dir(aux_dir)}")
     latexmk_cmd.append("-interaction=nonstopmode")
     latexmk_cmd.append(f"-usepretex={calkit.latex.PDF_PRETEX}")
     # User pass-through args come last so they can override Calkit's defaults.

calkit/cli/main/core.py

@@ -42,6 +42,7 @@ from calkit.cli import (
     raise_error,
     run_cmd,
     warn,
+    warn_app_control,
 )
 from calkit.cli.check import (
     check_app,
@@ -55,6 +56,7 @@ from calkit.cli.config import config_app
 from calkit.cli.delete import delete_app
 from calkit.cli.describe import describe_app
 from calkit.cli.dev import dev_app
+from calkit.cli.export import export_app
 from calkit.cli.fix import (
     OursOpt,
     PreferOpt,
@@ -95,6 +97,11 @@ app.add_typer(show_app, name="show", help="Show Calkit objects.")
 app.add_typer(describe_app, name="describe|desc", help="Describe things.")
 app.add_typer(import_app, name="import", help="Import objects.")
 app.add_typer(office_app, name="office", help="Work with Microsoft Office.")
+app.add_typer(
+    export_app,
+    name="export",
+    help="Export data and artifacts from various applications.",
+)
 app.add_typer(update_app, name="update", help="Update objects.")
 app.add_typer(check_app, name="check", help="Check things.")
 app.add_typer(fix_app, name="fix", help="Fix things.")
@@ -942,11 +949,22 @@ def get_status(
             return
         elif pipeline_status and pipeline_status.is_stale:
             typer.echo("Stale stages:")
+            ck_stages = (ck_info.get("pipeline") or {}).get("stages") or {}
             for stage_name in pipeline_status.stale_stage_names:
                 stale_stage = pipeline_status.stale_stages.get(stage_name)
                 if stale_stage is None:
                     continue
                 typer.echo(f"        {typer.style(stage_name, fg='yellow')}:")
+                ck_stage = ck_stages.get(stage_name)
+                if (
+                    isinstance(ck_stage, dict)
+                    and ck_stage.get("kind") == "manual-step"
+                ):
+                    warn(
+                        "          is a manual step: "
+                        f"{ck_stage.get('message')}",
+                        prefix="",
+                    )
                 if stale_stage.cannot_run_here:
                     warn(
                         "          can't run on this machine: "
@@ -3052,6 +3070,7 @@ def run(
                 stage_names=stage_names,
                 system_info=system_info,
                 interactive=False if spinner else None,
+                manual=can_ask,
             )
             # A machine that couldn't be reached may only need asking about,
             # e.g., for its host
@@ -3918,6 +3937,27 @@ def run(
                 }
                 if isinstance(props.get("memory_gb"), (int, float)):
                     entry["system"]["memory_gb"] = round(props["memory_gb"], 1)
+            # What the stage itself says to record, e.g., the version of an
+            # app it runs, read here, so only for a run on this machine
+            base_name = lock_name.split("@")[0]
+            ck_stage = (
+                run_stages.get(base_name)
+                if lock_dir is None and machine_info[outer] is None
+                else None
+            )
+            if isinstance(ck_stage, dict):
+                from calkit.models.pipeline import Pipeline
+
+                try:
+                    model = Pipeline.model_validate(
+                        {"stages": {base_name: ck_stage}}
+                    ).stages[base_name]
+                    extra = model.run_properties()
+                except Exception as e:
+                    warn(f"Couldn't read what {stage_name} records: {e}")
+                    extra = {}
+                if extra:
+                    entry.setdefault("system", {}).update(extra)
             stage_runs[lock_dir][lock_name] = entry
             # Replacing a record made from here, from before a subproject's
             # were kept with it
@@ -3951,6 +3991,7 @@ def run(
             calkit.dvc.restore_output_ignores()
         except Exception as e:
             warn(f"Failed to re-ignore pipeline outputs: {e}")
+        warn_app_control(output=log_content)
         raise_error("Pipeline failed")
     elif skipped_stale:
         for addressing, reason in skipped_stale.items():
@@ -3994,6 +4035,25 @@ def manual_step(
     cmd: Annotated[
         str | None, typer.Option("--cmd", help="Command to run.")
     ] = None,
+    app: Annotated[
+        str | None,
+        typer.Option("--app", help="App to open, e.g., 'Inkscape'."),
+    ] = None,
+    open_path: Annotated[
+        str | None,
+        typer.Option("--open", help="File to open with the app."),
+    ] = None,
+    outputs: Annotated[
+        list[str],
+        typer.Option(
+            "--output",
+            "-o",
+            help=(
+                "File the step writes, which must change for it to be "
+                "considered done."
+            ),
+        ),
+    ] = [],
     show_stdout: Annotated[
         bool, typer.Option("--show-stdout", help="Show stdout.")
     ] = False,
@@ -4001,6 +4061,53 @@ def manual_step(
         bool, typer.Option("--show-stderr", help="Show stderr.")
     ] = False,
 ) -> None:
+    import textwrap
+
+    from calkit.provenance import hash_path
+
+    def ask() -> str:
+        # Windows calls an empty stdin a terminal, so its end says so too
+        try:
+            return input()
+        except EOFError:
+            raise_error(
+                f"Needs a manual step, with nobody here to do it: {message}"
+            )
+
+    # Waiting for someone who isn't there would hang, e.g., in CI
+    if not sys.stdin.isatty():
+        raise_error(
+            f"Needs a manual step, with nobody here to do it: {message}"
+        )
+    # The app is named rather than left to the default for the file, which
+    # differs between machines, e.g., Illustrator rather than Inkscape
+    if open_path is not None and app is None:
+        raise_error("Name the app to open it with, with --app")
+    if open_path is not None and not os.path.exists(open_path):
+        raise_error(f"{open_path} doesn't exist")
+    before = {path: hash_path(path) for path in outputs}
+    if app is not None:
+        args = [] if open_path is None else [open_path]
+        typer.echo(f"Opening {open_path or app} with {app}")
+        # macOS finds apps by name wherever they're installed
+        exe = shutil.which(app) or shutil.which(app.lower())
+        opened = False
+        if sys.platform == "darwin":
+            opened = (
+                subprocess.run(
+                    ["open", "-a", app, *args], capture_output=True
+                ).returncode
+                == 0
+            )
+        elif exe is not None:
+            subprocess.Popen(
+                [exe, *args],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            opened = True
+        if not opened:
+            warn(f"Can't find {app} here; open it yourself")
     if cmd is not None:
         typer.echo(f"Running command: {cmd}")
         subprocess.Popen(
@@ -4009,7 +4116,38 @@ def manual_step(
             stdout=subprocess.PIPE if not show_stdout else None,
             shell=True,
         )
-    input(message + " (press enter to confirm): ")
+    # Each prompt ends its line, since 'calkit run' passes stage output on
+    # a line at a time, so a prompt waiting on its line wouldn't show
+    while True:
+        prompt = f"{message.rstrip('.')}. Press Enter when done."
+        # Paths aren't split, so they can still be copied
+        print(
+            textwrap.fill(
+                prompt,
+                width=80,
+                break_long_words=False,
+                break_on_hyphens=False,
+            ),
+            flush=True,
+        )
+        ask()
+        missing = [path for path in outputs if not os.path.exists(path)]
+        if missing:
+            typer.echo(f"Not there yet: {', '.join(missing)}")
+            continue
+        # Unchanged is fine when it was already done, but is asked about,
+        # since otherwise it's an old output recorded as a new one
+        unchanged = [
+            path for path in outputs if hash_path(path) == before[path]
+        ]
+        if unchanged:
+            print(
+                f"Unchanged: {', '.join(unchanged)}. Keep as is? [y/N]",
+                flush=True,
+            )
+            if ask().strip().lower() not in ("y", "yes"):
+                continue
+        break
     typer.echo("Done")
 
 
@@ -4429,7 +4567,8 @@ def run_in_env(
             typer.echo(f"Running command: {cmd}")
         try:
             subprocess.check_call(cmd, cwd=wdir)
-        except subprocess.CalledProcessError:
+        except subprocess.CalledProcessError as e:
+            warn_app_control(returncode=e.returncode)
             raise_error("Failed to run in Conda environment")
     elif env["kind"] == "pixi":
         env_cmd = []
@@ -4443,7 +4582,8 @@ def run_in_env(
             typer.echo(f"Running command: {cmd}")
         try:
             subprocess.check_call(cmd, cwd=wdir)
-        except subprocess.CalledProcessError:
+        except subprocess.CalledProcessError as e:
+            warn_app_control(returncode=e.returncode)
             raise_error("Failed to run in Pixi environment")
     elif env["kind"] == "uv":
         env_dir = os.path.dirname(os.path.abspath(env["path"]))
@@ -4452,7 +4592,8 @@ def run_in_env(
             typer.echo(f"Running command: {cmd}")
         try:
             subprocess.check_call(cmd, cwd=wdir)
-        except subprocess.CalledProcessError:
+        except subprocess.CalledProcessError as e:
+            warn_app_control(returncode=e.returncode)
             raise_error("Failed to run in uv environment")
     elif (kind := env["kind"]) in ["uv-venv", "venv"]:
         if "path" not in env:
@@ -4490,7 +4631,8 @@ def run_in_env(
             typer.echo(f"Running command: {cmd}")
         try:
             subprocess.check_call(cmd, shell=True, cwd=wdir)
-        except subprocess.CalledProcessError:
+        except subprocess.CalledProcessError as e:
+            warn_app_control(returncode=e.returncode)
             raise_error(f"Failed to run in {kind}")
     elif env["kind"] == "julia":
         if not no_check: