calkit-python 0.31.4__tar.gz → 0.32.0__tar.gz

{calkit_python-0.31.4 → calkit_python-0.32.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.31.4
+Version: 0.32.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -15,9 +15,10 @@ Requires-Dist: arithmeval
 Requires-Dist: bibtexparser
 Requires-Dist: checksumdir
 Requires-Dist: docx2pdf
-Requires-Dist: dvc==3.62.0
+Requires-Dist: dvc==3.64.0
 Requires-Dist: fastapi
 Requires-Dist: gitpython
+Requires-Dist: json2latex>=0.0.2
 Requires-Dist: jupyterlab>=4.4.5
 Requires-Dist: keyring
 Requires-Dist: nbconvert
@@ -30,6 +31,7 @@ Requires-Dist: pyjwt
 Requires-Dist: python-dotenv>=1
 Requires-Dist: pywin32; platform_system == 'Windows'
 Requires-Dist: requests
+Requires-Dist: sqlitedict>=2.1.0
 Requires-Dist: tqdm>=4.67.1
 Requires-Dist: typer==0.20.0
 Requires-Dist: uvicorn

{calkit_python-0.31.4 → calkit_python-0.32.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.31.4"
+__version__ = "0.32.0"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.31.4 → calkit_python-0.32.0}/calkit/cli/check.py

@@ -208,6 +208,35 @@ def check_environment(
     return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)
 
 
+@check_app.command(
+    name="envs",
+    help="Check that all environments are up-to-date.",
+)
+@check_app.command(name="environments")
+def check_environments(
+    verbose: Annotated[
+        bool, typer.Option("--verbose", help="Print verbose output.")
+    ] = False,
+) -> str | None:
+    ck_info = calkit.load_calkit_info(process_includes="environments")
+    envs = ck_info.get("environments", {})
+    if not envs:
+        typer.echo("No environments defined in calkit.yaml")
+        return
+    failures = []
+    for env_name in envs.keys():
+        typer.echo(f"Checking environment: '{env_name}'")
+        try:
+            check_environment(env_name=env_name, verbose=verbose)
+        except Exception as e:
+            warn(f"Error checking environment '{env_name}': {e}")
+            failures.append(env_name)
+    if failures:
+        raise_error(
+            f"Failed to check the following environments: {', '.join(failures)}"
+        )
+
+
 @check_app.command(name="docker-env")
 def check_docker_env(
     tag: Annotated[str, typer.Argument(help="Image tag.")],

calkit_python-0.32.0/calkit/cli/latex.py

@@ -0,0 +1,206 @@
+"""Commands for working with LaTeX."""
+
+from __future__ import annotations
+
+import json
+import os
+import string
+import subprocess
+from copy import deepcopy
+
+import typer
+from typing_extensions import Annotated
+
+import calkit
+from calkit.cli import raise_error
+
+latex_app = typer.Typer(no_args_is_help=True)
+
+
+@latex_app.command(name="from-json")
+def from_json(
+    input_fpaths: Annotated[
+        list[str], typer.Argument(help="Input JSON file path(s).")
+    ],
+    output_fpaths: Annotated[
+        list[str],
+        typer.Option("--output", "-o", help="Output LaTeX file path(s)."),
+    ],
+    command_name: Annotated[
+        str | None,
+        typer.Option("--command", help="Command name to use in LaTeX output."),
+    ] = None,
+    fmt_json: Annotated[
+        str | None,
+        typer.Option(
+            "--format-json",
+            help=(
+                "Additional JSON input to use for formatting. "
+                "Can be used to add extra keys with simple expressions, etc."
+            ),
+        ),
+    ] = None,
+):
+    """Convert a JSON file to LaTeX.
+
+    This is useful for referencing calculated values in LaTeX documents.
+    """
+    import arithmetic_eval
+    import json2latex
+
+    def tokens_from_format_string(fmt: str):
+        return [
+            field.strip()
+            for _, field, _, _ in string.Formatter().parse(fmt)
+            if field
+        ]
+
+    # Validate some stuff
+    if fmt_json is not None:
+        try:
+            fmt_dict = json.loads(fmt_json)
+        except json.JSONDecodeError:
+            raise_error("Format JSON is not valid JSON")
+    else:
+        fmt_dict = {}
+    data = {}
+    for input_fpath in input_fpaths:
+        if not os.path.isfile(input_fpath):
+            raise_error(f"Input file {input_fpath} does not exist")
+        if not input_fpath.endswith(".json"):
+            raise_error("Input file must be a JSON file")
+        with open(input_fpath) as f:
+            try:
+                data_i = json.load(f)
+                data.update(data_i)
+            except json.JSONDecodeError:
+                raise_error("Input JSON file is not valid JSON")
+    for output_fpath in output_fpaths:
+        if not output_fpath.endswith(".tex"):
+            raise_error("Output file must be a .tex file")
+    # Format the data
+    formatted = deepcopy(data)
+    for tex_var_name, fmt_string in fmt_dict.items():
+        fmt_string = str(fmt_string)
+        data_for_formatting = deepcopy(data)
+        # Do any relevant evals and add them to the data for formatting
+        tokens = tokens_from_format_string(fmt_string)
+        for t in tokens:
+            try:
+                data_for_formatting[t] = arithmetic_eval.evaluate(t, data)
+            except Exception:
+                raise_error(
+                    f"Error evaluating expression '{t}' for formatting"
+                )
+        formatted[tex_var_name] = fmt_string.format(**data_for_formatting)
+    for out_path in output_fpaths:
+        # If no command is provided, use the output file name without extension
+        if command_name is None:
+            cmd_name = os.path.splitext(os.path.basename(out_path))[0]
+        else:
+            cmd_name = command_name
+        # Create output directory if it doesn't exist
+        outdir = os.path.dirname(out_path)
+        if outdir:
+            os.makedirs(outdir, exist_ok=True)
+        with open(out_path, "w") as f:
+            json2latex.dump(cmd_name, formatted, f)
+
+
+@latex_app.command(name="build")
+def build(
+    tex_file: Annotated[str, typer.Argument(help="The .tex file to compile.")],
+    environment: Annotated[
+        str | None,
+        typer.Option(
+            "--env",
+            "-e",
+            help=("Environment in which to run latexmk, if applicable."),
+        ),
+    ] = None,
+    no_check: Annotated[
+        bool,
+        typer.Option(
+            "--no-check",
+            help=(
+                "Don't check the environment is valid before running latexmk."
+            ),
+        ),
+    ] = False,
+    latexmk_rc_path: Annotated[
+        str | None,
+        typer.Option(
+            "--latexmk-rc",
+            "-r",
+            help="Path to a latexmkrc file to use for compilation.",
+        ),
+    ] = None,
+    no_synctex: Annotated[
+        bool,
+        typer.Option(
+            "--no-synctex",
+            help="Don't generate synctex file for source-to-pdf mapping.",
+        ),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option(
+            "--force",
+            "-f",
+            help=(
+                "Force latexmk to recompile all files, even if they are up to "
+                "date."
+            ),
+        ),
+    ] = False,
+    verbose: Annotated[
+        bool, typer.Option("--verbose", "-v", help="Print verbose output.")
+    ] = False,
+):
+    """Build a PDF of a LaTeX document with latexmk.
+
+    If a Calkit environment is not specified, latexmk will be run in the
+    system environment if available. If not available, a TeX Live Docker
+    container will be used.
+    """
+    # Now formulate the command
+    latexmk_cmd = ["latexmk", "-pdf", "-cd"]
+    if latexmk_rc_path is not None:
+        latexmk_cmd += ["-r", latexmk_rc_path]
+    if not no_synctex:
+        latexmk_cmd.append("-synctex=1")
+    if not verbose:
+        latexmk_cmd.append("-silent")
+    if force:
+        latexmk_cmd.append("-f")
+    latexmk_cmd += ["-interaction=nonstopmode", tex_file]
+    if environment is not None:
+        if no_check:
+            check_cmd = ["--no-check"]
+        else:
+            check_cmd = []
+        cmd = (
+            ["calkit", "xenv", "--name", environment]
+            + check_cmd
+            + ["--"]
+            + latexmk_cmd
+        )
+        if verbose:
+            typer.echo(f"Running command: {cmd}")
+    elif calkit.check_dep_exists("latexmk"):
+        cmd = latexmk_cmd
+    else:
+        cmd = [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{os.getcwd()}:/work",
+            "-w",
+            "/work",
+            "texlive/texlive:latest-full",
+        ] + latexmk_cmd
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError:
+        raise_error("latexmk failed")

{calkit_python-0.31.4 → calkit_python-0.32.0}/calkit/cli/main.py

@@ -16,7 +16,7 @@ import time
 import uuid
 from copy import deepcopy
 from datetime import datetime
-from pathlib import PurePosixPath
+from pathlib import Path
 
 import dotenv
 import git
@@ -40,6 +40,7 @@ from calkit.cli.cloud import cloud_app
 from calkit.cli.config import config_app
 from calkit.cli.describe import describe_app
 from calkit.cli.import_ import import_app
+from calkit.cli.latex import latex_app
 from calkit.cli.list import list_app
 from calkit.cli.new import new_app
 from calkit.cli.notebooks import notebooks_app
@@ -70,6 +71,7 @@ app.add_typer(import_app, name="import", help="Import objects.")
 app.add_typer(office_app, name="office", help="Work with Microsoft Office.")
 app.add_typer(update_app, name="update", help="Update objects.")
 app.add_typer(check_app, name="check", help="Check things.")
+app.add_typer(latex_app, name="latex", help="Work with LaTeX.")
 app.add_typer(overleaf_app, name="overleaf", help="Interact with Overleaf.")
 app.add_typer(cloud_app, name="cloud", help="Interact with a Calkit Cloud.")
 app.add_typer(slurm_app, name="slurm", help="Work with SLURM.")
@@ -264,6 +266,11 @@ def get_status():
         calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
     except Exception as e:
         warn(f"Failed to compile pipeline: {e.__class__.__name__}: {e}")
+    # Clean all notebooks in the pipeline
+    try:
+        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
+    except Exception as e:
+        warn(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
     print_sep("Project")
     # Print latest status
     status = calkit.get_latest_project_status()
@@ -287,7 +294,7 @@ def get_status():
     print_sep("Data (DVC)")
     run_cmd([sys.executable, "-m", "dvc", "data", "status"])
     typer.echo()
-    print_sep("Pipeline (DVC)")
+    print_sep("Pipeline")
     run_cmd([sys.executable, "-m", "dvc", "status"])
 
 
@@ -769,7 +776,7 @@ def ignore(
     """Ignore a file, i.e., keep it out of version control."""
     repo = git.Repo()
     # Ensure path makes it into .gitignore as a POSIX path
-    path = PurePosixPath(path).as_posix()
+    path = Path(path).as_posix()
     if repo.ignored(path):
         typer.echo(f"{path} is already ignored")
         return
@@ -1022,6 +1029,9 @@ def run(
     import dvc.ui
     from dvc.cli import main as dvc_cli_main
 
+    import calkit.environments
+    import calkit.pipeline
+
     if (target_inputs or target_outputs) and targets:
         raise_error("Cannot specify both targets and inputs")
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
@@ -1029,8 +1039,14 @@ def run(
     ck_info = calkit.load_calkit_info()
     # Set env vars
     calkit.set_env_vars(ck_info=ck_info)
+    # Clean all notebooks in the pipeline
+    try:
+        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
+    except Exception as e:
+        raise_error(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
     if not quiet:
         typer.echo("Getting system information")
+    # Get system information
     system_info = calkit.get_system_info()
     if save_log:
         # Save the system to .calkit/systems
@@ -1047,11 +1063,23 @@ def run(
     if not quiet:
         typer.echo("Checking system-level dependencies")
     try:
-        calkit.check_system_deps(system_info=system_info)
+        calkit.check_system_deps(ck_info=ck_info, system_info=system_info)
     except Exception as e:
         os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
         raise_error(str(e))
-    # Compile the pipeline
+    # Check all environments in the pipeline (with caching)
+    # If any failed, warn the user that we might have problems running
+    typer.echo("Checking environments")
+    env_check_results = calkit.environments.check_all_in_pipeline(
+        ck_info=ck_info, targets=targets, force=force
+    )
+    for env_name, result in env_check_results.items():
+        if verbose:
+            typer.echo(f"{env_name}: {result}")
+        failed = not result.get("success", False)
+        if failed:
+            warn(f"Failed to check environment '{env_name}'")
+    # Compile the DVC pipeline
     dvc_stages = None
     if ck_info.get("pipeline", {}):
         if not quiet:
@@ -2013,49 +2041,123 @@ def run_jupyter(
     sys.exit(process.returncode)
 
 
-@app.command(name="latexmk")
-def run_latexmk(
-    tex_file: Annotated[str, typer.Argument(help="The .tex file to compile.")],
-    environment: Annotated[
-        str | None,
+@app.command(name="map-paths")
+def map_paths(
+    file_to_file: Annotated[
+        list[str],
         typer.Option(
-            "--env",
-            "-e",
-            help=("Environment in which to run latexmk, if applicable."),
+            "--file-to-file",
+            help=(
+                "Map a file to another file, e.g., "
+                "--file-to-file 'results.tex->paper/results.tex'."
+            ),
         ),
-    ] = None,
+    ] = [],
+    file_to_dir: Annotated[
+        list[str],
+        typer.Option(
+            "--file-to-dir",
+            help=(
+                "Map a file into a directory, e.g., "
+                "--file-to-dir 'results.tex->paper/results'."
+            ),
+        ),
+    ] = [],
+    dir_to_dir_replace: Annotated[
+        list[str],
+        typer.Option(
+            "--dir-to-dir-replace",
+            help=(
+                "Copy directory to another directory and replace it, "
+                "e.g., --dir-to-dir-replace 'figures->paper/figures'."
+            ),
+        ),
+    ] = [],
+    dir_to_dir_merge: Annotated[
+        list[str],
+        typer.Option(
+            "--dir-to-dir-merge",
+            help=(
+                "Merge directory into another directory. "
+                "This is useful for merging contents of one directory into "
+                "another, e.g., --dir-to-dir-merge 'figures->paper/figures'."
+            ),
+        ),
+    ] = [],
 ):
-    """Compile a LaTeX document with latexmk.
+    """Map paths in a project.
 
-    If a Calkit environment is not specified, latexmk will be run in the
-    system environment if available. If not available, a TeX Live Docker
-    container will be used.
+    Currently this is done with copying. Outputs are ensured to be ignored by
+    Git.
     """
-    latexmk_cmd = [
-        "latexmk",
-        "-pdf",
-        "-cd",
-        "-silent",
-        "-synctex=1",
-        "-interaction=nonstopmode",
-        tex_file,
-    ]
-    if environment is not None:
-        cmd = ["calkit", "xenv", "--name", environment] + latexmk_cmd
-    elif calkit.check_dep_exists("latexmk"):
-        cmd = latexmk_cmd
-    else:
-        cmd = [
-            "docker",
-            "run",
-            "--rm",
-            "-v",
-            f"{os.getcwd()}:/work",
-            "-w",
-            "/work",
-            "texlive/texlive:latest-full",
-        ] + latexmk_cmd
-    try:
-        subprocess.check_call(cmd)
-    except subprocess.CalledProcessError:
-        raise_error("latexmk failed")
+    repo = git.Repo()
+
+    def ensure_path_is_ignored(path):
+        if not repo.ignored(path):
+            typer.echo(f"Adding {path} to .gitignore")
+            with open(".gitignore", "a") as f:
+                f.write(f"\n{path}\n")
+
+    def validate_and_split(mapping: str) -> tuple[str, str]:
+        if "->" not in mapping:
+            raise_error(
+                f"Invalid path mapping format: '{mapping}'; "
+                "Expected format: 'src->dest'"
+            )
+        parts = mapping.split("->")
+        if len(parts) != 2:
+            raise_error(
+                f"Invalid path mapping format: '{mapping}'; "
+                "Expected exactly one '->' separator"
+            )
+        return parts[0].strip(), parts[1].strip()
+
+    for copy_file in file_to_file:
+        src_path, dest_path = validate_and_split(copy_file)
+        if os.path.isdir(dest_path):
+            raise_error(f"Destination path '{dest_path}' is a directory")
+        parent_dir = os.path.dirname(dest_path)
+        if parent_dir:
+            os.makedirs(parent_dir, exist_ok=True)
+        shutil.copy2(src_path, dest_path)
+        ensure_path_is_ignored(dest_path)
+    for copy_file in file_to_dir:
+        src_path, dest_dir = validate_and_split(copy_file)
+        if os.path.isfile(dest_dir):
+            raise_error(f"Destination path '{dest_dir}' is a file")
+        if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir, exist_ok=True)
+        dest_path = os.path.join(dest_dir, os.path.basename(src_path))
+        shutil.copy2(src_path, dest_path)
+        ensure_path_is_ignored(dest_path)
+    for replace_dir_with_dir in dir_to_dir_replace:
+        src_dir, dest_dir = validate_and_split(replace_dir_with_dir)
+        if os.path.isfile(dest_dir):
+            raise_error(f"Destination path '{dest_dir}' is a file")
+        if os.path.isfile(src_dir):
+            raise_error(f"Source path '{src_dir}' is a file")
+        if os.path.isdir(dest_dir):
+            shutil.rmtree(dest_dir)
+        parent_dir = os.path.dirname(dest_dir)
+        if parent_dir:
+            os.makedirs(parent_dir, exist_ok=True)
+        shutil.copytree(src_dir, dest_dir)
+        ensure_path_is_ignored(dest_dir)
+    for merge_dir_to_dir in dir_to_dir_merge:
+        src_dir, dest_dir = validate_and_split(merge_dir_to_dir)
+        if os.path.isfile(dest_dir):
+            raise_error(f"Destination path '{dest_dir}' is a file")
+        if os.path.isfile(src_dir):
+            raise_error(f"Source path '{src_dir}' is a file")
+        if not os.path.isdir(dest_dir):
+            os.makedirs(dest_dir, exist_ok=True)
+        for item in os.listdir(src_dir):
+            if item.startswith("."):
+                continue
+            src_item = os.path.join(src_dir, item)
+            dest_item = os.path.join(dest_dir, item)
+            if os.path.isdir(src_item):
+                shutil.copytree(src_item, dest_item, dirs_exist_ok=True)
+            else:
+                shutil.copy2(src_item, dest_item)
+            ensure_path_is_ignored(dest_item)

{calkit_python-0.31.4 → calkit_python-0.32.0}/calkit/cli/new.py

@@ -2838,7 +2838,9 @@ def new_stage(
     if kind.value == "python-script":
         cmd += f"python {target}"
     elif kind.value == "latex":
-        cmd += f"latexmk -cd -interaction=nonstopmode -pdf {target}"
+        cmd = f"calkit latex build {target}"
+        if environment is not None:
+            cmd += f" --environment {environment}"
         out_target = target.removesuffix(".tex") + ".pdf"
         if out_target not in (
             outs + outs_no_cache + outs_persist + outs_persist_no_cache

{calkit_python-0.31.4 → calkit_python-0.32.0}/calkit/cli/notebooks.py

@@ -7,12 +7,13 @@ import json
 import os
 import subprocess
 import sys
-from pathlib import PurePosixPath
+from pathlib import Path
 from typing import Any
 
 import typer
 from typing_extensions import Annotated
 
+import calkit
 import calkit.notebooks
 from calkit.cli.core import raise_error
 
@@ -20,32 +21,43 @@ notebooks_app = typer.Typer(no_args_is_help=True)
 
 
 @notebooks_app.command("clean")
-def clean_notebook_outputs(path: str):
+def clean_notebook_outputs(
+    path: str,
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Do not print output.")
+    ] = False,
+):
     """Clean notebook and place a copy in the cleaned notebooks directory.
 
     This can be useful to use as a preprocessing DVC stage to use a clean
     notebook as a dependency for a stage that caches and executed notebook.
     """
     if os.path.isabs(path):
-        raise ValueError("Path must be relative")
-    fpath_out = calkit.notebooks.get_cleaned_notebook_path(path)
-    folder = os.path.dirname(fpath_out)
-    os.makedirs(folder, exist_ok=True)
-    fpath_out = os.path.abspath(fpath_out)
-    subprocess.call(
-        [
-            sys.executable,
-            "-m",
-            "jupyter",
-            "nbconvert",
-            path,
-            "--clear-output",
-            "--to",
-            "notebook",
-            "--output",
-            fpath_out,
-        ]
-    )
+        raise_error("Path must be relative")
+    if not quiet:
+        typer.echo(f"Cleaning notebook: {path}")
+    try:
+        calkit.notebooks.clean_notebook_outputs(path)
+    except Exception as e:
+        raise_error(str(e))
+
+
+@notebooks_app.command("clean-all")
+def clean_all_in_pipeline(
+    quiet: Annotated[
+        bool, typer.Option("--quiet", "-q", help="Do not print output.")
+    ] = False,
+):
+    """Clean all notebooks in the pipeline."""
+    if not quiet:
+        typer.echo("Cleaning all notebooks in pipeline")
+    try:
+        cleaned = calkit.notebooks.clean_all_in_pipeline()
+        if not quiet:
+            for path in cleaned:
+                typer.echo(f"Cleaned: {path}")
+    except Exception as e:
+        raise_error(str(e))
 
 
 def _parse_params(params: list[str]) -> dict[str, Any]:
@@ -379,7 +391,7 @@ def execute_notebook(
                 "--to",
                 to_fmt,
                 "--output-dir",
-                PurePosixPath(folder).as_posix(),
+                Path(folder).as_posix(),
                 "--output",
                 fname_out,
             ]