calkit-python 0.48.2__tar.gz → 0.48.3__tar.gz

{calkit_python-0.48.2 → calkit_python-0.48.3}/.pre-commit-config.yaml

@@ -85,10 +85,10 @@ repos:
         # the pipeline changes. Prettier and ruamel disagree about where a
         # long 'desc:' value goes, so formatting it here just means the two
         # take turns reformatting the same file.
-        # Likewise results files, here and in the example projects, which
-        # are stage outputs and have to match what the stage wrote byte for
-        # byte.
-        exclude: calkit|^hub/|(^|/)dvc\.yaml$|^(examples/.*/)?results/
+        # Likewise results files, here, in the example projects, and in
+        # decisions, which are stage outputs and have to match what the stage
+        # wrote byte for byte.
+        exclude: calkit|^hub/|(^|/)dvc\.yaml$|^((examples/.*|decisions)/)?results/
 
   - repo: local
     hooks:

{calkit_python-0.48.2 → calkit_python-0.48.3}/AGENTS.md

@@ -25,6 +25,9 @@ To run tests, use `uv run pytest`.
 
 To sync the docs and format all the code, run `make format`.
 
+Before pushing a pull request, run `make pr`, which also runs the project's
+pipeline and pushes its outputs, so the pipeline check passes.
+
 Before finishing a change, type-check it. The CLI gate is mypy
 (`uv run mypy <changed files>`, or `make check` for the full suite), and the
 VS Code editor uses Pylance (Pyright). Keep new code clean under both, and do

{calkit_python-0.48.2 → calkit_python-0.48.3}/Makefile

@@ -28,6 +28,23 @@ format: ## Automatically format files and regenerate what's generated.
 	@echo "🚀 Linting code with pre-commit"
 	@uv run pre-commit run -a
 
+.PHONY: pr
+pr: ## Get a pull request ready to pass: format, and run the pipeline.
+	@echo "🚀 Formatting, then checking it's done"
+	@uv run pre-commit run -a > /dev/null || uv run pre-commit run -a
+	@echo "🚀 Pulling the pipeline's outputs, so only what changed reruns"
+	@uv run calkit dvc pull || true
+	@echo "🚀 Running the pipeline"
+	@uv run calkit run
+	@echo "🚀 Pushing the pipeline's outputs"
+	@uv run calkit dvc push
+	@echo "🚀 Checking the run left nothing to format or rerun"
+	@files="$$(git diff --name-only)"; \
+		[ -z "$$files" ] || uv run pre-commit run --files $$files
+	@uv run calkit status --json | uv run python -c \
+		"import json, sys; sys.exit(json.load(sys.stdin)['pipeline']['is_stale'])" \
+		|| (echo "The pipeline is stale; see 'calkit status'" && exit 1)
+
 .PHONY: check
 check: format ## Run code quality tools.
 	@echo "🚀 Checking lock file consistency with 'pyproject.toml'"

{calkit_python-0.48.2 → calkit_python-0.48.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: calkit-python
-Version: 0.48.2
+Version: 0.48.3
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -47,6 +47,7 @@ Requires-Dist: python-dotenv>=1
 Requires-Dist: pywin32; platform_system == 'Windows'
 Requires-Dist: requests
 Requires-Dist: rich
+Requires-Dist: ruamel-yaml-clib>=0.2.15; platform_python_implementation == 'CPython' and python_version < '3.15' and (sys_platform == 'darwin' or (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64' or platform_machine == 'i686')) or (sys_platform == 'win32' and (platform_machine == 'AMD64' or platform_machine == 'x86')))
 Requires-Dist: ruamel-yaml<0.20,>=0.19.1
 Requires-Dist: sqlitedict>=2.1.0
 Requires-Dist: texsoup>=0.3.3

{calkit_python-0.48.2 → calkit_python-0.48.3}/calkit/calc.py

@@ -4,14 +4,20 @@ from __future__ import annotations
 
 from typing import Annotated, Literal
 
-from pydantic import BaseModel, Discriminator, model_validator
+from pydantic import BaseModel, ConfigDict, Discriminator, model_validator
 
 DTYPES = {"int": int, "float": float, "str": str}
 DEFAULT_IN_TYPE = "float"
 DEFAULT_OUT_TYPE = "float"
 
 
-class Input(BaseModel):
+class _Model(BaseModel):
+    # As calkit.models.base.Model, which importing here would import
+    # calkit.models, which imports this
+    model_config = ConfigDict(defer_build=True)
+
+
+class Input(_Model):
     name: str
     description: str | None = None
     dtype: Literal["int", "float", "str"] = DEFAULT_IN_TYPE
@@ -19,14 +25,14 @@ class Input(BaseModel):
     max: int | float | None = None
 
 
-class Output(BaseModel):
+class Output(_Model):
     name: str
     description: str | None = None
     dtype: Literal["int", "float", "str"] = DEFAULT_OUT_TYPE
     template: str | None = None
 
 
-class Calculation(BaseModel):
+class Calculation(_Model):
     kind: str
     params: dict = {}
     name: str | None = None
@@ -124,7 +130,7 @@ class Calculation(BaseModel):
         return template.format(**(inputs | {out_name: res}))
 
 
-class FormulaParams(BaseModel):
+class FormulaParams(_Model):
     formula: str
 
 
@@ -139,7 +145,7 @@ class Formula(Calculation):
         return arithmetic_eval.evaluate(self.params.formula, inputs)
 
 
-class LinearParams(BaseModel):
+class LinearParams(_Model):
     coeffs: dict[str, float]
     offset: float = 0.0
 
@@ -163,7 +169,7 @@ class Linear(Calculation):
         return val
 
 
-class LookupTableParams(BaseModel):
+class LookupTableParams(_Model):
     x_values: list[float]
     y_values: list[float]
     method: Literal["floor", "ceil", "round", "interpolate"] = "interpolate"
@@ -176,7 +182,7 @@ class LookupTable(Calculation):
     params: LookupTableParams
 
 
-class HttpRequestParams(BaseModel):
+class HttpRequestParams(_Model):
     url: str
     inputs_as_params: bool = True  # Otherwise, use body
     method: Literal["get", "post", "put"] = "get"
@@ -210,7 +216,7 @@ class HttpRequest(Calculation):
             return resp.text
 
 
-class XGBoostModelParams(BaseModel):
+class XGBoostModelParams(_Model):
     path: str
     type: Literal["classifier", "regressor"]
 

{calkit_python-0.48.2 → calkit_python-0.48.3}/calkit/cli/check.py

@@ -2026,6 +2026,95 @@ def check_pipeline(
             )
 
 
+@check_app.command(name="storage")
+def check_storage(
+    fix: Annotated[
+        bool,
+        typer.Option(
+            "--fix", help="Move each path to where it's declared to be stored."
+        ),
+    ] = False,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Output the problems as JSON.")
+    ] = False,
+) -> None:
+    """Check that paths are tracked where they're declared to be stored.
+
+    That is, nothing is tracked by both Git and DVC, no pipeline output
+    stored in Git is ignored by it, and nothing in Git is too large for it.
+    With --fix, the pipeline's declarations, or a file's .dvc file, decide
+    where each path belongs, and the changes are staged.
+    """
+    import calkit.dvc
+    import calkit.git
+    import calkit.pipeline
+
+    if fix and as_json:
+        raise_error("--json can't be used with --fix")
+    problems = calkit.dvc.get_storage_problems()
+    if as_json:
+        typer.echo(json.dumps(problems, indent=2))
+    if not fix:
+        if not as_json:
+            for problem in problems:
+                warn(problem["message"])
+        if any(p["kind"] != "large" for p in problems):
+            if not as_json:
+                typer.echo("Run 'calkit check storage --fix' to fix these")
+            raise typer.Exit(1)
+        if not problems and not as_json:
+            calkit.echo("✅ Every path is stored where it's declared to be")
+        return
+    repo = calkit.git.get_repo()
+    gitignores = calkit.git.get_dirty_gitignores(repo)
+    fixed = []
+    unfixed = []
+    # Pipeline outputs are recorded in DVC, which records their stages'
+    # inputs too, so only for stages that are up to date
+    to_record = []
+    for problem in problems:
+        path = os.path.abspath(problem["path"])
+        if problem["store_in"] == "dvc":
+            repo.git.rm("-r", "-q", "--cached", "--", path)
+            calkit.git.ensure_path_is_ignored(repo, path)
+            if problem["kind"] == "both":
+                to_record.append(problem)
+        elif problem["store_in"] == "git":
+            calkit.git.ensure_path_is_not_ignored(repo, path)
+            repo.git.add("--", path)
+        else:
+            unfixed.append(problem)
+            continue
+        fixed.append(problem)
+        typer.echo(f"Fixed: {problem['message']}")
+    # None when it can't be told which are stale, so none are recorded
+    stale: set[str] | None = set()
+    if any(p["stage"] for p in to_record):
+        status = calkit.pipeline.get_status(
+            check_environments=False, clean_notebooks=False
+        )
+        if not status.errors:
+            stale = {n.split("@")[0] for n in status.stale_stage_names}
+        else:
+            stale = None
+    for problem in to_record:
+        stage = problem["stage"]
+        if stage and (stale is None or stage.split("@")[0] in stale):
+            typer.echo(f"Run '{stage}' to record {problem['path']} in DVC")
+            continue
+        target = problem["dvc_file"] or stage
+        if calkit.dvc.run_dvc_command(["commit", "-f", "-q", target]) != 0:
+            warn(f"Couldn't record {problem['path']} in DVC")
+    for message in calkit.git.stage_gitignores(repo, gitignores):
+        warn(message)
+    for problem in unfixed:
+        warn(problem["message"])
+    if fixed:
+        typer.echo("Commit the staged changes")
+    if any(p["kind"] != "large" for p in unfixed):
+        raise typer.Exit(1)
+
+
 @check_app.command(name="call")
 def check_call(
     cmd: Annotated[str, typer.Argument(help="Command to check.")],
@@ -2082,8 +2171,9 @@ def check_questions(
     pushed, or pinned to a Git ref that doesn't exist); broken references
     (a key that doesn't resolve, a placeholder that names no evidence, a
     label missing from the LaTeX); evidence the pipeline would rebuild;
-    and evidence from a frozen stage, or downstream of one, which nothing
-    will ever report out of date unless the citation pins a git_ref.
+    and evidence from a frozen stage, or downstream of one, that no longer
+    matches dvc.lock, which won't be rerun, unless the citation pins a
+    git_ref. Subprojects' questions are checked too.
 
     Evidence pinned with a git_ref is checked at that ref rather than in
     the working tree. Exits with an error if any answered question is

{calkit_python-0.48.2 → calkit_python-0.48.3}/calkit/cli/describe.py

@@ -9,7 +9,6 @@ import typer
 
 import calkit
 from calkit.cli import AliasGroup, echo_json, raise_error
-from calkit.environments import get_env_lock_fpath
 
 describe_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
@@ -20,6 +19,10 @@ def _describe_env(env: dict, env_name: str) -> dict:
     The keys here are a stable contract for tools consuming ``--json``, so
     they are always present, even if their value is null.
     """
+    # Here rather than at the top, since every command imports this module;
+    # what that saves is a question in decisions/calkit.yaml
+    from calkit.environments import get_env_lock_fpath
+
     return {
         "kind": env.get("kind"),
         "spec_path": env.get("path"),

{calkit_python-0.48.2 → calkit_python-0.48.3}/calkit/cli/latex.py

@@ -328,8 +328,10 @@ def from_markdown(
         if "://" not in hub:
             hub = "https://" + hub
         project_url = f"{hub}/{ck_info['owner']}/{ck_info['name']}"
-    # A local version's commit is the project's own, which isn't Calkit's
-    version = calkit.__version__.split("+")[0]
+    # A local version's commit is the project's own, which isn't Calkit's,
+    # and a development build's count of commits since the last release
+    # changes with every commit, which would change the PDF on every rebuild
+    version = re.sub(r"\.dev\d+$", ".dev", calkit.__version__.split("+")[0])
     with open(md_path, encoding="utf-8") as f:
         text = f.read()
     try:
@@ -2008,6 +2010,11 @@ def to_docx(
             return out
 
         existing = listing() - {job + ".bbl"}
+        # Dated like the PDF, so \today matches it and a rebuild of an
+        # unchanged source doesn't change
+        tex_env_vars = _tex_env_vars(
+            calkit.latex.get_source_date_epoch(source)
+        )
         # TeX can loop forever, e.g., older TeX4ht on a cases environment,
         # so a run gets a time limit, and loses its LaTeX child with it
         import time
@@ -2021,6 +2028,7 @@ def to_docx(
                 proc = subprocess.Popen(
                     m4h,
                     cwd=src_dir,
+                    env=os.environ | tex_env_vars,
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
                     start_new_session=sys.platform != "win32",
@@ -2053,6 +2061,7 @@ def to_docx(
                         no_check=True,
                         verbose=False,
                         dep="make4ht",
+                        env_vars=tex_env_vars,
                     ),
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
@@ -2571,7 +2580,20 @@ def to_docx(
         dirty = any(repo.is_dirty(path=p) for p in {ln.path for ln in lines})
     except Exception:
         pass
-    export_id = str(uuid.uuid4())
+    # At random, so each export can be told apart, unless the build is asked
+    # to be reproducible, in which case it comes from what was exported
+    if os.environ.get("SOURCE_DATE_EPOCH"):
+        doc.save()
+        export_id = str(
+            uuid.uuid5(
+                uuid.NAMESPACE_URL,
+                json.dumps(
+                    [calkit.get_md5(output), rev, source, original, equations]
+                ),
+            )
+        )
+    else:
+        export_id = str(uuid.uuid4())
     doc.write_original(
         calkit.docx.Original(
             export_id,

{calkit_python-0.48.2 → calkit_python-0.48.3}/calkit/cli/main/core.py

@@ -589,6 +589,7 @@ def get_status(
             clean_notebooks=True,
             compile_to_dvc=True,
             with_data_status="dvc" in categories,
+            with_storage_problems="dvc" in categories,
         )
         if pipeline_status.failed_environment_checks:
             warn(
@@ -629,6 +630,17 @@ def get_status(
             frozen_stages=frozen_stages,
             check_history=False,
         )
+    # Paths tracked where they shouldn't be, found with the pipeline's status,
+    # which builds the DVC index this needs anyway, unless it stopped short
+    storage_problems = None
+    if "dvc" in categories and running_status is None:
+        if pipeline_status is not None:
+            storage_problems = pipeline_status.storage_problems
+        if storage_problems is None:
+            try:
+                storage_problems = calkit.dvc.get_storage_problems()
+            except Exception:
+                pass
     if as_json:
         status_dict: dict[str, Any] = {}
         if "project" in categories:
@@ -694,6 +706,7 @@ def get_status(
                 )
             except Exception as e:
                 status_dict["dvc"] = {"error": f"{e.__class__.__name__}: {e}"}
+            status_dict["storage"] = storage_problems
         if "pipeline" in categories or "dvc" in categories:
             if running_status is not None:
                 status_dict["pipeline"] = running_status
@@ -760,6 +773,10 @@ def get_status(
             raw.pop("git", None)
             raw = calkit.dvc.data_status_as_posix(raw)
             typer.echo(_format_dvc_data_status(raw, zip_path_map))
+            for problem in storage_problems or []:
+                warn(problem["message"])
+            if storage_problems:
+                typer.echo("Run 'calkit check storage' for detail.\n")
     if "pipeline" in categories or "dvc" in categories:
         print_sep("Pipeline")
         if running_status is not None:
@@ -1011,6 +1028,22 @@ def add(
         )
         repo.git.add(".dvc/config")
     dvc_paths = [] if dvc_repo is None else calkit.dvc.list_paths()
+    pipeline_output_storage = calkit.pipeline.get_output_storage_map()
+    # An ignore rule left on an output stored in Git, e.g., by an older
+    # Calkit, would hide its new files from everything below
+    for out in calkit.git.get_ignored_by_rule(
+        repo,
+        *[
+            out
+            for out, storage in pipeline_output_storage.items()
+            if storage == "git" and os.path.exists(out)
+        ],
+    ):
+        if dry_run:
+            typer.echo(f"Would un-ignore {out} since it's stored in Git")
+        else:
+            typer.echo(f"Un-ignoring {out} since it's stored in Git")
+            calkit.git.ensure_path_is_not_ignored(repo, out)
     untracked_git_files = repo.untracked_files
     if auto_commit_message:
         # See if this path is in the repo already
@@ -1118,12 +1151,31 @@ def add(
             ]:
                 paths.append(changed_file)
         zip_path_map = calkit.dvc.zip.get_zip_path_map()
-        pipeline_output_storage = calkit.pipeline.get_output_storage_map()
         lock_out_paths = calkit.dvc.get_lock_out_paths()
+        cached_out_paths: set[str] | None = None
         dvc_scm = None
         for path in paths:
             # Check if this path is already registered as a zip
             posix_path = Path(path).as_posix()
+            # A file inside an output stored in Git, e.g., one written to a
+            # persistent output directory, goes to Git like the output does
+            output_storage = pipeline_output_storage.get(posix_path)
+            if output_storage is None and any(
+                posix_path.startswith(out + "/")
+                for out, storage in pipeline_output_storage.items()
+                if storage == "git"
+            ):
+                output_storage = "git"
+            locked_out = next(
+                (
+                    out
+                    for out in lock_out_paths
+                    if posix_path == out or posix_path.startswith(out + "/")
+                ),
+                None,
+            )
+            if locked_out is not None and cached_out_paths is None:
+                cached_out_paths = calkit.dvc.get_cached_out_paths()
             if posix_path in zip_path_map:
                 if dry_run:
                     typer.echo(f"Would add {path} via DVC zip")
@@ -1155,10 +1207,9 @@ def add(
                     )
                     calkit.git.ensure_dvc_pointer_is_not_ignored(repo, path)
                     calkit.dvc.run_dvc_command(["add", path])
-            elif posix_path in pipeline_output_storage:
+            elif output_storage is not None:
                 # Respect storage explicitly set in the pipeline definition
-                pipeline_storage = pipeline_output_storage[posix_path]
-                if pipeline_storage == "git":
+                if output_storage == "git":
                     if dry_run:
                         typer.echo(
                             f"Would add {path} to Git "
@@ -1169,7 +1220,7 @@ def add(
                             f"Adding {path} to Git per pipeline output storage"
                         )
                         git_add(path)
-                elif pipeline_storage == "dvc-zip":
+                elif output_storage == "dvc-zip":
                     if dry_run:
                         typer.echo(
                             f"Would add {path} as a DVC zip "
@@ -1193,22 +1244,13 @@ def add(
                             f"({path} is a DVC pipeline output)"
                         )
                         git_add("dvc.lock")
-            elif (
-                locked_out := next(
-                    (
-                        out
-                        for out in lock_out_paths
-                        if posix_path == out
-                        or posix_path.startswith(out + "/")
-                    ),
-                    None,
-                )
-            ) is not None:
+            elif locked_out in (cached_out_paths or set()):
                 # DVC already caches this, and it's only showing up at all
                 # because its ignore entry went missing, e.g., DVC removes
                 # every entry a failed repro added, including those for the
                 # stages that succeeded. Sized up like a new file, a small
-                # one would go to Git alongside DVC's copy.
+                # one would go to Git alongside DVC's copy. Outputs DVC
+                # doesn't cache were never ignored, so they don't count.
                 if dry_run:
                     typer.echo(
                         f"Would ignore {path} ({path} is a DVC pipeline output)"