calkit-python 0.48.1__py3-none-any.whl → 0.48.3__py3-none-any.whl

calkit/calc.py

@@ -4,14 +4,20 @@ from __future__ import annotations
 
 from typing import Annotated, Literal
 
-from pydantic import BaseModel, Discriminator, model_validator
+from pydantic import BaseModel, ConfigDict, Discriminator, model_validator
 
 DTYPES = {"int": int, "float": float, "str": str}
 DEFAULT_IN_TYPE = "float"
 DEFAULT_OUT_TYPE = "float"
 
 
-class Input(BaseModel):
+class _Model(BaseModel):
+    # As calkit.models.base.Model, which importing here would import
+    # calkit.models, which imports this
+    model_config = ConfigDict(defer_build=True)
+
+
+class Input(_Model):
     name: str
     description: str | None = None
     dtype: Literal["int", "float", "str"] = DEFAULT_IN_TYPE
@@ -19,14 +25,14 @@ class Input(BaseModel):
     max: int | float | None = None
 
 
-class Output(BaseModel):
+class Output(_Model):
     name: str
     description: str | None = None
     dtype: Literal["int", "float", "str"] = DEFAULT_OUT_TYPE
     template: str | None = None
 
 
-class Calculation(BaseModel):
+class Calculation(_Model):
     kind: str
     params: dict = {}
     name: str | None = None
@@ -124,7 +130,7 @@ class Calculation(BaseModel):
         return template.format(**(inputs | {out_name: res}))
 
 
-class FormulaParams(BaseModel):
+class FormulaParams(_Model):
     formula: str
 
 
@@ -139,7 +145,7 @@ class Formula(Calculation):
         return arithmetic_eval.evaluate(self.params.formula, inputs)
 
 
-class LinearParams(BaseModel):
+class LinearParams(_Model):
     coeffs: dict[str, float]
     offset: float = 0.0
 
@@ -163,7 +169,7 @@ class Linear(Calculation):
         return val
 
 
-class LookupTableParams(BaseModel):
+class LookupTableParams(_Model):
     x_values: list[float]
     y_values: list[float]
     method: Literal["floor", "ceil", "round", "interpolate"] = "interpolate"
@@ -176,7 +182,7 @@ class LookupTable(Calculation):
     params: LookupTableParams
 
 
-class HttpRequestParams(BaseModel):
+class HttpRequestParams(_Model):
     url: str
     inputs_as_params: bool = True  # Otherwise, use body
     method: Literal["get", "post", "put"] = "get"
@@ -210,7 +216,7 @@ class HttpRequest(Calculation):
             return resp.text
 
 
-class XGBoostModelParams(BaseModel):
+class XGBoostModelParams(_Model):
     path: str
     type: Literal["classifier", "regressor"]
 

calkit/cli/check.py

@@ -2026,6 +2026,95 @@ def check_pipeline(
             )
 
 
+@check_app.command(name="storage")
+def check_storage(
+    fix: Annotated[
+        bool,
+        typer.Option(
+            "--fix", help="Move each path to where it's declared to be stored."
+        ),
+    ] = False,
+    as_json: Annotated[
+        bool, typer.Option("--json", help="Output the problems as JSON.")
+    ] = False,
+) -> None:
+    """Check that paths are tracked where they're declared to be stored.
+
+    That is, nothing is tracked by both Git and DVC, no pipeline output
+    stored in Git is ignored by it, and nothing in Git is too large for it.
+    With --fix, the pipeline's declarations, or a file's .dvc file, decide
+    where each path belongs, and the changes are staged.
+    """
+    import calkit.dvc
+    import calkit.git
+    import calkit.pipeline
+
+    if fix and as_json:
+        raise_error("--json can't be used with --fix")
+    problems = calkit.dvc.get_storage_problems()
+    if as_json:
+        typer.echo(json.dumps(problems, indent=2))
+    if not fix:
+        if not as_json:
+            for problem in problems:
+                warn(problem["message"])
+        if any(p["kind"] != "large" for p in problems):
+            if not as_json:
+                typer.echo("Run 'calkit check storage --fix' to fix these")
+            raise typer.Exit(1)
+        if not problems and not as_json:
+            calkit.echo("✅ Every path is stored where it's declared to be")
+        return
+    repo = calkit.git.get_repo()
+    gitignores = calkit.git.get_dirty_gitignores(repo)
+    fixed = []
+    unfixed = []
+    # Pipeline outputs are recorded in DVC, which records their stages'
+    # inputs too, so only for stages that are up to date
+    to_record = []
+    for problem in problems:
+        path = os.path.abspath(problem["path"])
+        if problem["store_in"] == "dvc":
+            repo.git.rm("-r", "-q", "--cached", "--", path)
+            calkit.git.ensure_path_is_ignored(repo, path)
+            if problem["kind"] == "both":
+                to_record.append(problem)
+        elif problem["store_in"] == "git":
+            calkit.git.ensure_path_is_not_ignored(repo, path)
+            repo.git.add("--", path)
+        else:
+            unfixed.append(problem)
+            continue
+        fixed.append(problem)
+        typer.echo(f"Fixed: {problem['message']}")
+    # None when it can't be told which are stale, so none are recorded
+    stale: set[str] | None = set()
+    if any(p["stage"] for p in to_record):
+        status = calkit.pipeline.get_status(
+            check_environments=False, clean_notebooks=False
+        )
+        if not status.errors:
+            stale = {n.split("@")[0] for n in status.stale_stage_names}
+        else:
+            stale = None
+    for problem in to_record:
+        stage = problem["stage"]
+        if stage and (stale is None or stage.split("@")[0] in stale):
+            typer.echo(f"Run '{stage}' to record {problem['path']} in DVC")
+            continue
+        target = problem["dvc_file"] or stage
+        if calkit.dvc.run_dvc_command(["commit", "-f", "-q", target]) != 0:
+            warn(f"Couldn't record {problem['path']} in DVC")
+    for message in calkit.git.stage_gitignores(repo, gitignores):
+        warn(message)
+    for problem in unfixed:
+        warn(problem["message"])
+    if fixed:
+        typer.echo("Commit the staged changes")
+    if any(p["kind"] != "large" for p in unfixed):
+        raise typer.Exit(1)
+
+
 @check_app.command(name="call")
 def check_call(
     cmd: Annotated[str, typer.Argument(help="Command to check.")],
@@ -2082,8 +2171,9 @@ def check_questions(
     pushed, or pinned to a Git ref that doesn't exist); broken references
     (a key that doesn't resolve, a placeholder that names no evidence, a
     label missing from the LaTeX); evidence the pipeline would rebuild;
-    and evidence from a frozen stage, or downstream of one, which nothing
-    will ever report out of date unless the citation pins a git_ref.
+    and evidence from a frozen stage, or downstream of one, that no longer
+    matches dvc.lock, which won't be rerun, unless the citation pins a
+    git_ref. Subprojects' questions are checked too.
 
     Evidence pinned with a git_ref is checked at that ref rather than in
     the working tree. Exits with an error if any answered question is

calkit/cli/describe.py

@@ -9,7 +9,6 @@ import typer
 
 import calkit
 from calkit.cli import AliasGroup, echo_json, raise_error
-from calkit.environments import get_env_lock_fpath
 
 describe_app = typer.Typer(cls=AliasGroup, no_args_is_help=True)
 
@@ -20,6 +19,10 @@ def _describe_env(env: dict, env_name: str) -> dict:
     The keys here are a stable contract for tools consuming ``--json``, so
     they are always present, even if their value is null.
     """
+    # Here rather than at the top, since every command imports this module;
+    # what that saves is a question in decisions/calkit.yaml
+    from calkit.environments import get_env_lock_fpath
+
     return {
         "kind": env.get("kind"),
         "spec_path": env.get("path"),