calkit-python 0.40.0__tar.gz → 0.40.1__tar.gz

{calkit_python-0.40.0 → calkit_python-0.40.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.40.0
+Version: 0.40.1
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.40.0 → calkit_python-0.40.1}/calkit/jupyterlab/routes.py

@@ -896,10 +896,16 @@ class PipelineStatusRouteHandler(APIHandler):
             calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
             dvc_repo = get_dvc_repo(os.getcwd())
             raw_status = dvc_repo.status()
+            # Frozen stages are intentionally pinned and hidden from status
+            frozen_stages = calkit.pipeline.frozen_stage_base_names(
+                ck_info=ck_info
+            )
             pipeline_status = {
                 k.split("dvc.yaml:")[-1]: v
                 for k, v in raw_status.items()
-                if v != ["always changed"] and not k.endswith(".dvc")
+                if v != ["always changed"]
+                and not k.endswith(".dvc")
+                and k.split("dvc.yaml:")[-1].split("@")[0] not in frozen_stages
             }
             is_outdated = len(pipeline_status) > 0
             self.finish(

{calkit_python-0.40.0 → calkit_python-0.40.1}/calkit/models/pipeline.py

@@ -189,6 +189,7 @@ class Stage(BaseModel):
     always_run: bool = False
     iterate_over: list[StageIteration] | None = None
     description: str | None = None
+    frozen: bool = False
     scheduler: StageSchedulerOptions | None = None
     # Do not allow extra keys
     model_config = ConfigDict(extra="forbid")
@@ -418,6 +419,8 @@ class Stage(BaseModel):
             stage["wdir"] = self.wdir
         if self.always_run:
             stage["always_changed"] = True
+        if self.frozen:
+            stage["frozen"] = True
         return stage
 
 
@@ -1182,6 +1185,7 @@ class Pipeline(BaseModel):
                 always_run=stage.always_run,
                 iterate_over=stage.iterate_over,
                 description=stage.description,
+                frozen=stage.frozen,
                 scheduler=StageSchedulerOptions(**sched_opts)
                 if sched_opts
                 else StageSchedulerOptions(),
@@ -1216,6 +1220,8 @@ class Pipeline(BaseModel):
                 calkit_yaml_stage["always_run"] = True
             if stage.description is not None:
                 calkit_yaml_stage["description"] = stage.description
+            if stage.frozen:
+                calkit_yaml_stage["frozen"] = True
             if stage.iterate_over is not None:
                 calkit_yaml_stage["iterate_over"] = [
                     it.model_dump() for it in stage.iterate_over

{calkit_python-0.40.0 → calkit_python-0.40.1}/calkit/pipeline.py

@@ -1,6 +1,7 @@
 """Pipeline-related functionality."""
 
 import itertools
+import logging
 import os
 from pathlib import Path
 
@@ -16,6 +17,54 @@ from calkit.models.pipeline import (
 )
 
 
+class _FrozenStageWarningFilter(logging.Filter):
+    """Drop only DVC's "stage is frozen" status warning.
+
+    Frozen stages are intentionally hidden from Calkit status output, so this
+    warning is noise; all other ``dvc.repo.status`` diagnostics pass through.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return "is frozen. Its dependencies are" not in record.getMessage()
+
+
+def _coerce_frozen(value: object) -> bool:
+    """Interpret a raw ``frozen`` value the same way the Stage model would.
+
+    ``ck_info`` holds unvalidated YAML, so a quoted ``frozen: "false"`` is a
+    truthy string here; defer to Pydantic's bool parsing so status filtering
+    matches the compiled pipeline's behavior.
+    """
+    from pydantic import TypeAdapter, ValidationError
+
+    try:
+        return TypeAdapter(bool).validate_python(value)
+    except ValidationError:
+        return bool(value)
+
+
+def frozen_stage_base_names(
+    ck_info: dict | None = None, wdir: str | None = None
+) -> set[str]:
+    """Return the base names of root pipeline stages marked ``frozen``.
+
+    Shared by the various status paths so frozen stages are consistently
+    hidden. Iterated stages use a ``name@param`` key in DVC status, so callers
+    should compare against the part before ``@``.
+    """
+    if ck_info is None:
+        ck_info = calkit.load_calkit_info(wdir=wdir)
+    names: set[str] = set()
+    stages = ck_info.get("pipeline", {}).get("stages", {})
+    if isinstance(stages, dict):
+        for name, cfg in stages.items():
+            if isinstance(cfg, dict) and _coerce_frozen(
+                cfg.get("frozen", False)
+            ):
+                names.add(name)
+    return names
+
+
 class PipelineStatus(BaseModel):
     """Current status of the project pipeline."""
 
@@ -537,7 +586,19 @@ def get_status(
                 return PipelineStatus.model_validate(result)
         try:
             dvc_repo = calkit.dvc.get_dvc_repo()
-            raw_status = dvc_repo.status(targets=targets)
+            # Frozen stages emit a "stage is frozen. Its dependencies are not
+            # going to be shown in the status output." warning from DVC's
+            # status module; suppress just that message since we intentionally
+            # hide frozen stages from the Calkit status output entirely.
+            # A targeted filter is used (rather than raising the logger level)
+            # so unrelated DVC diagnostics are still emitted.
+            dvc_status_logger = logging.getLogger("dvc.repo.status")
+            frozen_filter = _FrozenStageWarningFilter()
+            dvc_status_logger.addFilter(frozen_filter)
+            try:
+                raw_status = dvc_repo.status(targets=targets)
+            finally:
+                dvc_status_logger.removeFilter(frozen_filter)
         except Exception as e:
             result["errors"].append(
                 "Failed to get pipeline status from DVC: "
@@ -666,6 +727,21 @@ def get_status(
             raw_stale_stages.keys(), key=_stage_sort_key
         ):
             bare_name, subproject, status_data = raw_stale_stages[display_name]
+            # Frozen stages are intentionally pinned: never report them as
+            # stale (and they are never reproduced). Iterated stages have a
+            # "name@param" bare name, so look up config by the base name too.
+            base_name = bare_name.split("@")[0]
+            if subproject is None:
+                frozen_cfg = root_stages_config.get(
+                    bare_name, root_stages_config.get(base_name, {})
+                )
+            else:
+                sp_cfg = sp_stages_config.get(subproject, {})
+                frozen_cfg = sp_cfg.get(bare_name, sp_cfg.get(base_name, {}))
+            if isinstance(frozen_cfg, dict) and _coerce_frozen(
+                frozen_cfg.get("frozen", False)
+            ):
+                continue
             if subproject is None:
                 stage_cfg = root_stages_config.get(bare_name, {})
             else:

{calkit_python-0.40.0 → calkit_python-0.40.1}/calkit/server.py

@@ -306,10 +306,16 @@ def get_status(
         # Remove any always changed entries so the pipeline doesn't look
         # out of date
         logger.info(f"Raw DVC pipeline status: {dvc_pipeline_status}")
+        # Frozen stages are intentionally pinned and hidden from status
+        frozen_stages = calkit.pipeline.frozen_stage_base_names(
+            wdir=project.wdir
+        )
         dvc_pipeline_status = {
             k.split("dvc.yaml:")[-1]: v
             for k, v in dvc_pipeline_status.items()
-            if v != ["always changed"] and not k.endswith(".dvc")
+            if v != ["always changed"]
+            and not k.endswith(".dvc")
+            and k.split("dvc.yaml:")[-1].split("@")[0] not in frozen_stages
         }
         logger.info(
             f"DVC pipeline status after filtering: {dvc_pipeline_status}"

{calkit_python-0.40.0 → calkit_python-0.40.1}/calkit/tests/test_pipeline.py

@@ -1209,6 +1209,34 @@ def test_slurm_field_renamed_to_scheduler(tmp_dir):
     assert stage["scheduler"]["options"] == ["--time=01:00:00"]
 
 
+def test_frozen_preserved_through_sbatch_conversion(tmp_dir):
+    """A frozen legacy ``sbatch`` stage stays frozen after conversion."""
+    subprocess.check_call(["calkit", "init"])
+    ck_yaml = {
+        "environments": {"mycluster": {"kind": "slurm"}},
+        "pipeline": {
+            "stages": {
+                "run": {
+                    "kind": "sbatch",
+                    "script_path": "scripts/run.sh",
+                    "environment": "mycluster",
+                    "frozen": True,
+                }
+            }
+        },
+    }
+    with open("calkit.yaml", "w") as _f:
+        calkit.ryaml.dump(ck_yaml, _f)
+    dvc_stages = calkit.pipeline.to_dvc(write=True)
+    # The converted shell-script stage and the rewritten calkit.yaml entry
+    # must both still be frozen, and DVC must not reproduce it
+    assert dvc_stages["run"]["frozen"] is True
+    updated = calkit.load_calkit_info()
+    converted = updated["pipeline"]["stages"]["run"]
+    assert converted["kind"] == "shell-script"
+    assert converted["frozen"] is True
+
+
 def test_gitignore_updated_when_stage_output_renamed(tmp_dir):
     """When a stage output path is renamed, stale .gitignore entries are
     replaced.
@@ -1383,6 +1411,59 @@ def test_get_status(tmp_dir):
     assert status.stale_stages["get-data"].modified_outputs == []
 
 
+def test_get_status_excludes_frozen_stage(tmp_dir):
+    subprocess.check_call(["calkit", "init"])
+    ck_info = {
+        "environments": {
+            "py": {
+                "kind": "uv-venv",
+                "path": "requirements.txt",
+                "prefix": ".venv",
+            }
+        },
+        "pipeline": {
+            "stages": {
+                "get-data": {
+                    "kind": "python-script",
+                    "environment": "py",
+                    "script_path": "something/my-cool-script.py",
+                    "outputs": ["my-output.out"],
+                    "frozen": True,
+                }
+            }
+        },
+    }
+    with open("calkit.yaml", "w") as f:
+        calkit.ryaml.dump(ck_info, f)
+    with open("requirements.txt", "w") as f:
+        f.write("requests\n")
+    os.makedirs("something", exist_ok=True)
+    with open("something/my-cool-script.py", "w") as f:
+        f.write(
+            "with open('my-output.out', 'w') as f:\n"
+            "    f.write('Hello, world!')\n"
+        )
+    # The compiled DVC stage must carry frozen: true so DVC itself never
+    # reproduces it (this is what actually prevents it from running)
+    dvc_stages = calkit.pipeline.to_dvc(ck_info=ck_info)
+    assert dvc_stages["get-data"]["frozen"] is True
+    # A frozen stage is never reported as stale, even though it has never
+    # been run and its outputs do not exist yet
+    status = calkit.pipeline.get_status(ck_info=ck_info)
+    assert not status.failed_environment_checks
+    assert not status.is_stale
+    assert "get-data" not in status.stale_stage_names
+    # Modifying the script must not make the frozen stage stale either
+    with open("something/my-cool-script.py", "w") as f:
+        f.write(
+            "with open('my-output.out', 'w') as f:\n"
+            "    f.write('Changed!')\n"
+        )
+    status = calkit.pipeline.get_status(ck_info=ck_info)
+    assert not status.is_stale
+    assert "get-data" not in status.stale_stage_names
+
+
 def test_stale_stage_detects_changed_command():
     stale_stage = calkit.pipeline.StaleStage.from_status_data(
         status_data=[{"changed command": "python src/new-script.py"}],

{calkit_python-0.40.0 → calkit_python-0.40.1}/docs/pipeline/index.md

@@ -286,6 +286,7 @@ Common stage parameters:
 | `always_run`   | bool                                | no       | False   |
 | `iterate_over` | list[StageIteration] \| None        | no       | null    |
 | `description`  | str \| None                         | no       | null    |
+| `frozen`       | bool                                | no       | False   |
 | `scheduler`    | StageSchedulerOptions \| None       | no       | null    |
 
 ### `command`