calkit-python 0.43.0__tar.gz → 0.44.0__tar.gz

{calkit_python-0.43.0 → calkit_python-0.44.0}/.gitmodules

@@ -1,3 +1,6 @@
 [submodule "examples/basic"]
 	path = examples/basic
 	url = https://github.com/calkit/example-basic.git
+[submodule "examples/ssh"]
+	path = examples/ssh
+	url = git@github.com:calkit/example-ssh.git

{calkit_python-0.43.0 → calkit_python-0.44.0}/Makefile

@@ -44,13 +44,23 @@ test-cov: ## Test the code coverage with pytest.
 test-docs: sync-docs ## Test if documentation can be built without warnings or errors.
 	@uv run mkdocs build -s
 
+.PHONY: schema
+schema: ## Generate the published JSON schema for calkit.yaml.
+	@echo "🚀 Generating calkit.yaml JSON schema"
+	@uv run calkit describe schema -o docs/schemas/calkit.json
+	@cp docs/schemas/calkit.json vscode-ext/schemas/calkit.json
+
 .PHONY: sync-docs
-sync-docs: ## Sync documentation content from docs/*.md into README.md.
+sync-docs: schema ## Sync documentation content from docs/*.md into README.md.
 	@echo "🚀 Generating docs references"
 	@uv run python scripts/generate-docs-references.py
 	@echo "🚀 Syncing documentation"
 	@uv run python scripts/sync-docs.py
 
+.PHONY: changes
+changes: ## List each product's commits since its last release.
+	@uv run python scripts/list-changes.py
+
 .PHONY: sync-resources
 sync-resources: ## Regenerate the dev container spec from the VS Code config.
 	@echo "🚀 Syncing project resources"

{calkit_python-0.43.0 → calkit_python-0.44.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: calkit-python
-Version: 0.43.0
+Version: 0.44.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -96,7 +96,7 @@ without the cognitive overhead.
 
 <!-- https://docs.google.com/drawings/d/1XMGnbgYYNFAVUBDyUaCyLfRB7efvJdrnrKmFlNmT19o/edit -->
 
-![pipeline](docs/img/pipeline.png)
+![pipeline](https://docs.calkit.org/img/pipeline.png)
 
 ## Features
 
@@ -261,7 +261,7 @@ This re-execs the CLI via `uvx --from calkit-python@<version> calkit`,
 so it requires [uv](https://docs.astral.sh/uv/) on `PATH`.
 You can also declare a minimum version in `calkit.yaml`;
 see
-[Pinning the Calkit CLI version](https://docs.calkit.org/dependencies.md#pinning-the-calkit-cli-version).
+[Pinning the Calkit CLI version](https://docs.calkit.org/requirements.md#pinning-the-calkit-cli-version).
 
 ### Calkit Assistant
 

{calkit_python-0.43.0 → calkit_python-0.44.0}/README.md

@@ -37,7 +37,7 @@ without the cognitive overhead.
 
 <!-- https://docs.google.com/drawings/d/1XMGnbgYYNFAVUBDyUaCyLfRB7efvJdrnrKmFlNmT19o/edit -->
 
-![pipeline](docs/img/pipeline.png)
+![pipeline](https://docs.calkit.org/img/pipeline.png)
 
 ## Features
 
@@ -202,7 +202,7 @@ This re-execs the CLI via `uvx --from calkit-python@<version> calkit`,
 so it requires [uv](https://docs.astral.sh/uv/) on `PATH`.
 You can also declare a minimum version in `calkit.yaml`;
 see
-[Pinning the Calkit CLI version](https://docs.calkit.org/dependencies.md#pinning-the-calkit-cli-version).
+[Pinning the Calkit CLI version](https://docs.calkit.org/requirements.md#pinning-the-calkit-cli-version).
 
 ### Calkit Assistant
 

{calkit_python-0.43.0 → calkit_python-0.44.0}/agent-plugin/skills/conventions/SKILL.md

@@ -18,6 +18,10 @@ is `calkit.yaml`, the project's metadata database.
 
 `calkit.yaml` lives at the repo root and contains:
 
+- `requirements`—what must be true of the machine before anything runs
+  (apps on `PATH`, environmental variables, setup steps, machine
+  properties like CPU count); formerly called `dependencies`, which
+  still works
 - `environments`—computational environments (Python venvs, Conda, Docker,
   R, Julia, MATLAB, etc.)
 - `pipeline.stages`—the reproducible pipeline

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/__init__.py

@@ -39,6 +39,7 @@ if TYPE_CHECKING:
         pipeline,
         releases,
         resources,
+        schema,
         server,
         templates,
     )
@@ -78,6 +79,7 @@ _SUBMODULES = {
     "install",
     "magics",
     "ops",
+    "schema",
     "server",
 }
 

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/calc.py

@@ -2,11 +2,9 @@
 
 from __future__ import annotations
 
-from typing import Literal
+from typing import Annotated, Literal
 
-import arithmetic_eval
-import requests
-from pydantic import BaseModel, model_validator
+from pydantic import BaseModel, Discriminator, model_validator
 
 DTYPES = {"int": int, "float": float, "str": str}
 DEFAULT_IN_TYPE = "float"
@@ -131,10 +129,12 @@ class FormulaParams(BaseModel):
 
 
 class Formula(Calculation):
-    kind: str = "formula"
+    kind: Literal["formula"] = "formula"
     params: FormulaParams
 
     def calculate(self, **inputs):
+        import arithmetic_eval
+
         inputs = self.check_inputs(**inputs)
         return arithmetic_eval.evaluate(self.params.formula, inputs)
 
@@ -147,7 +147,7 @@ class LinearParams(BaseModel):
 class Linear(Calculation):
     """Calculation for a simple linear relationship."""
 
-    kind: str = "linear"
+    kind: Literal["linear"] = "linear"
     params: LinearParams
 
     @model_validator(mode="after")
@@ -172,7 +172,7 @@ class LookupTableParams(BaseModel):
 class LookupTable(Calculation):
     """A 1-D lookup table."""
 
-    kind: str = "lookup-table"
+    kind: Literal["lookup-table"] = "lookup-table"
     params: LookupTableParams
 
 
@@ -191,15 +191,17 @@ class HttpRequest(Calculation):
     sensitive data.
     """
 
-    kind: str = "http"
+    kind: Literal["http"] = "http"
     params: HttpRequestParams
 
     def calculate(self, **inputs):
+        import requests
+
         func = getattr(requests, self.params.method)
         if self.params.inputs_as_params:
             kws = {"params": inputs}
         else:
-            kws = {"json", inputs}
+            kws = {"json": inputs}
         resp: requests.Response = func(url=self.params.url, **kws)
         resp.raise_for_status()
         if self.params.as_json:
@@ -222,7 +224,7 @@ class XGBoostModel(Calculation):
     ``predict`` method.
     """
 
-    kind: str = "xgboost"
+    kind: Literal["xgboost"] = "xgboost"
     params: XGBoostModelParams
 
     def calculate(self, **inputs):
@@ -239,6 +241,13 @@ class XGBoostModel(Calculation):
         return model.predict(**inputs)
 
 
+# The calculation kinds that ``parse`` can run, as a discriminated union so
+# ``calkit.yaml`` gets per-kind validation and autocompletion of ``params``
+CalculationType = Annotated[
+    Formula | Linear | LookupTable, Discriminator("kind")
+]
+
+
 def parse(data: dict) -> Calculation:
     if isinstance(data, BaseModel):
         data = data.model_dump()

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/check.py

@@ -206,7 +206,7 @@ def check_reproducibility(
     else:
         res["has_readme"] = False
         res["instructions_in_readme"] = False
-    ck_info = calkit.load_calkit_info(wdir=wdir, process_includes=False)
+    ck_info = calkit.load_calkit_info(wdir=wdir)
     pipeline = calkit.dvc.read_pipeline(wdir=wdir)
     # Check for non-imported artifacts not produced by the pipeline
     for artifact_type in ["datasets", "figures", "publications"]:

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/cli/check.py

@@ -324,10 +324,11 @@ def check_environment(
         get_default_venv_prefix,
         get_env_lock_fpath,
         write_scheduler_env_lock,
+        write_system_env_lock,
     )
 
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
-    ck_info = calkit.load_calkit_info(process_includes="environments")
+    ck_info = calkit.load_calkit_info()
     envs = ck_info.get("environments", {})
     if not envs:
         raise_error("No environments defined in calkit.yaml")
@@ -434,12 +435,6 @@ def check_environment(
             alt_lock_fpaths=alt_lock_fpaths,
             verbose=verbose,
         )
-    elif env["kind"] == "ssh":
-        # TODO: How to check SSH environments?
-        # Maybe just check that we can connect
-        raise_error(
-            "Environment checking not implemented for SSH environments"
-        )
     elif env["kind"] == "renv":
         env_path = env.get("path")
         if env_path is None:
@@ -473,6 +468,104 @@ def check_environment(
         # env config so DVC stages that depend on the env get invalidated
         # when the config changes.
         write_scheduler_env_lock(env_name=env_name, env=env)
+    elif env["kind"] == "system":
+        # Nothing is installed or built for a system env; checking it means
+        # making sure the machine is as the project requires, then reading
+        # the properties it declared it depends on and recording them, so
+        # stages depending on the env see them change.
+        if calkit.environments.env_is_local(env):
+            try:
+                calkit.check_requirements(
+                    requirements=env.get("requirements", [])
+                )
+                write_system_env_lock(env_name=env_name, env=env)
+            except ValueError as e:
+                # A requirement that isn't met, or a property that can't be
+                # locked -- a misspelled one, or a tool that isn't
+                # installed. Both are the user's to fix. The remote branch
+                # below already reports these; this one used to let them
+                # out as a traceback.
+                raise_error(f"Environment '{env_name}': {e}")
+        else:
+            # For a host that isn't this one, checking means getting to the
+            # point where we can actually reach it -- better sorted out here
+            # than partway through a pipeline -- and reading the properties
+            # from that machine, since those are the ones the results
+            # depend on.
+            import calkit.workspace as workspace
+            from calkit.dependencies import _is_interactive
+
+            interactive = _is_interactive()
+            ck_info = calkit.load_calkit_info()
+            try:
+                # Anything written as ${CK_SSH_HOST} is the first thing to
+                # be missing for anyone but the project's author, since a
+                # project shares its calkit.yaml but not its .env
+                env = dict(env)
+                for key in workspace.CONNECTION_FIELDS:
+                    if isinstance(env.get(key), str):
+                        env[key] = workspace.expand_with_prompts(
+                            env[key],
+                            interactive=interactive,
+                            described_as=(
+                                f"environment '{env_name}' "
+                                f"{key.replace('_', ' ')}"
+                            ),
+                        )
+                ws = workspace.Workspace.from_env(
+                    env=env,
+                    env_name=env_name,
+                    ck_info=ck_info,
+                )
+                # Reassigned: a key created during setup belongs to the
+                # workspace we go on to use
+                ws = workspace.ensure_reachable(
+                    ws, interactive=interactive, verbose=verbose
+                )
+                # Calkit is needed there to read the machine's properties,
+                # and to activate an inner env, but an environment that
+                # only dispatches a command never calls it. A declared
+                # machine ID needs it for the same reason a lock does: the
+                # far end is what reports its own ID, so it has to be asked.
+                # Requirements only need it when one of them asks about the
+                # machine itself rather than about what's installed on it.
+                locks = bool(env.get("lock"))
+                requirements = env.get("requirements", []) or []
+                needs_system_info = (
+                    locks
+                    or bool(ws.machine_id)
+                    or workspace.requirements_need_system_info(requirements)
+                )
+                workspace.ensure_calkit_installed(
+                    ws,
+                    interactive=interactive,
+                    required=needs_system_info,
+                    verbose=verbose,
+                )
+                system_info = None
+                if needs_system_info:
+                    system_info = workspace.remote_system_info(ws)
+                    # Before anything else is decided from it, so a
+                    # mismatched machine is reported rather than measured
+                    # and recorded as what results depend on
+                    workspace.verify_machine_id(ws, system_info)
+                # Before the lock: a machine that doesn't meet the project's
+                # requirements shouldn't have its properties written down as
+                # though stages had run there.
+                workspace.check_requirements(
+                    ws,
+                    requirements,
+                    system_info=system_info,
+                    verbose=verbose,
+                )
+                if locks:
+                    write_system_env_lock(
+                        env_name=env_name,
+                        env=env,
+                        system_info=system_info,
+                    )
+            except ValueError as e:
+                raise_error(f"Environment '{env_name}': {e}")
     elif env["kind"] == "nix":
         check_nix_env(env=env, verbose=verbose)
     else:
@@ -520,7 +613,7 @@ def check_environments(
         bool, typer.Option("--verbose", help="Print verbose output.")
     ] = False,
 ) -> None:
-    ck_info = calkit.load_calkit_info(process_includes="environments")
+    ck_info = calkit.load_calkit_info()
     envs = ck_info.get("environments", {})
     if not envs:
         typer.echo("No environments defined in calkit.yaml")
@@ -540,6 +633,12 @@ def check_environments(
         typer.echo(f"Checking environment: '{env_name}'")
         try:
             check_environment(env_name=env_name, verbose=verbose)
+        except typer.Exit:
+            # The check reported its own reason through raise_error, which
+            # has already printed it. Repeating it here would replace a
+            # real message ("Unknown system property to lock: ...") with
+            # the exit code, since that is all this exception carries.
+            failures.append(env_name)
         except Exception as e:
             warn(f"Error checking environment '{env_name}': {e}")
             failures.append(env_name)
@@ -1297,8 +1396,16 @@ def check_matlab_env(
     )
 
 
-@check_app.command(name="deps|dependencies")
-def check_dependencies(
+@check_app.command(
+    name="deps|dependencies",
+    hidden=True,
+    help=(
+        "Check that a project's system-level requirements are met "
+        "(alias for 'reqs')."
+    ),
+)
+@check_app.command(name="reqs|requirements")
+def check_project_requirements(
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output")
     ] = False,
@@ -1307,21 +1414,19 @@ def check_dependencies(
         typer.Option(
             "--no-cache",
             help=(
-                "Re-probe every setup dependency, ignoring (and clearing) "
+                "Re-probe every setup requirement, ignoring (and clearing) "
                 "the cache at .calkit/local/dep-checks.sqlite."
             ),
         ),
     ] = False,
 ) -> None:
-    """Check that a project's system-level dependencies are set up
-    correctly.
-    """
-    typer.echo("Checking project dependencies")
+    """Check that a project's system-level requirements are met."""
+    typer.echo("Checking project requirements")
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     if no_cache:
         calkit.dependencies.cache_clear()
     try:
-        calkit.check_system_deps(use_cache=not no_cache)
+        calkit.check_requirements(use_cache=not no_cache)
     except Exception as e:
         raise_error(str(e))
     message = "✅ All set!"
@@ -1338,7 +1443,7 @@ def check_env_vars(
     typer.echo("Checking project environmental variables")
     dotenv.load_dotenv(dotenv_path=".env")
     ck_info = calkit.load_calkit_info()
-    deps = ck_info.get("dependencies", [])
+    deps = calkit.get_requirements(ck_info)
     env_var_dep_names = calkit.get_env_var_dep_names(ck_info)
     for name in env_var_dep_names:
         if verbose:

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/cli/describe.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import os
 from typing import Annotated
 
 import typer
@@ -95,3 +96,29 @@ def describe_envs(
     for env_name, desc in result.items():
         typer.echo(env_name + ":")
         _echo_description(desc, indent="    ")
+
+
+@describe_app.command(name="schema")
+def describe_schema(
+    output: Annotated[
+        str | None,
+        typer.Option(
+            "--output",
+            "-o",
+            help="Path at which to write the schema instead of printing it.",
+        ),
+    ] = None,
+) -> None:
+    """Print the JSON schema for calkit.yaml.
+
+    Editors can use this to validate and autocomplete the file. See
+    https://docs.calkit.org/calkit-yaml for how to set that up.
+    """
+    txt = calkit.schema.generate_json()
+    if output is None:
+        typer.echo(txt, nl=False)
+        return
+    os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
+    with open(output, "w", encoding="utf-8") as f:
+        f.write(txt)
+    typer.echo(f"Wrote schema to {output}")

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/cli/import_.py

@@ -271,9 +271,7 @@ def import_environment(
         raise_error("Invalid source environment specification")
     if os.path.isdir(project):
         typer.echo(f"Importing from local project directory: {project}")
-        src_ck_info = dict(
-            calkit.load_calkit_info(wdir=project, process_includes=True)
-        )
+        src_ck_info = dict(calkit.load_calkit_info(wdir=project))
         environments = src_ck_info.get("environments", {})
         if env_name not in environments:
             raise_error(f"Environment {env_name} not found in project")

{calkit_python-0.43.0 → calkit_python-0.44.0}/calkit/cli/list.py

@@ -71,8 +71,8 @@ def _list_artifacts(
     declared and auto-detected artifacts apart.
     """
     ck_info = calkit.load_calkit_info()
-    declared = ck_info.get(kind, []) or []
-    declared_paths = {o.get("path") for o in declared if isinstance(o, dict)}
+    declared = [o for o in ck_info.get(kind) or [] if isinstance(o, dict)]
+    declared_paths = {o.get("path") for o in declared}
     detected: list[dict] = []
     if not declared_only:
         found = calkit.detect.detect_project_artifacts(ck_info=ck_info)