mkbundle 0.2.0__py3-none-any.whl

mkbundle/rules.py

@@ -0,0 +1,417 @@
+"""The build-time cross-file rules.
+
+The schemas cannot express these: each needs context beyond one document
+(`schema/README.md`). The rules that need *tenant* state — budgets against a
+tenant's constraints, connections against a target environment — are deploy-time
+checks and deliberately absent here. A builder has no tenant.
+
+The one that carries the most weight is `missing-idempotency-key`. Durable
+execution is at-least-once for external effects (ADR 0009 Amendment 1), so the
+idempotency key is what makes an effect happen once. A write step without one is
+not a style problem, it is a duplicate email waiting for the first crash.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+from mkp_contracts import expressions, gates
+
+from .catalog import Connector
+from .errors import Report
+
+SECRET_REF_RE = re.compile(r"\{\{\s*secrets\.([a-zA-Z_][a-zA-Z0-9_]*)")
+
+
+def walk_steps(workflow: dict[str, Any]) -> list[tuple[dict[str, Any], str]]:
+    """Every step in a workflow, including those nested in branches and foreach.
+
+    Returns each step with a location, so a failure points at the step and not
+    merely at the file.
+    """
+    found: list[tuple[dict[str, Any], str]] = []
+
+    def visit(steps: Any, prefix: str) -> None:
+        if not isinstance(steps, list):
+            return
+        for index, step in enumerate(steps):
+            if not isinstance(step, dict):
+                continue
+            location = f"{prefix}[{index}]"
+            found.append((step, location))
+            visit(step.get("steps"), f"{location}.steps")
+            for case_index, case in enumerate(step.get("cases") or []):
+                if isinstance(case, dict):
+                    visit(case.get("steps"), f"{location}.cases[{case_index}].steps")
+            visit(step.get("default"), f"{location}.default")
+
+    visit(workflow.get("steps"), "steps")
+    visit(workflow.get("on_failure"), "on_failure")
+    return found
+
+
+def secret_refs(document: Any) -> set[str]:
+    """secretRefs a workflow uses: `verify` on a webhook trigger, and expressions."""
+    found: set[str] = set()
+
+    def visit(node: Any, key: str | None) -> None:
+        if isinstance(node, dict):
+            for child_key, value in node.items():
+                visit(value, child_key)
+        elif isinstance(node, list):
+            for value in node:
+                visit(value, key)
+        elif isinstance(node, str):
+            if key == "verify":
+                found.add(node)
+            found.update(SECRET_REF_RE.findall(node))
+
+    visit(document, None)
+    return found
+
+
+def connections(workflow: dict[str, Any]) -> set[str]:
+    used = {
+        trigger["connection"]
+        for trigger in workflow.get("triggers") or []
+        if isinstance(trigger, dict) and trigger.get("connection")
+    }
+    used.update(step["connection"] for step, _ in walk_steps(workflow) if step.get("connection"))
+    return used
+
+
+def check_workflow(
+    workflow: dict[str, Any],
+    path: str,
+    *,
+    machines: set[str],
+    declared_connections: set[str],
+    declared_secrets: set[str],
+    connector_pins: dict[str, int],
+    catalog: dict[str, Connector] | None,
+) -> Report:
+    report = Report()
+    steps = walk_steps(workflow)
+
+    _step_ids_unique(steps, path, report)
+    _machines_present(steps, path, machines, report)
+    _declared_names(workflow, path, declared_connections, declared_secrets, report)
+    _connector_actions(steps, path, connector_pins, catalog, report)
+    _waits_are_decidable(steps, path, report)
+    _expressions_are_evaluable(workflow, path, report)
+    _schedule_can_satisfy_inputs(workflow, path, report)
+
+    return report
+
+
+def _schedule_can_satisfy_inputs(workflow: dict[str, Any], path: str, report: Report) -> None:
+    """A cron-triggered workflow can actually be started by its cron.
+
+    The scheduler starts a run with `{"scheduled_for": …}` and nothing else — a schedule
+    fires on a standing instruction, so there is no caller to supply anything. A required
+    input with no default is therefore unsatisfiable on every tick: the run fails at the
+    start (once the runner binds inputs) or, before that, walked the document over
+    `None`s. Either way the schedule is decoration.
+
+    Declaring a `default` is the fix, and it is per-bundle, which is per environment —
+    which is where a value like the mailbox to poll belongs.
+    """
+    triggers = workflow.get("triggers") or []
+    if not any(isinstance(t, dict) and t.get("kind") == "schedule" for t in triggers):
+        return
+
+    declared = workflow.get("inputs") or {}
+    if not isinstance(declared, dict):
+        return
+
+    unsatisfiable = sorted(
+        name
+        for name, spec in declared.items()
+        if isinstance(spec, dict) and spec.get("required") and "default" not in spec
+    )
+    if unsatisfiable:
+        report.fail(
+            "schedule-cannot-satisfy-inputs",
+            f"workflow has a `schedule` trigger but requires input(s) "
+            f"{', '.join(unsatisfiable)} with no default. A schedule supplies only "
+            "`scheduled_for`, so every tick would start a run that cannot proceed. "
+            "Give them defaults, or drop the schedule trigger and start the workflow "
+            "with its inputs",
+            path=path,
+        )
+
+
+def _expressions_are_evaluable(workflow: dict[str, Any], path: str, report: Report) -> None:
+    """Every `{{ … }}` is one the runner can evaluate (`mkp_contracts.expressions`).
+
+    A bundle is validated before it can be pointed at (ADR 0008), and until this rule
+    existed that promise stopped at the document's shape: an expression the grammar does
+    not have was schema-valid, signed, deployed, and then either failed the first run
+    that reached the step or — worse, for a half-quoted comparison — compared false
+    forever without failing anything.
+    """
+    for location, text in _template_strings(workflow, "").items():
+        for expression in expressions.expressions_in(text):
+            refusal = expressions.refusal(expression)
+            if refusal:
+                report.fail("unevaluable-expression", refusal, path=path, location=location)
+
+
+def _template_strings(node: Any, location: str) -> dict[str, str]:
+    """Every string in the document that contains a `{{ … }}`, by location."""
+    found: dict[str, str] = {}
+    if isinstance(node, dict):
+        for key, value in node.items():
+            found.update(_template_strings(value, f"{location}.{key}" if location else str(key)))
+    elif isinstance(node, list):
+        for index, value in enumerate(node):
+            found.update(_template_strings(value, f"{location}[{index}]"))
+    elif isinstance(node, str) and expressions.TEMPLATE.search(node):
+        found[location] = node
+    return found
+
+
+def _waits_are_decidable(
+    steps: list[tuple[dict[str, Any], str]], path: str, report: Report
+) -> None:
+    """A `wait` says either how long or what for, and never both or neither.
+
+    The schema declares `duration`, `until_event` and `deadline` as three independent
+    optional fields, which makes all four combinations schema-valid — including the two a
+    runner cannot act on. `duration` sleeps and `until_event` waits to be told; a step
+    carrying both is a document whose author meant one of them, and choosing for them would
+    be a runner deciding what a workflow does.
+
+    The interpreter refuses the same two shapes, because a bundle built before this rule
+    existed is still loadable. This is the half that means nobody ships one.
+    """
+    for step, location in steps:
+        if step.get("kind") != "wait":
+            continue
+        duration, event = step.get("duration"), step.get("until_event")
+        if duration and event:
+            report.fail(
+                "wait-declares-both",
+                f"step {step.get('id')!r} is a wait with both `duration` and "
+                "`until_event`. One sleeps for a fixed time and the other holds until "
+                "something arrives — declare the one that was meant",
+                path=path,
+                location=location,
+            )
+        elif not duration and not event:
+            report.fail(
+                "wait-declares-nothing",
+                f"step {step.get('id')!r} is a wait with neither `duration` nor "
+                "`until_event`, so nothing says what it waits for or for how long. A run "
+                "reaching it would stop until its deadline and then fail",
+                path=path,
+                location=location,
+            )
+
+
+def _step_ids_unique(steps: list[tuple[dict[str, Any], str]], path: str, report: Report) -> None:
+    seen: dict[str, str] = {}
+    for step, location in steps:
+        step_id = step.get("id")
+        if not step_id:
+            continue
+        if step_id in seen:
+            report.fail(
+                "duplicate-step-id",
+                f"step id {step_id!r} is already used at {seen[step_id]}; "
+                "step ids are how results are referenced and how attempts are audited",
+                path=path,
+                location=location,
+            )
+        else:
+            seen[step_id] = location
+
+
+def _machines_present(
+    steps: list[tuple[dict[str, Any], str]], path: str, machines: set[str], report: Report
+) -> None:
+    for step, location in steps:
+        if step.get("kind") != "mklang":
+            continue
+        machine = step.get("machine")
+        if machine and machine not in machines:
+            report.fail(
+                "machine-missing",
+                f"machine {machine!r} is not in the bundle; a bundle pins everything that "
+                "affects behaviour (ADR 0008)",
+                path=path,
+                location=f"{location}.machine",
+            )
+
+
+def _declared_names(
+    workflow: dict[str, Any],
+    path: str,
+    declared_connections: set[str],
+    declared_secrets: set[str],
+    report: Report,
+) -> None:
+    for name in sorted(connections(workflow) - declared_connections):
+        report.fail(
+            "undeclared-connection",
+            f"connection {name!r} is used but not declared in requires.connections; "
+            "a missing connection must fail the deploy, not the first run",
+            path=path,
+        )
+    for name in sorted(secret_refs(workflow) - declared_secrets):
+        report.fail(
+            "undeclared-secret",
+            f"secretRef {name!r} is used but not declared in requires.secrets",
+            path=path,
+        )
+
+
+def _connector_actions(
+    steps: list[tuple[dict[str, Any], str]],
+    path: str,
+    connector_pins: dict[str, int],
+    catalog: dict[str, Connector] | None,
+    report: Report,
+) -> None:
+    for step, location in steps:
+        if step.get("kind") != "connector":
+            continue
+        connector_name = step.get("connector")
+        action_name = step.get("action")
+        if not connector_name or not action_name:
+            continue
+
+        if connector_name not in connector_pins:
+            report.fail(
+                "unknown-connector",
+                f"connector {connector_name!r} is not pinned in lock.connectors; "
+                "a newer manifest requires a new bundle",
+                path=path,
+                location=f"{location}.connector",
+            )
+            continue
+
+        if catalog is None:
+            continue
+
+        connector = catalog.get(connector_name)
+        if connector is None:
+            report.fail(
+                "unknown-connector",
+                f"connector {connector_name!r} is not in the catalog",
+                path=path,
+                location=f"{location}.connector",
+            )
+            continue
+
+        if connector.version != connector_pins[connector_name]:
+            report.fail(
+                "connector-version-mismatch",
+                f"lock pins {connector_name} v{connector_pins[connector_name]} but the "
+                f"catalog holds v{connector.version}",
+                path=path,
+                location=f"{location}.connector",
+            )
+
+        action = connector.action(action_name)
+        if action is None:
+            report.fail(
+                "unknown-action",
+                f"{connector_name} has no action {action_name!r}",
+                path=path,
+                location=f"{location}.action",
+            )
+            continue
+
+        if action.get("effect") == "write" and not step.get("idempotency_key"):
+            report.fail(
+                "missing-idempotency-key",
+                f"{connector_name}.{action_name} is effect: write and this step has no "
+                "idempotency_key; durable execution is at-least-once, so without a key a "
+                "crash or a retry duplicates the effect (ADR 0009 Amendment 1)",
+                path=path,
+                location=location,
+            )
+
+
+def check_machine_gates(text: str, path: str) -> Report:
+    """Every `hook:` a machine declares names a gate the platform can resolve (`C21`).
+
+    mklang resolves a hook by name and answers `None` for one it does not know — and a gate
+    it cannot resolve **halts the run when the gate is reached**. Not at load, not at the
+    first state: when that gate is reached, which may be one model call later and only for
+    some inputs. So a bundle that names one is a bundle that builds, deploys, and then stops
+    somewhere expensive.
+
+    What a gate may name is `mkp_contracts.gates` — the platform's declarative vocabulary,
+    plus the two mklang builtins and the two parametric forms a machine may keep using. It
+    deliberately cannot name a Python predicate: ADR 0003 runs machines in the runner process
+    *because* "there is no customer code in a `.mkl`, only data", and a bundle carrying code
+    for its gates would take that reason away.
+
+    Read as YAML rather than through mklang, because this is the builder: `mklang check` has
+    already run over the same file and has no opinion about which hooks a host provides.
+    """
+    report = Report()
+    try:
+        document = yaml.safe_load(text)
+    except yaml.YAMLError:
+        # `mklang check` owns "this file does not parse" and says it better. A second,
+        # vaguer copy of that failure would be two rules for one mistake.
+        return report
+    if not isinstance(document, dict):
+        return report
+
+    states = document.get("states")
+    if not isinstance(states, dict):
+        return report
+
+    for state_id, state in states.items():
+        if not isinstance(state, dict):
+            continue
+        for index, gate in enumerate(state.get("gates") or []):
+            if not isinstance(gate, dict):
+                continue
+            hook = gate.get("hook")
+            if not hook or gates.resolvable(hook):
+                continue
+            report.fail(
+                "machine-gate-unresolvable",
+                gates.reason(hook),
+                path=path,
+                location=f"states.{state_id}.gates[{index}].hook",
+            )
+    return report
+
+
+def check_spec_versions(specs: dict[str, str]) -> Report:
+    """Every machine in a bundle must target one language spec version.
+
+    Two versions in one bundle means `lock.spec` cannot honestly say what the
+    machines were validated against.
+    """
+    report = Report()
+    distinct = sorted(set(specs.values()))
+    if len(distinct) > 1:
+        detail = ", ".join(f"{path} -> {spec}" for path, spec in sorted(specs.items()))
+        report.fail(
+            "spec-mismatch",
+            f"machines declare more than one mklang spec version ({', '.join(distinct)}): {detail}",
+        )
+    return report
+
+
+def read_spec_version(text: str) -> str | None:
+    """The `mklang:` key a `.mkl` declares — the language spec it targets."""
+    for line in text.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("mklang:"):
+            return stripped.split(":", 1)[1].strip().strip("\"'")
+    return None
+
+
+def machine_paths(paths: list[Path]) -> set[str]:
+    return {p.as_posix() for p in paths}

mkbundle/signing.py

@@ -0,0 +1,87 @@
+"""Signing a bundle with minisign. Verification lives in `mkp_contracts`.
+
+A runner refuses an unsigned or unverifiable bundle (ADR 0008), so the signature is what
+makes a pulled artifact trustworthy in a plane we may not operate. The algorithm is
+minisign at launch and cosign when an audit needs provenance, which is why
+`signature.algorithm` exists in the schema rather than being assumed.
+
+**Signing is a build-time act; verifying is a runtime one.** This module signs, and
+re-exports the verifier so the builder can check its own output — but the verifier
+itself is in `mkp_contracts.signature`, because the runner has to do the same check at
+load and must not depend on a build tool to do it.
+
+What is signed is exactly the canonical manifest bytes — the same bytes the digest is
+taken over, so there is one canonical form and not two. `signature` is outside that
+form, which is what lets a signature be inserted into a manifest without changing the
+bundle's identity.
+
+`signature.value` holds the whole `.minisig` file, base64-encoded: it round-trips to
+bytes minisign verifies as-is, and it stays a single JSON-friendly line.
+"""
+
+from __future__ import annotations
+
+import base64
+import subprocess
+import tempfile
+from pathlib import Path
+
+from mkp_contracts.signature import ALGORITHM, CANONICAL_NAME, available, key_id, verify
+
+from .errors import BuildError
+
+
+def _require() -> None:
+    if not available():
+        raise BuildError("minisign is not on PATH; a bundle cannot be signed or verified")
+
+
+def sign(canonical: bytes, secret_key: Path, public_key: Path, *, digest: str) -> dict[str, str]:
+    """Sign the canonical manifest, returning the manifest's `signature` object.
+
+    The trusted comment carries the bundle digest rather than minisign's default
+    (a timestamp and the temporary file's name), so what the signature asserts is
+    legible without unpacking anything.
+    """
+    _require()
+    with tempfile.TemporaryDirectory() as workspace:
+        target = Path(workspace) / CANONICAL_NAME
+        target.write_bytes(canonical)
+        signature_path = target.with_suffix(".minisig")
+
+        # A fixed argv list and no shell. The key path is the operator's own argument
+        # and reaches the process as one list element, never as text to be parsed.
+        completed = subprocess.run(
+            [
+                ALGORITHM,
+                "-S",
+                "-s",
+                str(secret_key),
+                "-m",
+                str(target),
+                "-x",
+                str(signature_path),
+                "-t",
+                f"bundle:{digest}",
+                "-c",
+                "mklang-platform bundle",
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if completed.returncode != 0:
+            raise BuildError(
+                f"minisign refused to sign (exit {completed.returncode}): "
+                f"{completed.stderr.strip() or completed.stdout.strip()}. "
+                "A build key must be unencrypted — minisign prompts on a terminal otherwise"
+            )
+        payload = signature_path.read_bytes()
+
+    return {
+        "algorithm": ALGORITHM,
+        "key_id": key_id(public_key),
+        "value": base64.b64encode(payload).decode("ascii"),
+    }
+
+
+__all__ = ["ALGORITHM", "available", "key_id", "sign", "verify"]

mkbundle/source.py

@@ -0,0 +1,107 @@
+"""Reading a bundle source directory.
+
+    src/
+      bundle.yaml          project, pinned connector versions, requires
+      workflows/*.yaml     one workflow definition each
+      machines/*.mkl       the machines those workflows commission
+      assets/**            templates, prompts, fixtures
+
+`bundle.yaml` declares intent. Everything a build observes — the file inventory,
+their digests, the mklang and spec versions actually used, the build time — is an
+output and may not be declared, because a declared value can disagree with what
+happened.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .errors import BuildError
+
+SOURCE_MANIFEST = "bundle.yaml"
+
+_DECLARABLE = {"schema_version", "project", "source", "lock", "requires"}
+_PRODUCED = {"created", "contents", "signature"}
+_PRODUCED_LOCK = {"mklang", "spec"}
+
+_WORKFLOW_SUFFIXES = (".yaml", ".yml", ".json")
+
+
+@dataclass
+class BundleSource:
+    root: Path
+    project: str
+    declared: dict[str, Any]
+    workflows: list[Path] = field(default_factory=list)
+    machines: list[Path] = field(default_factory=list)
+    assets: list[Path] = field(default_factory=list)
+
+    @property
+    def connector_pins(self) -> dict[str, int]:
+        return dict(self.declared.get("lock", {}).get("connectors", {}))
+
+    @property
+    def requires(self) -> dict[str, Any]:
+        return dict(self.declared.get("requires", {}))
+
+    def read(self, relative: Path) -> bytes:
+        return (self.root / relative).read_bytes()
+
+
+def load(root: Path) -> BundleSource:
+    root = root.resolve()
+    manifest_path = root / SOURCE_MANIFEST
+    if not manifest_path.is_file():
+        raise BuildError(f"{root}: no {SOURCE_MANIFEST} — this is not a bundle source directory")
+
+    declared = yaml.safe_load(manifest_path.read_text()) or {}
+    if not isinstance(declared, dict):
+        raise BuildError(f"{manifest_path}: expected a mapping")
+
+    _reject_produced_fields(declared, manifest_path)
+
+    project = declared.get("project")
+    if not project:
+        raise BuildError(f"{manifest_path}: `project` is required")
+
+    return BundleSource(
+        root=root,
+        project=project,
+        declared=declared,
+        workflows=_collect(root, "workflows", _WORKFLOW_SUFFIXES),
+        machines=_collect(root, "machines", (".mkl",)),
+        assets=_collect(root, "assets", None),
+    )
+
+
+def _reject_produced_fields(declared: dict[str, Any], path: Path) -> None:
+    """Refuse a source manifest that declares what the build observes."""
+    for key in sorted(_PRODUCED & declared.keys()):
+        raise BuildError(
+            f"{path}: `{key}` is produced by the build and must not be declared — "
+            "a declared value can disagree with what was actually built"
+        )
+    for key in sorted(_PRODUCED_LOCK & set(declared.get("lock", {}))):
+        raise BuildError(
+            f"{path}: `lock.{key}` is read off the toolchain that validates the bundle "
+            "and must not be declared"
+        )
+    unknown = sorted(declared.keys() - _DECLARABLE)
+    if unknown:
+        raise BuildError(f"{path}: unknown field(s) {', '.join(unknown)}")
+
+
+def _collect(root: Path, subdir: str, suffixes: tuple[str, ...] | None) -> list[Path]:
+    directory = root / subdir
+    if not directory.is_dir():
+        return []
+    found = [
+        p
+        for p in sorted(directory.rglob("*"))
+        if p.is_file() and (suffixes is None or p.suffix in suffixes)
+    ]
+    return [p.relative_to(root) for p in found]

mkbundle/toolchain.py

@@ -0,0 +1,86 @@
+"""The mklang toolchain, called as a subprocess.
+
+mklang is a pinned dependency and stays a language (ADR 0010): the platform runs
+its CLI, it does not reimplement `check`. The versions recorded in a bundle's
+lock are the ones that actually validated it, read off the toolchain rather than
+declared by the author — a declared version can be wrong, an observed one cannot.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+from .errors import BuildError, Report
+
+_VERSION_RE = re.compile(r"(\d+\.\d+\.\d+\S*)")
+
+
+def available() -> bool:
+    return shutil.which("mklang") is not None
+
+
+def version() -> str:
+    """The mklang package version, for `lock.mklang`."""
+    if not available():
+        raise BuildError("mklang is not on PATH; the bundle's machines cannot be validated")
+    # `mklang` by name, from PATH, on purpose (S607): the version it reports is what
+    # goes into the bundle's lock, so the toolchain that validated a bundle is the one the
+    # builder had installed. A hard-coded path would be wrong on the next machine.
+    out = subprocess.run(
+        ["mklang", "--version"],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+    match = _VERSION_RE.search(out)
+    if not match:
+        raise BuildError(f"cannot read a version out of `mklang --version` output: {out!r}")
+    return match.group(1)
+
+
+def check(machines: list[Path], root: Path) -> Report:
+    """Run `mklang check` over every machine in the bundle.
+
+    A bundle whose machines do not check does not become a deployment candidate
+    (ADR 0008), so this is a build-time gate and not a warning.
+    """
+    report = Report()
+    if not machines:
+        return report
+    if not available():
+        raise BuildError("mklang is not on PATH; the bundle's machines cannot be validated")
+
+    # Same reasoning as `version()`: by name from PATH, fixed argv, no shell. The
+    # machine paths are inside the bundle root and arrive as separate list elements.
+    completed = subprocess.run(
+        ["mklang", "check", "--format", "json", *[str(root / m) for m in machines]],
+        capture_output=True,
+        text=True,
+    )
+    try:
+        result = json.loads(completed.stdout)
+    except json.JSONDecodeError as exc:
+        raise BuildError(
+            f"`mklang check` produced output that is not JSON (exit {completed.returncode}): "
+            f"{completed.stdout[:400]}{completed.stderr[:400]}"
+        ) from exc
+
+    for item in result.get("items", []):
+        relative = Path(item["path"]).resolve().relative_to(root.resolve()).as_posix()
+        for error in item.get("errors", []):
+            report.fail("machine-check", _describe(error), path=relative)
+        if item.get("status") not in ("ok", "warning") and not item.get("errors"):
+            report.fail("machine-check", f"status {item.get('status')!r}", path=relative)
+    return report
+
+
+def _describe(error: object) -> str:
+    if isinstance(error, dict):
+        code = error.get("code") or error.get("kind") or "error"
+        message = error.get("message") or error.get("detail") or ""
+        return f"{code}: {message}".strip(": ")
+    return str(error)