mkbundle 0.2.0__py3-none-any.whl

mkbundle/__init__.py

@@ -0,0 +1,12 @@
+"""mkbundle — build and validate the artifacts this platform deploys.
+
+Bundles are immutable, content-addressed and signed (ADR 0008). A bundle that
+fails validation never becomes a deployment candidate, so this tool is the gate
+between authoring and everything downstream.
+
+Named for what it makes rather than for the platform: `mkp` is the platform, and
+that name belongs to its roles, its settings and its services rather than to one of
+its tools.
+"""
+
+__version__ = "0.1.0"

mkbundle/build.py

@@ -0,0 +1,286 @@
+"""Building a bundle: validate everything, then emit an archive identified by its digest.
+
+The order matters. A bundle that fails validation never becomes a deployment
+candidate (ADR 0008), so nothing is written until every rule has passed. The
+manifest is assembled from what the build observed — file digests, the toolchain
+versions it validated with — and its digest is the bundle's identity.
+"""
+
+from __future__ import annotations
+
+import gzip
+import io
+import json
+import tarfile
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from mkp_contracts import schemas
+from mkp_contracts.archive import MANIFEST_NAME, manifest_of
+from mkp_contracts.canonical import canonical_bytes, file_digest, manifest_digest
+from mkp_contracts.errors import ContractError
+
+from . import catalog as catalog_module
+from . import rules, signing, source, toolchain
+from .errors import BuildError, Report
+
+
+@dataclass
+class BuildResult:
+    manifest: dict[str, Any]
+    digest: str
+    report: Report
+    archive: Path | None = None
+
+
+def build(
+    src: Path,
+    *,
+    connectors_dir: Path | None = None,
+    output: Path | None = None,
+    created: datetime | None = None,
+    sign_key: Path | None = None,
+    public_key: Path | None = None,
+) -> BuildResult:
+    bundle = source.load(src)
+    report = Report()
+
+    catalog = None
+    if connectors_dir is not None:
+        catalog = catalog_module.load(connectors_dir)
+    else:
+        report.skipped_rules.append(
+            "unknown-action, connector-version-mismatch, missing-idempotency-key — "
+            "no connector catalog given (--connectors)"
+        )
+    if sign_key is None:
+        report.skipped_rules.append(
+            "signing — no key given (--sign-key); a runner refuses an unsigned bundle"
+        )
+
+    machine_names = rules.machine_paths(bundle.machines)
+    declared = bundle.requires
+    declared_connections = set(declared.get("connections") or [])
+    declared_secrets = set(declared.get("secrets") or [])
+
+    seen_names: dict[str, str] = {}
+    for relative in bundle.workflows:
+        path = relative.as_posix()
+        document = schemas.load_document(bundle.root / relative)
+        shape = schemas.validate_document(document, "workflow", path=path)
+        report.extend(shape)
+        if not shape.ok or not isinstance(document, dict):
+            continue
+
+        name = document.get("name", "")
+        if name in seen_names:
+            report.fail(
+                "duplicate-workflow-name",
+                f"workflow name {name!r} is already used by {seen_names[name]}; "
+                "names are unique within a project",
+                path=path,
+            )
+        else:
+            seen_names[name] = path
+
+        report.extend(
+            rules.check_workflow(
+                document,
+                path,
+                machines=machine_names,
+                declared_connections=declared_connections,
+                declared_secrets=declared_secrets,
+                connector_pins=bundle.connector_pins,
+                catalog=catalog,
+            )
+        )
+
+    specs = {}
+    for relative in bundle.machines:
+        text = (bundle.root / relative).read_text()
+        report.extend(rules.check_machine_gates(text, relative.as_posix()))
+        spec = rules.read_spec_version(text)
+        if spec is None:
+            report.fail(
+                "spec-missing",
+                "machine declares no `mklang:` spec version",
+                path=relative.as_posix(),
+            )
+        else:
+            specs[relative.as_posix()] = spec
+    report.extend(rules.check_spec_versions(specs))
+    report.extend(toolchain.check(bundle.machines, bundle.root))
+
+    manifest = _assemble(bundle, specs, created=created)
+    report.extend(schemas.validate_document(manifest, "bundle", path=MANIFEST_NAME))
+
+    digest = manifest_digest(manifest)
+    if not report.ok:
+        return BuildResult(manifest=manifest, digest=digest, report=report)
+
+    if sign_key is not None:
+        if public_key is None:
+            raise BuildError("--sign-key needs --public-key: the key id is read off the public key")
+        # Signing after the digest, and it does not move it: `signature` is outside
+        # the canonical form, so a bundle keeps its identity when it gains one.
+        manifest["signature"] = signing.sign(
+            canonical_bytes(manifest), sign_key, public_key, digest=digest
+        )
+        if manifest_digest(manifest) != digest:
+            # Raised rather than asserted (`H3`): `python -O` removes an `assert`, and this
+            # is the one line standing between a signed bundle and a digest that names
+            # something else. `signature` is excluded from the canonical form precisely so
+            # that this cannot happen, which makes it the invariant worth paying for.
+            raise BuildError(
+                "signing moved the bundle's digest, which must never happen: "
+                "`signature` is excluded from the canonical form (ADR 0008)"
+            )
+
+    archive = None
+    if output is not None:
+        archive = _write_archive(bundle, manifest, digest, output)
+    return BuildResult(manifest=manifest, digest=digest, report=report, archive=archive)
+
+
+def verify(archive: Path, public_key: Path) -> Report:
+    """Check a built bundle: its signature, and every file against its recorded digest.
+
+    Verify-at-load is the runner's (C3) and is not this. This exists so the tool can
+    prove its own output rather than assert it.
+    """
+    report = Report()
+    manifest = read_archive(archive)
+
+    signature = manifest.get("signature")
+    if not signature:
+        report.fail("unsigned", "the bundle carries no signature; a runner refuses it (ADR 0008)")
+    else:
+        report.extend(signing.verify(canonical_bytes(manifest), signature, public_key))
+
+    with tarfile.open(archive, mode="r:gz") as tar:
+        for group in ("workflows", "machines", "assets"):
+            for entry in manifest["contents"].get(group, []):
+                member = tar.extractfile(entry["path"])
+                if member is None:
+                    report.fail(
+                        "content-missing", "listed in the manifest but absent", path=entry["path"]
+                    )
+                    continue
+                if file_digest(member.read()) != entry["sha256"]:
+                    report.fail(
+                        "content-digest",
+                        "content does not match the digest the manifest records",
+                        path=entry["path"],
+                    )
+    return report
+
+
+def _assemble(
+    bundle: source.BundleSource,
+    specs: dict[str, str],
+    *,
+    created: datetime | None,
+) -> dict[str, Any]:
+    spec_versions = sorted(set(specs.values()))
+    if not spec_versions:
+        declared_spec = bundle.declared.get("lock", {}).get("spec")
+        if not declared_spec:
+            raise BuildError(
+                "the bundle has no machines, so `lock.spec` cannot be observed — "
+                "declare it in bundle.yaml or add a machine"
+            )
+        spec_versions = [str(declared_spec)]
+
+    contents: dict[str, Any] = {
+        "workflows": [_file_entry(bundle, p) for p in bundle.workflows],
+        "machines": [_file_entry(bundle, p) for p in bundle.machines],
+    }
+    if bundle.assets:
+        contents["assets"] = [_file_entry(bundle, p) for p in bundle.assets]
+
+    stamp = created or datetime.now(UTC)
+    manifest: dict[str, Any] = {
+        "schema_version": 1,
+        "project": bundle.project,
+        "created": stamp.replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+        "contents": contents,
+        "lock": {
+            "mklang": toolchain.version(),
+            "spec": spec_versions[0],
+            "connectors": bundle.connector_pins,
+        },
+    }
+    if "source" in bundle.declared:
+        manifest["source"] = bundle.declared["source"]
+    if bundle.requires:
+        manifest["requires"] = bundle.requires
+    return manifest
+
+
+def _file_entry(bundle: source.BundleSource, relative: Path) -> dict[str, str]:
+    return {"path": relative.as_posix(), "sha256": file_digest(bundle.read(relative))}
+
+
+def _write_archive(
+    bundle: source.BundleSource, manifest: dict[str, Any], digest: str, output: Path
+) -> Path:
+    """A deterministic tar.gz: same content in, same bytes out.
+
+    Identity is the manifest digest, not the archive's, but a reproducible
+    archive is what makes "nothing is rebuilt on promote" checkable rather than
+    merely asserted.
+    """
+    target = output / f"{bundle.project}-{digest[:12]}.tar.gz" if output.is_dir() else output
+    target.parent.mkdir(parents=True, exist_ok=True)
+
+    full_manifest = json.dumps(
+        manifest, sort_keys=True, separators=(",", ":"), ensure_ascii=False
+    ).encode("utf-8")
+
+    buffer = io.BytesIO()
+    with tarfile.open(fileobj=buffer, mode="w") as tar:
+        _add(tar, MANIFEST_NAME, full_manifest)
+        for entry in _all_files(manifest):
+            _add(tar, entry, bundle.read(Path(entry)))
+
+    # filename="" keeps the output file's own name out of the gzip header, which
+    # would otherwise make two builds of identical content differ.
+    with (
+        open(target, "wb") as handle,
+        gzip.GzipFile(filename="", fileobj=handle, mode="wb", mtime=0) as gz,
+    ):
+        gz.write(buffer.getvalue())
+    return target
+
+
+def _all_files(manifest: dict[str, Any]) -> list[str]:
+    contents = manifest["contents"]
+    paths: list[str] = []
+    for group in ("workflows", "machines", "assets"):
+        paths.extend(entry["path"] for entry in contents.get(group, []))
+    return sorted(paths)
+
+
+def _add(tar: tarfile.TarFile, name: str, data: bytes) -> None:
+    info = tarfile.TarInfo(name)
+    info.size = len(data)
+    info.mtime = 0
+    info.mode = 0o644
+    info.uid = info.gid = 0
+    info.uname = info.gname = ""
+    tar.addfile(info, io.BytesIO(data))
+
+
+def read_archive(path: Path) -> dict[str, Any]:
+    """The manifest inside a built bundle.
+
+    Delegates to `mkp_contracts`, which the control plane also uses to decide whether an
+    uploaded archive belongs under the digest it was filed as. One reader, so the two
+    cannot disagree about what a bundle contains.
+    """
+    try:
+        return manifest_of(path.read_bytes())
+    except ContractError as exc:
+        raise BuildError(f"{path}: {exc}") from exc

mkbundle/catalog.py

@@ -0,0 +1,48 @@
+"""The connector catalog a build validates workflow steps against.
+
+A connector is a document, not a module (ADR 0007). The catalog is a directory of
+them — `<dir>/<name>/connector.yaml`, or `<dir>/<name>.yaml` — which is what lets
+a build answer "does this action exist, and does it write?" without importing
+anything.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from .errors import BuildError
+
+
+@dataclass(frozen=True)
+class Connector:
+    name: str
+    version: int
+    actions: dict[str, dict[str, Any]]
+    path: Path
+
+    def action(self, name: str) -> dict[str, Any] | None:
+        return self.actions.get(name)
+
+
+def load(directory: Path) -> dict[str, Connector]:
+    directory = directory.resolve()
+    if not directory.is_dir():
+        raise BuildError(f"{directory}: connector catalog is not a directory")
+
+    catalog: dict[str, Connector] = {}
+    for path in sorted([*directory.glob("*/connector.yaml"), *directory.glob("*.yaml")]):
+        document = yaml.safe_load(path.read_text()) or {}
+        name = document.get("name")
+        if not name:
+            raise BuildError(f"{path}: connector manifest has no `name`")
+        catalog[name] = Connector(
+            name=name,
+            version=document.get("version", 0),
+            actions=document.get("actions", {}) or {},
+            path=path,
+        )
+    return catalog

mkbundle/cli.py

@@ -0,0 +1,202 @@
+"""The `mkp` command line.
+
+Three verbs: validate a document, build a bundle, inspect a built one. One tool
+rather than a builder and a separate validator, because the rules would otherwise
+be written twice and drift once.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.metadata
+import json
+import sys
+from pathlib import Path
+
+from mkp_contracts import schemas
+from mkp_contracts.canonical import manifest_digest
+
+from . import build as build_module
+from .errors import BuildError, Report
+
+
+def main(argv: list[str] | None = None) -> int:
+    # `mkp` is the product namespace and deliberately not the name of any one tool
+    # (README.md §Stack); the binary is `mkbundle` and `--version` prints `%(prog)s`, so
+    # this said the wrong name in `--help` and in `--version` both.
+    parser = argparse.ArgumentParser(prog="mkbundle", description=(__doc__ or "").splitlines()[0])
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"%(prog)s {importlib.metadata.version('mkbundle')}",
+    )
+    parser.add_argument("--format", choices=("text", "json"), default="text")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    validate = subparsers.add_parser(
+        "validate", help="validate a workflow, connector manifest, or bundle manifest"
+    )
+    validate.add_argument("paths", nargs="+", type=Path)
+    validate.add_argument(
+        "--kind",
+        choices=schemas.KINDS,
+        help="skip detection and validate against this schema",
+    )
+
+    build = subparsers.add_parser("build", help="validate a bundle source tree and archive it")
+    build.add_argument("src", type=Path)
+    build.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        help="archive path, or a directory to name it in. Omit to validate without writing",
+    )
+    build.add_argument(
+        "--connectors",
+        type=Path,
+        help="connector catalog. Without it the action and idempotency rules cannot run",
+    )
+    build.add_argument("--sign-key", type=Path, help="minisign secret key, unencrypted")
+    build.add_argument("--public-key", type=Path, help="its public key, for the key id")
+
+    inspect = subparsers.add_parser("inspect", help="print the manifest of a built bundle")
+    inspect.add_argument("archive", type=Path)
+
+    verify = subparsers.add_parser(
+        "verify", help="check a built bundle's signature and its contents against the manifest"
+    )
+    verify.add_argument("archive", type=Path)
+    verify.add_argument("--public-key", type=Path, required=True)
+
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "validate":
+            return _validate(args)
+        if args.command == "build":
+            return _build(args)
+        if args.command == "verify":
+            return _verify(args)
+        return _inspect(args)
+    except BuildError as exc:
+        _emit_error(str(exc), args.format)
+        return 2
+
+
+def _validate(args: argparse.Namespace) -> int:
+    report = Report()
+    for path in args.paths:
+        if not path.is_file():
+            report.fail("not-found", "no such file", path=str(path))
+            continue
+        document = schemas.load_document(path)
+        kind = args.kind or schemas.detect_kind(document)
+        if kind is None:
+            report.fail(
+                "unknown-kind",
+                "cannot tell which contract this document is against; pass --kind",
+                path=str(path),
+            )
+            continue
+        report.extend(schemas.validate_document(document, kind, path=str(path)))
+    return _report(report, args.format, ok_message=f"{len(args.paths)} document(s) valid")
+
+
+def _build(args: argparse.Namespace) -> int:
+    connectors = args.connectors
+    if connectors is None:
+        default = Path.cwd() / "connectors"
+        if (default / "README.md").exists() and any(default.glob("*/connector.yaml")):
+            connectors = default
+
+    result = build_module.build(
+        args.src,
+        connectors_dir=connectors,
+        output=args.output,
+        sign_key=args.sign_key,
+        public_key=args.public_key,
+    )
+
+    if args.format == "json":
+        print(
+            json.dumps(
+                {
+                    "ok": result.report.ok,
+                    "digest": result.digest,
+                    "archive": str(result.archive) if result.archive else None,
+                    "skipped_rules": result.report.skipped_rules,
+                    "failures": [f.as_dict() for f in result.report.failures],
+                },
+                indent=2,
+            )
+        )
+        return 0 if result.report.ok else 1
+
+    for rule in result.report.skipped_rules:
+        print(f"not checked: {rule}", file=sys.stderr)
+    if not result.report.ok:
+        _print_failures(result.report)
+        return 1
+    print(f"digest  {result.digest}")
+    if result.archive:
+        print(f"archive {result.archive}")
+    return 0
+
+
+def _verify(args: argparse.Namespace) -> int:
+    report = build_module.verify(args.archive, args.public_key)
+    return _report(
+        report, args.format, ok_message=f"{args.archive.name}: signature and contents ok"
+    )
+
+
+def _inspect(args: argparse.Namespace) -> int:
+    manifest = build_module.read_archive(args.archive)
+    if args.format == "json":
+        print(json.dumps({"digest": manifest_digest(manifest), "manifest": manifest}, indent=2))
+        return 0
+
+    print(f"digest   {manifest_digest(manifest)}")
+    print(f"project  {manifest['project']}")
+    print(f"created  {manifest.get('created', '-')}")
+    lock = manifest["lock"]
+    print(f"mklang   {lock['mklang']} (spec {lock['spec']})")
+    for name, version in sorted(lock.get("connectors", {}).items()):
+        print(f"  pin    {name} v{version}")
+    for group in ("workflows", "machines", "assets"):
+        for entry in manifest["contents"].get(group, []):
+            print(f"  {group[:-1]:<9} {entry['path']}")
+    signature = manifest.get("signature")
+    print(f"signed   {signature['algorithm'] + ' ' + signature['key_id'] if signature else 'no'}")
+    return 0
+
+
+def _report(report: Report, output_format: str, *, ok_message: str) -> int:
+    if output_format == "json":
+        print(
+            json.dumps(
+                {"ok": report.ok, "failures": [f.as_dict() for f in report.failures]}, indent=2
+            )
+        )
+        return 0 if report.ok else 1
+    if report.ok:
+        print(ok_message)
+        return 0
+    _print_failures(report)
+    return 1
+
+
+def _print_failures(report: Report) -> None:
+    for failure in report.failures:
+        print(str(failure), file=sys.stderr)
+    print(f"\n{len(report.failures)} problem(s)", file=sys.stderr)
+
+
+def _emit_error(message: str, output_format: str) -> None:
+    if output_format == "json":
+        print(json.dumps({"ok": False, "error": message}, indent=2))
+    else:
+        print(f"mkp: {message}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

mkbundle/errors.py

@@ -0,0 +1,17 @@
+"""Build-specific failures, over the shared reporting vocabulary.
+
+`Failure` and `Report` belong to `mkp_contracts`: reporting every rule a document
+breaks rather than the first is a property of checking a contract, not of building
+something. What is local here is the one exception that means *the build cannot
+proceed at all*, as opposed to a rule the artifact fails.
+"""
+
+from __future__ import annotations
+
+from mkp_contracts.errors import ContractError, Failure, Report
+
+__all__ = ["BuildError", "Failure", "Report"]
+
+
+class BuildError(ContractError):
+    """Something stopped the build before rules could be evaluated at all."""