composable-data-stack 0.10.0__tar.gz → 0.11.0__tar.gz

{composable_data_stack-0.10.0 → composable_data_stack-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: composable-data-stack
-Version: 0.10.0
+Version: 0.11.0
 Summary: CLI for validating and composing composable data stack profiles
 Author: Ronald Hensbergen
 License-Expression: Apache-2.0
@@ -32,7 +32,7 @@ Provides-Extra: dev
 Requires-Dist: coverage>=7.6; extra == "dev"
 Requires-Dist: bandit>=1.7; extra == "dev"
 Requires-Dist: pip-audit>=2.7; extra == "dev"
-Requires-Dist: ruff==0.16.9; extra == "dev"
+Requires-Dist: ruff==0.16.10; extra == "dev"
 Dynamic: license-file
 
 # 🚀 Composable Data Stack (CDS)
@@ -1034,10 +1034,9 @@ Please read these first:
 
 - [CONTRIBUTING.md](CONTRIBUTING.md)
 - [docs/maintainer-merge-policy.md](docs/maintainer-merge-policy.md)
-- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
-- [SECURITY.md](SECURITY.md)
-- [SUPPORT.md](SUPPORT.md)
-- [docs/security-support-policy.md](docs/security-support-policy.md)
+- [SECURITY.md](SECURITY.md) — how to report a vulnerability
+- [LEGAL.md](LEGAL.md) — license, code of conduct, security, support, and
+  regulatory-compliance documentation index
 - [CHANGELOG.md](CHANGELOG.md)
 - [RELEASE.md](RELEASE.md)
 
@@ -1060,4 +1059,5 @@ Development helper tools are located in the gitignored `tools/` directory. See
 
 ## 📜 License
 
-See `LICENSE`.
+See `LICENSE`. For security, support, and regulatory-compliance
+documentation, see [LEGAL.md](LEGAL.md).

{composable_data_stack-0.10.0 → composable_data_stack-0.11.0}/README.md

@@ -997,10 +997,9 @@ Please read these first:
 
 - [CONTRIBUTING.md](CONTRIBUTING.md)
 - [docs/maintainer-merge-policy.md](docs/maintainer-merge-policy.md)
-- [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
-- [SECURITY.md](SECURITY.md)
-- [SUPPORT.md](SUPPORT.md)
-- [docs/security-support-policy.md](docs/security-support-policy.md)
+- [SECURITY.md](SECURITY.md) — how to report a vulnerability
+- [LEGAL.md](LEGAL.md) — license, code of conduct, security, support, and
+  regulatory-compliance documentation index
 - [CHANGELOG.md](CHANGELOG.md)
 - [RELEASE.md](RELEASE.md)
 
@@ -1023,4 +1022,5 @@ Development helper tools are located in the gitignored `tools/` directory. See
 
 ## 📜 License
 
-See `LICENSE`.
+See `LICENSE`. For security, support, and regulatory-compliance
+documentation, see [LEGAL.md](LEGAL.md).

{composable_data_stack-0.10.0 → composable_data_stack-0.11.0}/cli/getter.py

@@ -154,11 +154,13 @@ def _prepare_source_repository(
     defaults to this project's upstream repository, and any `owner/repo` or
     `github.com/...` value is fetched as a tarball for `ref`. Pass `local` to
     explicitly use an existing local directory instead (e.g. an offline/dev
-    checkout) -- `remote`/`ref` are ignored in that case.
+    checkout) -- `remote` and a non-default `ref` are rejected in that case.
     """
     if local is not None:
         if remote is not None:
             raise GetError("Specify only one of --remote and --local")
+        if ref != DEFAULT_REF:
+            raise GetError(f"--ref {ref!r} is ignored with --local: specify only one of --local and --ref")
         yield _validate_source_repository(Path(local).expanduser())
         return
 

composable_data_stack-0.11.0/cli/image_digest_check.py

@@ -0,0 +1,171 @@
+# cli/image_digest_check.py
+"""
+Pinned image digest staleness check (issue #736).
+
+Modules pin third-party base images by digest (e.g.
+`postgres:18@sha256:...`) so deploys are reproducible. When the upstream
+registry republishes the same tag with a new digest (a patched rebuild),
+that pin silently goes stale until someone notices out-of-band. This module
+adds an opt-in, network-gated check that compares a module's pinned digest
+against the digest the registry currently serves for that tag, and reports
+a stable warning diagnostic (W100) when they differ.
+
+The check is off by default so `cds validate`/`cds up` stay usable offline
+and in CI without network access; it is enabled explicitly via
+`--check-image-digests` or the CDS_CHECK_IMAGE_DIGESTS=1 environment
+variable. Any lookup failure (offline, auth failure, unknown tag, ...) is
+reported as "skipped", never as an error, per the issue's acceptance
+criteria.
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+from .image_updates import find_images_in_compose, parse_image_reference
+
+ENV_VAR = "CDS_CHECK_IMAGE_DIGESTS"
+_TIMEOUT = 10
+_MANIFEST_ACCEPT = ", ".join(
+    [
+        "application/vnd.oci.image.index.v1+json",
+        "application/vnd.oci.image.manifest.v1+json",
+        "application/vnd.docker.distribution.manifest.list.v2+json",
+        "application/vnd.docker.distribution.manifest.v2+json",
+    ]
+)
+_USER_AGENT = "cds-digest-check/1.0"
+
+_PINNED_DIGEST_PATTERN = re.compile(r"^([^@]+)@(sha256:[a-f0-9]{64})$")
+_AUTH_PARAM_PATTERN = re.compile(r'(\w+)="([^"]*)"')
+
+
+def is_enabled(explicit: bool | None = None) -> bool:
+    """
+    Resolve whether the staleness check should run: an explicit CLI flag
+    wins, otherwise CDS_CHECK_IMAGE_DIGESTS=1/true/yes/on opts in. Off by
+    default so validate/up/render stay network-free.
+    """
+    if explicit:
+        return True
+    return os.getenv(ENV_VAR, "").strip().lower() in ("1", "true", "yes", "on")
+
+
+def parse_pinned_reference(image: str) -> tuple[str, str] | None:
+    """Split "repo:tag@sha256:..." into ("repo:tag", "sha256:...")."""
+    match = _PINNED_DIGEST_PATTERN.match(image)
+    if not match:
+        return None
+    return match.group(1), match.group(2)
+
+
+def _registry_base_url(registry: str) -> str:
+    if registry == "docker.io":
+        return "https://registry-1.docker.io"
+    return f"https://{registry}"
+
+
+def _fetch_bearer_token(www_authenticate: str) -> str | None:
+    """Follow the OAuth2 token challenge described by a 401 response's
+    WWW-Authenticate header (realm/service/scope), per the Docker Registry
+    v2 auth spec."""
+    if not www_authenticate.lower().startswith("bearer"):
+        return None
+    params = dict(_AUTH_PARAM_PATTERN.findall(www_authenticate))
+    realm = params.get("realm")
+    if not realm:
+        return None
+    query = "&".join(f"{key}={value}" for key, value in params.items() if key != "realm")
+    url = f"{realm}?{query}" if query else realm
+    if not url.startswith(("http://", "https://")):
+        return None
+    try:
+        req = Request(url, headers={"User-Agent": _USER_AGENT})  # nosec B310  # noqa: S310
+        # This check is opt-in and only ever targets registries already
+        # referenced by a module's own pinned image (not arbitrary
+        # user/CLI input). The scheme is restricted to http(s) above, but
+        # the realm host itself is not otherwise restricted: a
+        # compromised/malicious registry could point "realm" at an
+        # internal address. The request carries no credentials and only
+        # a bearer token string is read back from the (JSON-parsed)
+        # response, bounding the impact to an anonymous GET against a
+        # registry-controlled URL rather than to full SSRF.
+        with urlopen(req, timeout=_TIMEOUT) as response:  # nosec B310  # noqa: S310
+            data = json.loads(response.read().decode())
+    except (HTTPError, URLError, ValueError):
+        return None
+    token = data.get("token") or data.get("access_token")
+    return token if isinstance(token, str) else None
+
+
+def fetch_registry_digest(registry: str, repository: str, tag: str) -> str | None:
+    """
+    Return the manifest digest the registry currently serves for
+    repository:tag, or None if it cannot be determined (offline, auth
+    failure, unknown tag, unsupported registry response, ...).
+    """
+    base = _registry_base_url(registry)
+    url = f"{base}/v2/{repository}/manifests/{tag}"
+    headers = {"Accept": _MANIFEST_ACCEPT, "User-Agent": _USER_AGENT}
+
+    for attempt in range(2):
+        try:
+            req = Request(url, headers=headers, method="HEAD")  # nosec B310  # noqa: S310
+            # url is built from a schema-validated registry/repository/tag
+            # triple, always http(s).
+            with urlopen(req, timeout=_TIMEOUT) as response:  # nosec B310  # noqa: S310
+                digest = response.headers.get("Docker-Content-Digest")
+                return digest if isinstance(digest, str) and digest else None
+        except HTTPError as exc:
+            if exc.code == 401 and attempt == 0:
+                www_auth = exc.headers.get("WWW-Authenticate", "") if exc.headers else ""
+                token = _fetch_bearer_token(www_auth)
+                if token:
+                    headers["Authorization"] = "Bearer " + token
+                    continue
+            return None
+        except URLError:
+            return None
+    return None
+
+
+def check_digest_staleness(image: str) -> dict[str, Any]:
+    """
+    Compare a pinned "repo:tag@sha256:..." reference against the digest the
+    registry currently publishes for that same tag.
+
+    Returns a dict with "image", "status" (one of "not-pinned",
+    "lookup-failed", "up-to-date", "stale"), and "latest_digest".
+    """
+    parsed = parse_pinned_reference(image)
+    if parsed is None:
+        return {"image": image, "status": "not-pinned", "latest_digest": None}
+
+    tag_ref, pinned_digest = parsed
+    info = parse_image_reference(tag_ref)
+    repository = info["repository"]
+    if info["namespace"]:
+        repository = f"{info['namespace']}/{repository}"
+    if not repository or not info["registry"]:
+        return {"image": image, "status": "lookup-failed", "latest_digest": None}
+
+    latest_digest = fetch_registry_digest(info["registry"], repository, info["tag"])
+    if latest_digest is None:
+        return {"image": image, "status": "lookup-failed", "latest_digest": None}
+    if latest_digest == pinned_digest:
+        return {"image": image, "status": "up-to-date", "latest_digest": latest_digest}
+    return {"image": image, "status": "stale", "latest_digest": latest_digest}
+
+
+def collect_pinned_images_for_module(module_def: dict[str, Any]) -> list[dict[str, str]]:
+    """Return {service, image} entries for a module's digest-pinned images."""
+    compose = module_def.get("spec", {}).get("implementation", {}).get("compose", {})
+    entries: list[dict[str, str]] = []
+    for service_name, image, _dockerfile in find_images_in_compose(compose):
+        if parse_pinned_reference(image) is not None:
+            entries.append({"service": service_name, "image": image})
+    return entries

{composable_data_stack-0.10.0 → composable_data_stack-0.11.0}/cli/image_verification.py

@@ -322,6 +322,22 @@ def _fixture_entry(
     return matches[0]
 
 
+def lookup_image_evidence(
+    image_ref: str, fixture_path: Path | None
+) -> dict[str, Any] | None:
+    """
+    Return the signed-images fixture entry (repository, digest, signed,
+    provenanceAttested, sbomAttested) matching image_ref, or None when no
+    fixture is configured/loadable or no matching entry exists.
+
+    Public wrapper around `_load_fixture`/`_fixture_entry` for callers
+    outside this module (e.g. `cli.report`'s compliance report) that need
+    read-only evidence lookup without pulling in cosign/live-verification.
+    """
+    fixture, _error = _load_fixture(fixture_path)
+    return _fixture_entry(fixture, image_ref)
+
+
 def _verification_findings(
     images: list[tuple[str, str, bool]],
     policy: ImagePolicy,

{composable_data_stack-0.10.0 → composable_data_stack-0.11.0}/cli/k8s_runner.py

@@ -7,13 +7,20 @@ import os
 import re
 import subprocess  # nosec B404
 import tempfile
+import time
+from collections.abc import Callable
 from pathlib import Path
 from typing import IO, Any
 
 import yaml
 
-from .state import parse_k8s_workloads_json
-from .up_runner import _validate_command, run_streamed
+from .state import group_services_by_health, parse_k8s_workloads_json
+from .up_runner import (
+    DEFAULT_POLL_INTERVAL_SECONDS,
+    _poll_until_settled,
+    _validate_command,
+    run_streamed,
+)
 
 # RFC 1123 DNS label: lowercase alphanumeric and `-`, not leading/trailing
 # with `-`, 1-63 chars. Both Kubernetes namespaces and Helm release names
@@ -90,16 +97,37 @@ def helm_up(
     timeout: float,
     detach: bool,
     log_file: IO[str],
+    poll_interval: float = DEFAULT_POLL_INTERVAL_SECONDS,
+    use_color: bool = False,
+    sleep_fn: Callable[[float], None] = time.sleep,
+    now_fn: Callable[[], float] = time.monotonic,
+    redraw_fn: Callable[[str], None] | None = None,
 ) -> int:
-    """Install or upgrade a rendered chart without persisting secret values."""
+    """
+    Install or upgrade a rendered chart without persisting secret values.
+
+    `timeout` is one overall budget shared across the whole call, the way
+    Compose's `--timeout` already is: it bounds `helm upgrade --install`
+    itself, then whatever remains bounds the readiness poll below, rather
+    than being reused in full for each step (which could let total
+    wall-clock time exceed `timeout` several times over on a
+    multi-workload release).
+
+    Readiness is decided by polling `get_k8s_state()` /
+    `group_services_by_health()` — the exact same functions `cds state
+    --target helm` uses — instead of a separate `helm --wait` +
+    `kubectl rollout status`/`wait` implementation, so "ready" here and
+    "healthy" in `cds state` can never disagree.
+    """
     namespace = _validate_k8s_name(namespace, "namespace")
     release = _validate_k8s_name(release, "release")
     kube_context = _validate_kube_context(kube_context)
     timeout = _validate_timeout(timeout)
+    deadline = now_fn() + timeout
     secret_values = _secret_values(plan)
     secret_path = _write_secret_values(secret_values)
     context_args = ["--kube-context", kube_context] if kube_context else []
-    timeout_arg = f"{timeout}s"
+    apply_timeout_arg = f"{timeout}s"
     command = [
         "helm",
         *context_args,
@@ -113,10 +141,8 @@ def helm_up(
         "--values",
         str(secret_path),
         "--timeout",
-        timeout_arg,
+        apply_timeout_arg,
     ]
-    if not detach:
-        command.append("--wait")
 
     try:
         result = run_streamed(command, log_file, timeout=timeout + 30)
@@ -126,31 +152,76 @@ def helm_up(
         return result
 
     workloads = get_k8s_workloads(namespace, release, kube_context)
-    for workload in workloads:
-        kind = str(workload.get("kind", "")).lower()
-        name = str((workload.get("metadata") or {}).get("name", ""))
-        if not kind or not name:
-            continue
-        if kind == "job":
-            wait_command = _kubectl_command(kube_context, namespace) + [
-                "wait",
-                "--for=condition=complete",
-                f"job/{name}",
-                f"--timeout={timeout_arg}",
-            ]
-        else:
-            wait_command = _kubectl_command(kube_context, namespace) + [
-                "rollout",
-                "status",
-                f"{kind}/{name}",
-                f"--timeout={timeout_arg}",
-            ]
-        result = run_streamed(wait_command, log_file, timeout=timeout + 30)
-        if result != 0:
-            return result
+    remaining = max(0.0, deadline - now_fn())
+    settled, grouped = poll_k8s_state_until_settled(
+        namespace,
+        release,
+        kube_context,
+        expected_service_count=len(workloads),
+        poll_interval=poll_interval,
+        timeout=remaining,
+        use_color=use_color,
+        sleep_fn=sleep_fn,
+        now_fn=now_fn,
+        redraw_fn=redraw_fn,
+    )
+    if not settled:
+        unhealthy = [name for bucket in ("UNHEALTHY", "UNHEALTHY EXIT") for name in grouped.get(bucket, [])]
+        log_file.write(
+            f"helm release {release} did not settle within {timeout:.0f}s"
+            + (f"; unhealthy: {', '.join(unhealthy)}" if unhealthy else "")
+            + "\n"
+        )
+        log_file.flush()
+        return 1
     return 0
 
 
+def poll_k8s_state_until_settled(
+    namespace: str,
+    release: str,
+    kube_context: str | None,
+    *,
+    expected_service_count: int | None = None,
+    poll_interval: float = DEFAULT_POLL_INTERVAL_SECONDS,
+    timeout: float = 180.0,
+    use_color: bool = False,
+    sleep_fn: Callable[[float], None] = time.sleep,
+    now_fn: Callable[[], float] = time.monotonic,
+    fetch_fn: Callable[[], list[dict[str, Any]]] | None = None,
+    redraw_fn: Callable[[str], None] | None = None,
+) -> tuple[bool, dict[str, list[str]]]:
+    """
+    Polls `get_k8s_state()` every `poll_interval` seconds, grouping with
+    `group_services_by_health()` — the same pair `cds state --target
+    helm` calls — until every workload settles into a terminal bucket or
+    `timeout` seconds elapse. Shares its settle/failure rule with
+    Compose's `poll_state_until_settled()` via `_poll_until_settled()`.
+
+    `fetch_fn` is injectable for tests; defaults to a real
+    `get_k8s_state(namespace, release, kube_context)` call.
+    """
+    if fetch_fn is None:
+        def fetch_fn() -> list[dict[str, Any]]:
+            return get_k8s_state(namespace, release, kube_context)
+
+    def fetch_grouped() -> dict[str, list[str]]:
+        return group_services_by_health(fetch_fn())
+
+    return _poll_until_settled(
+        fetch_grouped,
+        expected_service_count=expected_service_count,
+        poll_interval=poll_interval,
+        timeout=timeout,
+        use_color=use_color,
+        sleep_fn=sleep_fn,
+        now_fn=now_fn,
+        redraw_fn=redraw_fn,
+        up_done_fn=None,
+        on_up_finished=None,
+    )
+
+
 def helm_down(
     *,
     namespace: str,

{composable_data_stack-0.10.0 → composable_data_stack-0.11.0}/cli/loader.py

@@ -252,12 +252,12 @@ def save_generated_profile(
         ]
 
     name_path = Path(name)
-    if name_path.is_absolute() or ".." in name_path.parts:
+    if name_path.is_absolute() or ".." in name_path.parts or len(name_path.parts) != 1:
         return None, [
             Diagnostic(
                 level="error",
                 code="E115",
-                message=f'Generated profile name "{name}" must be a relative name without ".." segments.',
+                message=f'Generated profile name "{name}" must be a single relative path segment.',
                 path="metadata.name",
             )
         ]