composable-data-stack 0.12.0__tar.gz → 0.13.0__tar.gz

{composable_data_stack-0.12.0/composable_data_stack.egg-info → composable_data_stack-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: composable-data-stack
-Version: 0.12.0
+Version: 0.13.0
 Summary: CLI for validating and composing composable data stack profiles
 Author: Ronald Hensbergen
 License-Expression: Apache-2.0
@@ -32,7 +32,7 @@ Provides-Extra: dev
 Requires-Dist: coverage>=7.6; extra == "dev"
 Requires-Dist: bandit>=1.7; extra == "dev"
 Requires-Dist: pip-audit>=2.7; extra == "dev"
-Requires-Dist: ruff==0.17.0; extra == "dev"
+Requires-Dist: ruff==0.17.1; extra == "dev"
 Dynamic: license-file
 
 # 🚀 Composable Data Stack (CDS)

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/getter.py

@@ -664,7 +664,7 @@ def _extract_dockerfile_instruction_sources(
             continue
         if _contains_template(source):
             continue
-        if Path(source).is_absolute():
+        if Path(source).is_absolute() or source.startswith(("/", "\\")):
             continue
         matches = list(context_path.glob(source))
         if not matches and "*" not in source and "?" not in source and "[" not in source:

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/k8s_renderer.py

@@ -38,6 +38,11 @@ from .renderer import (
     _resolve_project_root,
     _substitute_values,
 )
+from .secret_leak import (
+    check_generated_text,
+    declared_secret_env_names,
+    resolved_secret_values,
+)
 from .utils import _atomic_write
 
 CHART_API_VERSION = "v2"
@@ -73,12 +78,14 @@ class K8sRenderError(Exception):
 def render_helm(
     plan: dict[str, Any],
     output_dir: str | None = None,
+    env_file: str | None = None,
 ) -> tuple[dict[str, str], list[Diagnostic]]:
     """
     Render a Helm chart from a resolved plan.
 
     Returns a mapping of chart-relative path -> file contents, plus diagnostics.
-    Files are written to `output_dir` when given.
+    Files are written to `output_dir` when given, unless an error was found
+    (including a resolved secret value in any chart file, CDS-SEC-030).
     """
     diagnostics: list[Diagnostic] = []
     metadata = plan.get("metadata", {})
@@ -183,6 +190,11 @@ def render_helm(
 
     diagnostics.extend(_check_unresolved(files))
     diagnostics.extend(_check_secret_references(files, secret_keys))
+    secret_values = resolved_secret_values(declared_secret_env_names(plan), env_file)
+    for relative_path, content in files.items():
+        diagnostics.extend(check_generated_text(
+            content, f"{output_dir or 'chart'}/{relative_path}", secret_values
+        ))
 
     if output_dir and not any(
         diagnostic.level == "error" for diagnostic in diagnostics

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/main.py

@@ -2,6 +2,8 @@
 from __future__ import annotations
 
 import argparse
+import contextlib
+import io
 import json
 import logging
 import os
@@ -48,6 +50,11 @@ from .planner import build_plan
 from .preflight import preflight_passed, run_preflight
 from .renderer import render_compose
 from .report import build_compliance_report
+from .secret_leak import (
+    CLI_OUTPUT_REDACTED_CODE,
+    RedactingStream,
+    resolved_secret_values,
+)
 from .security import PrecomputedRender, run_security_validation
 from .security_common import (
     COMPLIANCE_CATEGORIES,
@@ -1016,6 +1023,10 @@ def _write_env_file(
     )
     lines.append("")
     _atomic_write(output_path, "\n".join(lines))
+    # Holds secrets: keep it owner-only even if _atomic_write's temp-file
+    # creation ever changes (CDS-SEC-032). No-op semantics on Windows.
+    if os.name != "nt":
+        output_path.chmod(0o600)
 
 
 def _cds_version() -> str:
@@ -1320,7 +1331,7 @@ def _run_image_verification(
         return [_unverifiable_image_finding(f"Image verification failed unexpectedly: {e}")]
 
 
-def _render_helm_chart(plan, output_dir, force):
+def _render_helm_chart(plan, output_dir, force, env_file=None):
     """
     Render the Helm chart for a plan and report where it landed.
 
@@ -1341,7 +1352,7 @@ def _render_helm_chart(plan, output_dir, force):
                 "like a CDS-generated chart. Pass --force to overwrite it."
             )
             return 1, []
-    files, diags = render_helm(plan, output_dir=str(target_dir))
+    files, diags = render_helm(plan, output_dir=str(target_dir), env_file=env_file)
     return 0, diags
 
 
@@ -1366,10 +1377,74 @@ def _maybe_notify_update() -> None:
         _LOGGER.debug("Automatic update notice failed", exc_info=True)
 
 
+_REDACTED_OUTPUT_COMMANDS = frozenset({"validate", "plan", "test", "security"})
+_output_redaction: tuple[Any, Any, set[str]] | None = None
+
+
+def _install_output_redaction(args: argparse.Namespace) -> None:
+    """CDS-SEC-071: redact resolved secret values from this command's stdout/stderr.
+
+    Runs for commands that print plans, diagnostics or findings, unless the
+    user explicitly asked for unredacted output with --reveal-secrets.
+    Failing to work out the secret set must not leave output unprotected, so
+    it falls back to the secret-named CDS_* variables alone.
+    """
+    global _output_redaction
+    if args.command not in _REDACTED_OUTPUT_COMMANDS or getattr(args, "reveal_secrets", False):
+        return
+    declared: set[str] = set()
+    env_file = None
+    try:
+        # The command resolves the profile itself and prints any warning;
+        # don't print it a second time from this pre-pass.
+        with contextlib.redirect_stdout(io.StringIO()):
+            profile_path = resolve_profile_path(args.profile)
+            env_file = str(resolve_env_file_path(profile_path))
+            declared = _collect_profile_env_vars(profile_path, getattr(args, "environment", None))[1]
+    except Exception:  # noqa: BLE001 - redaction must never break the command itself
+        _LOGGER.debug("Could not collect declared secrets for output redaction", exc_info=True)
+    secret_values = resolved_secret_values(declared, env_file)
+    if not secret_values:
+        return
+    hits: set[str] = set()
+    _output_redaction = (sys.stdout, sys.stderr, hits)
+    sys.stdout = RedactingStream(sys.stdout, secret_values, hits)
+    sys.stderr = RedactingStream(sys.stderr, secret_values, hits)
+
+
+def _finish_output_redaction() -> None:
+    global _output_redaction
+    if _output_redaction is None:
+        return
+    original_stdout, original_stderr, hits = _output_redaction
+    _output_redaction = None
+    for stream in (sys.stdout, sys.stderr):
+        if isinstance(stream, RedactingStream):
+            stream.finish()
+        else:
+            stream.flush()
+    sys.stdout, sys.stderr = original_stdout, original_stderr
+    if hits:
+        print(
+            Diagnostic(
+                level="warning",
+                code=CLI_OUTPUT_REDACTED_CODE,
+                message=(
+                    "[CDS-SEC-071] Output contained the resolved value of "
+                    f"{', '.join(sorted(hits))}; it was redacted. Find where the literal "
+                    "value is being copied into config and use a secrets.<alias> reference instead."
+                ),
+                path="cli-output",
+            ).format(),
+            file=sys.stderr,
+        )
+
+
 def main() -> int:
     try:
         return _run()
     finally:
+        _finish_output_redaction()
         _maybe_notify_update()
 
 
@@ -1897,6 +1972,8 @@ def _run() -> int:
     if args.command in {"validate", "render", "up", "down", "test", "state", "security"}:
         args.target = getattr(args, "target", None) or load_saved_target() or "compose"
     
+    _install_output_redaction(args)
+
     if args.command == "validate":
         try:
             profile_path = resolve_profile_path(args.profile)
@@ -2158,7 +2235,7 @@ def _run() -> int:
 
             env_file = str(resolve_env_file_path(str(source_profile)))
             if args.target == "helm":
-                code, render_diags = _render_helm_chart(plan, output_path, args.force)
+                code, render_diags = _render_helm_chart(plan, output_path, args.force, env_file=env_file)
                 all_diags = render_diags
                 if code != 0:
                     _record_audit("render", str(source_profile), "failure", plan=plan)
@@ -2222,7 +2299,7 @@ def _run() -> int:
                 )
 
             if args.target == "helm":
-                code, render_diags = _render_helm_chart(plan, output_path, args.force)
+                code, render_diags = _render_helm_chart(plan, output_path, args.force, env_file=env_file)
                 all_diags = all_diags + render_diags
                 if code != 0:
                     _record_audit("render", profile_or_plan, "failure", environment=args.environment, plan=plan)
@@ -2302,7 +2379,7 @@ def _run() -> int:
         if args.target == "helm":
             project_root = resolve_project_root(profile_path)
             chart_dir = (Path(args.chart_dir) if args.chart_dir else project_root / "chart").resolve()
-            code, render_diags = _render_helm_chart(plan, chart_dir, force=True)
+            code, render_diags = _render_helm_chart(plan, chart_dir, force=True, env_file=env_file)
             all_diags.extend(render_diags)
             if code != 0 or has_errors(all_diags):
                 print_diagnostics(all_diags)

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/preflight.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
+import json
 import os
 import re
 import shutil
 import socket
+import stat
 import subprocess  # nosec B404
 from dataclasses import dataclass
 from pathlib import Path
@@ -88,17 +90,156 @@ def _check_runtime(runtime: dict[str, Any]) -> list[PreflightCheck]:
             "Docker Compose is unavailable. Install the Compose plugin.",
         )
     )
-    checks.append(
-        _run_runtime_command(
-            ["docker", "info"],
-            "runtime.daemon",
-            "Docker daemon is reachable.",
-            "Docker daemon is unreachable. Start Docker and verify access with `docker info`.",
+    daemon_check, docker_info = _check_daemon()
+    checks.append(daemon_check)
+    if docker_info is not None:
+        checks.extend(_check_host_hardening(docker_info))
+    return checks
+
+
+def _check_daemon() -> tuple[PreflightCheck, dict[str, Any] | None]:
+    """Probe the daemon with one `docker info` call; also return its parsed JSON."""
+    command = ["docker", "info", "--format", "{{json .}}"]
+    try:
+        result = subprocess.run(  # nosec B603  # noqa: S603
+            command,
+            capture_output=True,
+            text=True,
+            timeout=15,
         )
-    )
+    except (OSError, subprocess.TimeoutExpired):
+        result = None
+
+    if result is None or result.returncode != 0:
+        message = "Docker daemon is unreachable. Start Docker and verify access with `docker info`."
+        detail = ((result.stderr or "").strip().splitlines() or [""])[0][:200] if result else ""
+        if detail:
+            message = f"{message} ({detail})"
+        return PreflightCheck("FAIL", "runtime.daemon", message), None
+
+    info: dict[str, Any] | None = None
+    try:
+        parsed = json.loads(result.stdout or "")
+    except (TypeError, ValueError):
+        parsed = None
+    if isinstance(parsed, dict):
+        info = parsed
+    return PreflightCheck("PASS", "runtime.daemon", "Docker daemon is reachable."), info
+
+
+def _check_host_hardening(info: dict[str, Any]) -> list[PreflightCheck]:
+    """
+    Inspect host/daemon settings from `docker info` without privileged access.
+
+    All findings are advisory (PASS/WARN): the host baseline in
+    docs/docker-host-hardening.md is an operator responsibility, so these
+    checks never fail preflight.
+    """
+    # Docker Desktop's daemon runs in a managed VM the operator cannot harden.
+    if "docker desktop" in str(info.get("OperatingSystem", "")).lower():
+        return []
+
+    raw_options = info.get("SecurityOptions")
+    options = {str(o) for o in raw_options} if isinstance(raw_options, list) else set()
+    rootless = "name=rootless" in options
+    checks = [
+        _host_check(
+            rootless,
+            "host.rootless",
+            "Docker daemon runs in rootless mode.",
+            "Docker daemon runs as root. Prefer rootless mode (HOST-001).",
+        ),
+        _host_check(
+            rootless or "name=userns" in options,
+            "host.userns",
+            "User namespace isolation is active.",
+            "User namespace remapping is not enabled (HOST-002).",
+        ),
+        _host_check(
+            any(o.startswith("name=seccomp") for o in options)
+            and "name=seccomp,profile=unconfined" not in options,
+            "host.seccomp",
+            "Daemon default seccomp profile is active.",
+            "Daemon default seccomp profile is not active (HOST-003).",
+        ),
+        _host_check(
+            info.get("LoggingDriver") not in {None, "", "none"},
+            "host.logging",
+            f'Daemon logging driver "{info.get("LoggingDriver")}" is enabled '
+            "(log rotation is not verified; check daemon.json, HOST-004).",
+            "Daemon logging driver is disabled; container logs are not retained (HOST-004).",
+        ),
+        _host_check(
+            info.get("LiveRestoreEnabled") is True,
+            "host.live-restore",
+            "Live restore is enabled.",
+            "Live restore is not enabled; daemon restarts stop containers (HOST-005).",
+        ),
+    ]
+    socket_check = _check_socket_exposure(rootless=rootless)
+    if socket_check is not None:
+        checks.append(socket_check)
     return checks
 
 
+def _host_check(ok: bool, name: str, pass_message: str, warn_message: str) -> PreflightCheck:
+    return PreflightCheck("PASS" if ok else "WARN", name, pass_message if ok else warn_message)
+
+
+def _uses_non_default_context() -> bool:
+    """True when the Docker CLI targets a named context (possibly a remote daemon)."""
+    context = os.environ.get("DOCKER_CONTEXT", "").strip()
+    if not context:
+        config_dir = Path(os.environ.get("DOCKER_CONFIG") or Path.home() / ".docker")
+        try:
+            config = json.loads((config_dir / "config.json").read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return False
+        context = str(config.get("currentContext", "")).strip() if isinstance(config, dict) else ""
+    return context not in {"", "default"}
+
+
+def _check_socket_exposure(rootless: bool = False) -> PreflightCheck | None:
+    """Return None when the socket cannot be meaningfully inspected."""
+    docker_host = os.environ.get("DOCKER_HOST", "").strip()
+    if not docker_host and _uses_non_default_context():
+        return None
+    scheme = docker_host.lower().split("://", 1)[0] if "://" in docker_host else ""
+    if scheme in {"tcp", "http"}:
+        # Docker treats any non-empty DOCKER_TLS_VERIFY value as enabled.
+        if os.environ.get("DOCKER_TLS_VERIFY", "").strip():
+            return PreflightCheck(
+                "PASS", "host.socket", "Daemon is reached over TCP with TLS verification."
+            )
+        return PreflightCheck(
+            "WARN",
+            "host.socket",
+            "DOCKER_HOST uses TCP without DOCKER_TLS_VERIFY; "
+            "an unauthenticated daemon socket grants root on the host (HOST-006).",
+        )
+    if scheme == "ssh":
+        return PreflightCheck("PASS", "host.socket", "Daemon is reached over SSH.")
+    if scheme not in {"", "unix"} or os.name == "nt":
+        return None
+
+    default_socket = "/var/run/docker.sock"
+    runtime_dir = os.environ.get("XDG_RUNTIME_DIR", "").strip()
+    if rootless and runtime_dir:
+        default_socket = str(Path(runtime_dir) / "docker.sock")
+    socket_path = Path(docker_host.removeprefix("unix://") or default_socket)
+    try:
+        mode = socket_path.stat().st_mode
+    except OSError:
+        return None
+    if mode & stat.S_IWOTH:
+        return PreflightCheck(
+            "WARN",
+            "host.socket",
+            f"Daemon socket {socket_path} is world-writable (HOST-006).",
+        )
+    return PreflightCheck("PASS", "host.socket", "Daemon socket is not world-writable.")
+
+
 def _run_runtime_command(
     command: list[str],
     name: str,

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/renderer.py

@@ -16,6 +16,11 @@ import yaml
 from .constants import MAX_NESTING_DEPTH, MaxNestingDepthExceeded
 from .diagnostics import Diagnostic
 from .loader import resolve_module_dir
+from .secret_leak import (
+    check_generated_text,
+    declared_secret_env_names,
+    resolved_secret_values,
+)
 from .utils import _atomic_write
 
 
@@ -30,12 +35,12 @@ def render_compose(
     Args:
         plan:        Composition plan.
         output_path: Optional output file path.
-        env_file:    Reserved for compatibility; not used.
+        env_file:    .env path used to look up resolved secret values so the
+                     output file is never written if it would contain one.
 
     Returns:
         Tuple of (output_yaml, diagnostics).
     """
-    _ = env_file
     diagnostics: list[Diagnostic] = []
     secrets = plan.get("secrets", {})
     compose_dir = Path(output_path).resolve().parent if output_path else Path.cwd().resolve()
@@ -146,9 +151,17 @@ def render_compose(
     diagnostics.extend(_check_unresolved_expressions(output))
 
     if output_path:
-        path = Path(output_path)
-        path.parent.mkdir(parents=True, exist_ok=True)
-        _atomic_write(path, output)
+        # CDS-SEC-030: scan before writing so a leaking file never lands on disk.
+        leak_diags = check_generated_text(
+            output,
+            output_path,
+            resolved_secret_values(declared_secret_env_names(plan), env_file),
+        )
+        diagnostics.extend(leak_diags)
+        if not leak_diags:
+            path = Path(output_path)
+            path.parent.mkdir(parents=True, exist_ok=True)
+            _atomic_write(path, output)
 
     return output, diagnostics
 

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/resources/rule-schema.json

@@ -77,6 +77,7 @@
         "service-env",
         "runtime",
         "rendered-compose",
+        "file-permissions",
         "none"
       ]
     },

{composable_data_stack-0.12.0 → composable_data_stack-0.13.0}/cli/resources/rule-set.json

@@ -156,8 +156,8 @@
     },
     {
       "id": "CDS-SEC-006",
-      "$comment": "Deferred: this rule is about secrets leaking into rendered Compose, generated artifacts, or CLI text output after resolution. The current scanner only flattens profile inputs and loaded .env secrets, so enabling a non-'none' scope here would still never inspect the output surfaces this rule names.",
-      "title": "Resolved secret present in generated or CLI output",
+      "$comment": "Enforced entirely in code by _check_rendered_secret_values in cli/security.py, not the declarative match engine: it looks for the literal value of every known secret (declared under spec.secrets.values or a secret-named CDS_* variable of at least 8 characters, per cli/secret_leak.py) anywhere in the rendered Compose model. CDS only ever renders ${CDS_*} placeholders, so a literal value means a secret was copied into config. The match block below is therefore documentation only. This rule was split by #356: written files are CDS-SEC-030 and printed CLI output is CDS-SEC-071. `enabled` stays false because `scope: [\"none\"]` rules can never produce a declarative-engine finding (see the scope-none `enabled` invariant test); `codeEnforced` is the actual on/off switch.",
+      "title": "Resolved secret present in rendered Compose output",
       "severity": "high",
       "category": "secrets",
       "complianceCategory": "secrets-management",
@@ -173,13 +173,14 @@
           }
         ]
       },
-      "message": "Rendered or planned output appears to contain a resolved secret",
+      "message": "Rendered Compose output contains a resolved secret value",
       "whyItMatters": "Generated artifacts are often stored, shared, or captured by CI systems.",
       "recommendation": [
         "Redact secrets in plan and render output.",
         "Avoid writing resolved secret values to generated files."
       ],
-      "enabled": false
+      "enabled": false,
+      "codeEnforced": true
     },
     {
       "id": "CDS-SEC-010",
@@ -410,7 +411,7 @@
     },
     {
       "id": "CDS-SEC-030",
-      "$comment": "Deferred: this rule needs an on-disk generated-file scan after render/write. The current matcher only sees profile values and loaded .env secrets, not file contents created by cds render or cds up.",
+      "$comment": "Enforced entirely in code by cli.secret_leak.check_generated_text, called from render_compose (docker-compose.yml) and render_helm (every chart file) before anything is written: if the content contains the literal value of a known secret (same value set as CDS-SEC-006) the file is not written and error E096 is reported, so cds render and cds up fail closed. `enabled` stays false because `scope: [\"none\"]` rules can never produce a declarative-engine finding; `codeEnforced` is the actual on/off switch. The match block below is documentation only.",
       "title": "Generated file contains resolved secrets",
       "severity": "high",
       "category": "file-output-safety",
@@ -433,7 +434,8 @@
         "Redact generated output or write secret references only.",
         "Keep rendered artifacts free of resolved secrets."
       ],
-      "enabled": false
+      "enabled": false,
+      "codeEnforced": true
     },
     {
       "id": "CDS-SEC-031",
@@ -463,12 +465,12 @@
     },
     {
       "id": "CDS-SEC-032",
-      "$comment": "Deferred: the rule schema exposes a permissions condition, but cli/security.py does not evaluate it and the scanner does not stat files on disk. Supporting this needs file metadata collection plus condition evaluation for actual mode bits.",
+      "$comment": "Evaluated against the \"file-permissions\" scope: _flatten_file_permissions in cli/security.py stats the .env file the scan loads secrets from and emits its octal mode (e.g. \"0644\") as a synthetic <path>.permissions item; the \"permissions\" condition then flags any group (\"group-readable\") or other (\"world-readable\") read/write/execute bit, i.e. anything broader than owner-only. POSIX only: on Windows (os.name == \"nt\") no item is emitted, so the rule never fires there rather than reporting meaningless modes. A missing .env emits nothing. Secret files CDS itself writes (cds init) are created owner-only (0600). Filesystems without POSIX modes (e.g. WSL /mnt/c) report 0777 and always trigger the rule.",
       "title": "Secret file permissions too broad",
       "severity": "medium",
       "category": "file-output-safety",
       "complianceCategory": "secrets-management",
-      "scope": ["none"],
+      "scope": ["file-permissions"],
       "match": {
         "any": [
           {
@@ -485,7 +487,7 @@
         "Set permissions to owner-only for files containing secrets.",
         "Tighten file creation umask for generated secret artifacts."
       ],
-      "enabled": false
+      "enabled": true
     },
     {
       "id": "CDS-SEC-033",
@@ -755,7 +757,7 @@
     },
     {
       "id": "CDS-SEC-071",
-      "$comment": "Deferred: this rule is about literal CLI plan/validate output text, which is produced after formatting findings/diagnostics. The current matcher has no capture of stdout/stderr or rendered plan text to scan.",
+      "$comment": "Enforced entirely in code: cds plan, validate, test and security wrap stdout/stderr in cli.secret_leak.RedactingStream, which replaces the literal value of any known secret (same value set as CDS-SEC-006) with ***REDACTED*** before it is printed and then warns (W102) on stderr that redaction happened. Passing --reveal-secrets (where offered) disables the wrapper. `enabled` stays false because `scope: [\"none\"]` rules can never produce a declarative-engine finding; `codeEnforced` is the actual on/off switch. The match block below is documentation only.",
       "title": "CLI plan or validate output prints secrets",
       "severity": "high",
       "category": "observability-leakage",
@@ -781,7 +783,8 @@
         "Redact sensitive output by default.",
         "Provide explicit unsafe flags for debugging only."
       ],
-      "enabled": false
+      "enabled": false,
+      "codeEnforced": true
     },
     {
       "id": "CDS-SEC-072",