composable-data-stack 0.4.0__py3-none-any.whl

cli/security_common.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+
+SEVERITY_ORDER = {"high": 0, "medium": 1, "low": 2}
+
+SECRET_KEY_RE = re.compile(r"(?i)(password|secret|token|key|credential|passwd|pwd)")
+
+_SECRET_KEY_WORDS = (
+    "password",
+    "passwd",
+    "pass",
+    "pwd",
+    "secret",
+    "token",
+    "key",
+    "credential",
+    "apikey",
+    "accesskey",
+    "secretkey",
+    "passphrase",
+)
+
+SECRET_KEY_SEGMENT_RE = re.compile(
+    r"(?i)(?:^|[-_])(?:" + "|".join(_SECRET_KEY_WORDS) + r")(?:$|[^a-z0-9])"
+)
+
+ENVIRONMENT_TO_CLASS = {
+    "local": "local",
+    "development": "dev",
+    "staging": "staging",
+    "production": "prod",
+}
+
+
+def infer_profile_class(profile: dict[str, Any]) -> str:
+    """Map a profile's declared environment to the security policy class."""
+    environment = (profile or {}).get("metadata", {}).get("environment", "local")
+    return ENVIRONMENT_TO_CLASS.get(environment, "local")

cli/state.py

@@ -0,0 +1,112 @@
+"""
+Groups `docker compose ps -a --format json` output by health state for
+`cds state`. Provider-neutral: reads only the generic fields Compose
+itself emits (Service/Name, Health, State), no module- or
+service-specific knowledge.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+def parse_compose_ps_json(raw_output: str) -> list[dict[str, Any]]:
+    """
+    Parses `docker compose ps --format json` output in either shape.
+    Returns [] for blank output. Skips lines that aren't valid JSON
+    objects rather than failing the whole parse on one bad line.
+    """
+    text = raw_output.strip()
+    if not text:
+        return []
+
+    try:
+        parsed = json.loads(text)
+        if isinstance(parsed, list):
+            return [entry for entry in parsed if isinstance(entry, dict)]
+        if isinstance(parsed, dict):
+            return [parsed]
+    except json.JSONDecodeError:
+        pass
+
+    services: list[dict[str, Any]] = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            entry = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(entry, dict):
+            services.append(entry)
+    return services
+
+
+def _bucket_for(service: dict[str, Any]) -> str:
+    health = str(service.get("Health") or "").strip()
+    if health:
+        return health.upper()
+
+    state = str(service.get("State") or "").strip().lower()
+    if state == "exited":
+        exit_code_raw = service.get("ExitCode")
+        try:
+            exit_code = int(exit_code_raw)
+        except (TypeError, ValueError):
+            return "UNKNOWN"
+        return "HEALTHY EXIT" if exit_code == 0 else "UNHEALTHY EXIT"
+    if state == "running":
+        return "RUNNING"
+
+    # Paused/Dead/etc. aren't states normal cds workflows produce, no
+    # dedicated bucket for them, they fall in with everything else unknown.
+    return "UNKNOWN"
+
+
+def group_services_by_health(services: list[dict[str, Any]]) -> dict[str, list[str]]:
+    """
+    Groups parsed compose-ps entries into {bucket_name: [service_name, ...]},
+    both deterministically sorted. Bucket is Health when the service has a
+    healthcheck ("starting"/"healthy"/"unhealthy"); else "healthy exit"/
+    "unhealthy exit" by ExitCode when State is "exited"; else "running";
+    else "UNKNOWN" (covers Paused/Dead/etc., which normal cds workflows
+    don't produce).
+    """
+    buckets: dict[str, set[str]] = {}
+    for service in services:
+        name = str(service.get("Service") or service.get("Name") or "").strip()
+        if not name:
+            continue
+        bucket = _bucket_for(service)
+        buckets.setdefault(bucket, set()).add(name)
+
+    return {bucket: sorted(names) for bucket, names in sorted(buckets.items())}
+
+
+_COLORS = {
+    "HEALTHY": "\033[32m",
+    "RUNNING": "\033[32m",
+    "UNHEALTHY": "\033[31m",
+    "HEALTHY EXIT": "\033[92m",
+    "UNHEALTHY EXIT": "\033[91m",
+    "STARTING": "\033[33m",
+    "UNKNOWN": "\033[2m",
+}
+_RESET = "\033[0m"
+
+
+def format_state_output(grouped: dict[str, list[str]], use_color: bool = False) -> str:
+    if not grouped:
+        return "No services found."
+
+    lines = []
+    for bucket, names in grouped.items():
+        label = f"{bucket}:"
+        if use_color:
+            color = _COLORS.get(bucket, "")
+            label = f"{color}{label}{_RESET}" if color else label
+        lines.append(label)
+        for name in names:
+            lines.append(f"  - {name}")
+    return "\n".join(lines)

cli/up_runner.py

@@ -0,0 +1,257 @@
+# cli/up_runner.py
+from __future__ import annotations
+
+import re
+import subprocess  # nosec B404
+import sys
+import time
+from collections import Counter
+from collections.abc import Callable
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import IO
+
+from .state import format_state_output, group_services_by_health, parse_compose_ps_json
+
+_BUILD_SERVICE_RE = re.compile(r"^([\w][\w.-]*?):\s+Building\b")
+
+DEFAULT_POLL_INTERVAL_SECONDS = 2.0
+DEFAULT_TIMEOUT_SECONDS = 180.0
+
+_SETTLED_BUCKETS = {"HEALTHY", "RUNNING", "HEALTHY EXIT", "UNHEALTHY EXIT", "UNHEALTHY"}
+_FAILURE_BUCKETS = {"UNHEALTHY", "UNHEALTHY EXIT"}
+
+
+def default_log_path(profile_name: str, logs_dir: Path | None = None) -> Path:
+    """
+    Default log path for a `cds up` run: `.cds/logs/up-<profile>-<UTC
+    timestamp>.log`, relative to the current working directory. Slashes
+    and spaces in `profile_name` (profiles can be passed as paths) are
+    flattened so the result is always a single valid filename.
+
+    `logs_dir` is injectable so tests don't have to write into a real
+    `.cds/logs` under the repo checkout.
+    """
+    base = logs_dir if logs_dir is not None else Path(".cds") / "logs"
+    timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    safe_profile = profile_name.strip().replace("/", "-").replace("\\", "-").replace(" ", "-") or "profile"
+    return base / f"up-{safe_profile}-{timestamp}.log"
+
+
+def run_streamed(
+    cmd: list[str],
+    log_file: IO[str],
+    echo: bool = True,
+    group_by_image: bool = False,
+    service_to_image: dict[str, str] | None = None,
+    use_color: bool = True,
+) -> int:
+    """
+    Runs `cmd` with stdout+stderr merged, writing each line to
+    `log_file` as it arrives (flushed immediately, so `tail -f` on the
+    log file works while the command is still running) and, if `echo`,
+    to this process's stdout too.
+
+    When `group_by_image` is True and `echo` is True, the output is
+    annotated with section headers that identify which Docker Compose
+    service each build phase belongs to.  The log file always receives
+    the raw, un-annotated output.
+
+    Section headers are colored only when `use_color` is True and
+    stdout is a TTY, matching the ANSI handling in `_default_redraw`
+    (e.g. CI logs and redirected output stay plain).
+
+    Returns the command's exit code. Raises FileNotFoundError if
+    `cmd[0]` isn't on PATH, same as subprocess.run.
+    """
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1)  # nosec B603
+    if process.stdout is None:
+        raise RuntimeError("subprocess.Popen returned no stdout despite stdout=PIPE")
+    try:
+        current_group: str | None = None
+        seen_groups: set[str] = set()
+        image_container_counts = Counter((service_to_image or {}).values())
+        for line in process.stdout:
+            log_file.write(line)
+            log_file.flush()
+            if echo:
+                if group_by_image:
+                    m = _BUILD_SERVICE_RE.match(line)
+                    if m:
+                        service = m.group(1)
+                        image = (service_to_image or {}).get(service, service)
+                        if image != current_group:
+                            current_group = image
+                            if image not in seen_groups:
+                                seen_groups.add(image)
+                                count = image_container_counts.get(image, 1)
+                                label = f"Building {image}"
+                                if count > 1:
+                                    label += f" ({count} containers)"
+                                header = f"── {label} "
+                                header += "─" * max(1, 60 - len(header))
+                                if use_color and sys.stdout.isatty():
+                                    sys.stdout.write(f"\n\033[36m{header}\033[0m\n")
+                                else:
+                                    sys.stdout.write(f"\n{header}\n")
+                sys.stdout.write(line)
+                sys.stdout.flush()
+    finally:
+        process.stdout.close()
+    return process.wait()
+
+
+def start_log_tail(compose_path: str, log_file: IO[str]) -> subprocess.Popen:
+    """
+    Starts `docker compose logs -f` in the background, piped only to
+    `log_file` (not the terminal; the terminal is showing the live
+    state view while this runs). Caller is responsible for stopping it
+    with `stop_log_tail` once the stack settles or `cds up` exits.
+    """
+    logs_cmd = ["docker", "compose", "-f", compose_path, "logs", "-f", "--no-color"]
+    return subprocess.Popen(logs_cmd, stdout=log_file, stderr=subprocess.STDOUT, text=True)  # nosec B603
+
+
+def start_up_in_background(cmd: list[str], log_file: IO[str]) -> subprocess.Popen:
+    """
+    Starts `docker compose up --detach` in the background, piped only to
+    `log_file`. Unlike `run_streamed`, this does not block: `docker
+    compose up` can itself block for a long time waiting on
+    healthcheck-gated `depends_on` dependencies to become healthy before
+    it returns, even though `--detach` is passed. Running it in the
+    background lets the live state view (which polls `docker compose ps`
+    directly) start rendering immediately instead of waiting for `up` to
+    finish. Caller is responsible for reaping it with `process.wait()`.
+    """
+    return subprocess.Popen(cmd, stdout=log_file, stderr=subprocess.STDOUT, text=True)  # nosec B603
+
+
+def stop_log_tail(process: subprocess.Popen, timeout: float = 5.0) -> None:
+    """Terminates a background log-tail process started by start_log_tail."""
+    if process.poll() is not None:
+        return
+    process.terminate()
+    try:
+        process.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        process.wait()
+
+
+def _is_settled(grouped: dict[str, list[str]], expected_service_count: int | None) -> bool:
+    if not grouped:
+        return expected_service_count == 0
+
+    pending = sum(len(names) for bucket, names in grouped.items() if bucket not in _SETTLED_BUCKETS)
+    if pending:
+        return False
+    if expected_service_count is not None:
+        seen = sum(len(names) for names in grouped.values())
+        if seen < expected_service_count:
+            return False
+    return True
+
+
+def _default_redraw(text: str) -> None:
+    if sys.stdout.isatty():
+        sys.stdout.write("\033[2J\033[H")
+        sys.stdout.write(text + "\n")
+    else:
+        print(text)
+    sys.stdout.flush()
+
+
+def poll_state_until_settled(
+    compose_path: str,
+    *,
+    expected_service_count: int | None = None,
+    poll_interval: float = DEFAULT_POLL_INTERVAL_SECONDS,
+    timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    use_color: bool = False,
+    sleep_fn: Callable[[float], None] = time.sleep,
+    now_fn: Callable[[], float] = time.monotonic,
+    ps_fn: Callable[[], subprocess.CompletedProcess] | None = None,
+    redraw_fn: Callable[[str], None] | None = None,
+    up_done_fn: Callable[[], int | None] | None = None,
+    on_up_finished: Callable[[int], None] | None = None,
+) -> tuple[bool, dict[str, list[str]]]:
+    """
+    Polls `docker compose ps -a --format json` every `poll_interval`
+    seconds, redrawing the grouped `cds state` view each time, until
+    every service `docker compose ps` reports is in a terminal bucket
+    (HEALTHY, RUNNING, HEALTHY EXIT, UNHEALTHY EXIT, or UNHEALTHY) or
+    `timeout` seconds elapse.
+
+    Returns `(settled, grouped)`. `settled` is False if the loop timed
+    out, or if any service ended in UNHEALTHY / UNHEALTHY EXIT.
+
+    `ps_fn`, `sleep_fn`, `now_fn`, and `redraw_fn` are injectable so this
+    can be unit tested with a fake clock and canned `ps` output instead
+    of real Docker calls and real sleeping.
+
+    `up_done_fn`, if given, is polled once per iteration and must return
+    `None` while `docker compose up` is still running, or its exit code
+    once it has finished. This lets the caller run `up` in the
+    background while this loop redraws the live view immediately,
+    without either process blocking the other:
+
+    - If `up` exits non-zero, this returns `(False, grouped)` right
+      away instead of burning through the full `timeout`, since
+      services that `up` never started will never settle.
+    - The `timeout` clock only starts once `up` finishes successfully,
+      so a stack whose healthchecks legitimately outlast `timeout`
+      isn't penalized for time `up` itself spent blocked on
+      healthcheck-gated `depends_on` dependencies. Omitting `up_done_fn`
+      preserves the old behavior of starting the clock immediately.
+
+    `on_up_finished`, if given, is called exactly once, the first time
+    `up_done_fn` reports a successful result (exit code 0), so callers can
+    defer setup (e.g. starting a log tail) until `up` is done rather
+    than running it concurrently with `up`'s own output.
+    """
+    if ps_fn is None:
+        def ps_fn() -> subprocess.CompletedProcess:
+            ps_cmd = ["docker", "compose", "-f", compose_path, "ps", "-a", "--format", "json"]
+            return subprocess.run(ps_cmd, capture_output=True, text=True)  # nosec B603
+
+    if redraw_fn is None:
+        redraw_fn = _default_redraw
+
+    if up_done_fn is None:
+        # No background `up` process to track: behave as if it had
+        # already finished successfully, so the timeout clock starts
+        # immediately (matches the pre-existing behavior).
+        def up_done_fn() -> int | None:
+            return 0
+
+    start: float | None = None
+    up_finished_seen = False
+    grouped: dict[str, list[str]] = {}
+    while True:
+        ps_result = ps_fn()
+        services = parse_compose_ps_json(ps_result.stdout) if ps_result.returncode == 0 else []
+        grouped = group_services_by_health(services)
+        redraw_fn(format_state_output(grouped, use_color=use_color))
+
+        if _is_settled(grouped, expected_service_count):
+            has_failure = any(bucket in _FAILURE_BUCKETS and names for bucket, names in grouped.items())
+            return (not has_failure), grouped
+
+        up_exit_code = up_done_fn()
+        if up_exit_code is not None:
+            if not up_finished_seen:
+                up_finished_seen = True
+                if up_exit_code == 0 and on_up_finished is not None:
+                    on_up_finished(up_exit_code)
+            if up_exit_code != 0:
+                # `up` itself already failed; services it never started
+                # will never settle, so don't wait out the full timeout.
+                return False, grouped
+            if start is None:
+                start = now_fn()
+
+        if start is not None and now_fn() - start >= timeout:
+            return False, grouped
+
+        sleep_fn(poll_interval)
+