composable-data-stack 0.5.1__tar.gz → 0.6.0__tar.gz

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: composable-data-stack
-Version: 0.5.1
+Version: 0.6.0
 Summary: CLI for validating and composing composable data stack profiles
 Author: Ronald Hensbergen
 License-Expression: Apache-2.0

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/getter.py

@@ -5,6 +5,7 @@ import os
 import re
 import shlex
 import shutil
+import sys
 import tarfile
 import tempfile
 from collections.abc import Iterator
@@ -19,7 +20,6 @@ from urllib.request import Request, urlopen
 from .loader import load_yaml_file, resolve_module_dir
 from .planner import MaxNestingDepthExceeded, apply_defaults, substitute_string
 
-
 # The upstream repository `cds get` downloads from when no `--remote` is
 # given. Keep in sync with the `Repository` URL in pyproject.toml.
 DEFAULT_REMOTE = "RonaldHensbergen/composable-data-stack"
@@ -91,9 +91,7 @@ def fetch_profile(
                 f"{rendered}{extra}"
             )
 
-        for action in actions:
-            action.destination.parent.mkdir(parents=True, exist_ok=True)
-            shutil.copy2(action.source, action.destination)
+        _write_actions(actions)
 
         _write_tracking_manifest(
             target_root=target_root,
@@ -168,7 +166,7 @@ def _download_github_repository(owner: str, repo: str, ref: str, work_dir: Path)
     url = f"https://api.github.com/repos/{owner}/{repo}/tarball/{ref}"
     request = Request(url, headers={"User-Agent": "composable-data-stack-cds-get"})
     try:
-        with urlopen(request, timeout=30) as response:  # nosec B310 - fixed https GitHub API host
+        with urlopen(request, timeout=30) as response:  # nosec B310 - fixed https GitHub API host  # noqa: S310
             archive_bytes = response.read()
     except HTTPError as exc:
         raise GetError(
@@ -598,7 +596,14 @@ def _add_copy_action(
     actions_by_destination: dict[Path, CopyAction],
 ) -> None:
     repo_relative = source_file.resolve().relative_to(source_repo.resolve())
-    destination = (destination_root / repo_relative).resolve()
+    # Deliberately do NOT call .resolve() on the combined destination path:
+    # destination_root is already an absolute, resolved path (see
+    # fetch_profile()), and resolving the full path here would follow a
+    # symlink planted at the destination leaf, silently swapping the
+    # CopyAction's destination for the symlink's target instead of the
+    # symlink path itself. That would defeat the is_symlink() conflict/
+    # write-through guards in _find_conflicts()/_write_actions() (#474).
+    destination = destination_root / repo_relative
     existing = actions_by_destination.get(destination)
     if existing is None:
         actions_by_destination[destination] = CopyAction(
@@ -617,6 +622,14 @@ def _find_conflicts(actions: list[CopyAction]) -> list[str]:
     conflicts: list[str] = []
     for action in actions:
         destination = action.destination
+        if destination.is_symlink():
+            # A symlink destination is always a conflict, even if it's
+            # dangling (Path.exists() follows symlinks and returns False for
+            # a broken link, which would otherwise let a pre-planted symlink
+            # silently bypass this check). Never write through a symlink:
+            # see _write_actions().
+            conflicts.append(action.repo_relative_path)
+            continue
         if not destination.exists():
             continue
         if destination.is_dir():
@@ -627,6 +640,21 @@ def _find_conflicts(actions: list[CopyAction]) -> list[str]:
     return conflicts
 
 
+def _write_actions(actions: list[CopyAction]) -> None:
+    """Copy planned actions to their destinations, refusing to write through
+    a symlinked destination even when --force allowed the conflict check to
+    pass. Any pre-existing symlink at a destination path (dangling or not)
+    is removed first so shutil.copy2() always creates/overwrites a regular
+    file there instead of following the link to some other location on disk
+    (e.g. a symlink planted at profiles/foo/profile.yaml pointing outside
+    the destination tree)."""
+    for action in actions:
+        action.destination.parent.mkdir(parents=True, exist_ok=True)
+        if action.destination.is_symlink():
+            action.destination.unlink()
+        shutil.copy2(action.source, action.destination)
+
+
 def _write_tracking_manifest(
     *,
     target_root: Path,
@@ -661,21 +689,59 @@ def _write_tracking_manifest(
     profiles[requested_profile] = entry
 
     manifest_path.parent.mkdir(parents=True, exist_ok=True)
+    if manifest_path.is_symlink():
+        # Same write-through-symlink guard as _write_actions(): never follow
+        # a symlink planted at the tracking manifest's path.
+        manifest_path.unlink()
     manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
 
 
 def _read_tracking_manifest(path: Path) -> dict[str, Any]:
     if not path.exists():
         return {"version": 1, "profiles": {}}
+
     try:
-        data = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        return {"version": 1, "profiles": {}}
-    if not isinstance(data, dict):
+        raw_text = path.read_text(encoding="utf-8")
+    except OSError as exc:
+        # The file could not be read at all, so it cannot be backed up either;
+        # skip the doomed copy attempt and report the read failure directly.
+        print(
+            f"WARNING {path} could not be read ({exc}); resetting tracking manifest.",
+            file=sys.stderr,
+        )
         return {"version": 1, "profiles": {}}
-    data.setdefault("version", 1)
-    data.setdefault("profiles", {})
-    return data
+
+    malformed_reason: str = ""
+    try:
+        data = json.loads(raw_text)
+    except json.JSONDecodeError as exc:
+        malformed_reason = f"invalid JSON: {exc}"
+    else:
+        if not isinstance(data, dict):
+            malformed_reason = "manifest root must be a JSON object"
+        else:
+            data.setdefault("version", 1)
+            data.setdefault("profiles", {})
+            return data
+
+    _backup_malformed_manifest(path, malformed_reason)
+    return {"version": 1, "profiles": {}}
+
+
+def _backup_malformed_manifest(path: Path, reason: str) -> None:
+    timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ")
+    backup_path = path.with_name(f"{path.name}.corrupt-{timestamp}")
+    try:
+        shutil.copy2(path, backup_path)
+        backup_message = f"backed up to {backup_path}"
+    except OSError as exc:
+        backup_message = f"backup failed: {exc}"
+
+    print(
+        f"WARNING {path} is malformed ({reason}); resetting tracking manifest "
+        f"({backup_message}).",
+        file=sys.stderr,
+    )
 
 
 def _asset_root_relative_path(asset_root: Path, source_repo: Path) -> str:

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/image_updates.py

@@ -239,10 +239,10 @@ def fetch_dockerhub_tags(namespace: str, repository: str, page_size: int = 100,
 
     while url and pages_read < page_limit:
         try:
-            req = Request(url, headers={"User-Agent": "cds-image-check/1.0"})
+            req = Request(url, headers={"User-Agent": "cds-image-check/1.0"})  # noqa: S310
             # url is always http(s), built from the hardcoded DOCKER_HUB_API
             # constant or the scheme-validated pagination url below.
-            with urlopen(req, timeout=10) as response:  # nosec B310
+            with urlopen(req, timeout=10) as response:  # nosec B310  # noqa: S310
                 data = json.loads(response.read().decode())
         except (HTTPError, URLError, ValueError):
             return None

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/image_verification.py

@@ -246,7 +246,7 @@ def _verify_with_cosign(
     command.append(image_ref)
 
     try:
-        result = subprocess.run(command, capture_output=True, text=True, timeout=120)  # nosec B603
+        result = subprocess.run(command, capture_output=True, text=True, timeout=120)  # nosec B603  # noqa: S603
     except (OSError, subprocess.TimeoutExpired) as exc:
         return False, f"cosign invocation failed: {exc}"
 

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/loader.py

@@ -8,7 +8,6 @@ import yaml
 
 from .diagnostics import Diagnostic
 
-
 _MODULE_ROOT_MARKERS = {"modules", "modules-experimental"}
 
 

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/main.py

@@ -7,9 +7,8 @@ import os
 import re
 import subprocess  # nosec B404
 import sys
-import tempfile
-from contextlib import suppress
-from importlib.metadata import PackageNotFoundError, version as _package_version
+from importlib.metadata import PackageNotFoundError
+from importlib.metadata import version as _package_version
 from pathlib import Path
 from typing import Any
 
@@ -18,22 +17,24 @@ try:
 except ImportError:
     argcomplete = None
 
-from .validator import has_errors, validate_profile
+import yaml
+
 from .diagnostics import Diagnostic
-from .utils import _atomic_write
-from .planner import build_plan
-from .renderer import render_compose
-from .image_updates import collect_module_images, check_image_update
-from .overlay import resolve_profile
-from .preflight import preflight_passed, run_preflight
-from .security import PrecomputedRender, run_security_validation
-from .security_common import SEVERITY_ORDER, infer_profile_class
+from .getter import GetError, fetch_profile, format_get_plan
+from .image_updates import check_image_update, collect_module_images
 from .image_verification import (
     ImagePolicy,
     default_fixture_path,
     load_policy_from_env,
     verify_images,
 )
+from .loader import load_yaml_file
+from .overlay import resolve_profile
+from .planner import build_plan
+from .preflight import preflight_passed, run_preflight
+from .renderer import render_compose
+from .security import PrecomputedRender, run_security_validation
+from .security_common import SEVERITY_ORDER, infer_profile_class
 from .state import format_state_output, group_services_by_health, parse_compose_ps_json
 from .up_runner import (
     DEFAULT_TIMEOUT_SECONDS,
@@ -44,9 +45,8 @@ from .up_runner import (
     start_up_in_background,
     stop_log_tail,
 )
-from .loader import load_yaml_file
-from .getter import GetError, fetch_profile, format_get_plan
-import yaml
+from .utils import _atomic_write
+from .validator import has_errors, validate_profile
 
 
 def load_env_file(env_file: str = ".env") -> None:
@@ -114,12 +114,19 @@ def get_modules_root() -> Path:
 def find_project_root(start: Path | None = None) -> Path:
     """
     Walk up from `start` (default: current working directory) looking for a
-    project root marker (pyproject.toml or .git). Falls back to `start` itself
-    if no marker is found.
+    project root marker: `.cds` (CDS's own state directory, created by
+    `cds get`/`cds use`), `pyproject.toml`, or `.git`. `.cds` is checked first
+    at each level so a CDS working directory takes priority over an unrelated
+    ancestor repository (e.g. a dotfiles repo at $HOME) that happens to sit
+    further up the tree. Falls back to `start` itself if no marker is found.
     """
     current = (start or Path.cwd()).resolve()
     for directory in [current, *current.parents]:
-        if (directory / "pyproject.toml").exists() or (directory / ".git").exists():
+        if (
+            (directory / ".cds").exists()
+            or (directory / "pyproject.toml").exists()
+            or (directory / ".git").exists()
+        ):
             return directory
     return current
 
@@ -335,13 +342,19 @@ def resolve_project_root(profile_path: str) -> Path:
     """
     Resolve a project root for output artifacts.
 
-    The resolver walks up from the selected profile location and picks the first
-    directory containing either pyproject.toml or .git. If no marker is found,
-    it falls back to the current working directory.
+    The resolver walks up from the selected profile location and picks the
+    first directory containing `.cds` (CDS's own state directory, checked
+    first so it takes priority over an unrelated ancestor repository),
+    `pyproject.toml`, or `.git`. If no marker is found, it falls back to the
+    current working directory.
     """
     start = Path(profile_path).resolve().parent
     for directory in [start, *start.parents]:
-        if (directory / "pyproject.toml").exists() or (directory / ".git").exists():
+        if (
+            (directory / ".cds").exists()
+            or (directory / "pyproject.toml").exists()
+            or (directory / ".git").exists()
+        ):
             return directory
     return Path.cwd().resolve()
 
@@ -432,6 +445,19 @@ def _add_environment_arg(subparser: argparse.ArgumentParser) -> None:
     )
 
 
+def _add_hardened_arg(subparser: argparse.ArgumentParser) -> None:
+    subparser.add_argument(
+        "--hardened",
+        action="store_true",
+        help=(
+            "Override config.image.variant to the Alpine-hardened build for any "
+            "module whose configSchema exposes an image.variant option (currently "
+            "only modules/orchestration/dagster), without editing the profile YAML. "
+            "Modules without that config option are left unchanged."
+        ),
+    )
+
+
 def _collect_profile_env_vars(
     profile_path: str, environment: str | None = None
 ) -> tuple[list[str], set[str]]:
@@ -756,6 +782,7 @@ def main() -> int:
         "-o",
         help="Output file path for rendered output (default: <project-root>/docker-compose.yml)",
     )
+    _add_hardened_arg(render_parser)
 
     up_parser = subparsers.add_parser(
         "up",
@@ -792,6 +819,7 @@ def main() -> int:
         action="store_true",
         help="Disable colored labels in the live state view",
     )
+    _add_hardened_arg(up_parser)
 
     test_parser = subparsers.add_parser(
         "test",
@@ -807,6 +835,7 @@ def main() -> int:
             "By default, values are redacted to avoid echoing real secrets to stdout/CI logs."
         ),
     )
+    _add_hardened_arg(test_parser)
 
     preflight_parser = subparsers.add_parser(
         "preflight",
@@ -1091,7 +1120,9 @@ def main() -> int:
                 return 1
 
             env_file = str(resolve_env_file_path(profile_path))
-            plan, plan_diags = build_plan(profile_path, env_file=env_file, environment=args.environment)
+            plan, plan_diags = build_plan(
+                profile_path, env_file=env_file, environment=args.environment, hardened=args.hardened
+            )
             all_diags = diagnostics + plan_diags
             if has_errors(all_diags):
                 print_diagnostics(all_diags)
@@ -1128,7 +1159,9 @@ def main() -> int:
             return 1
 
         env_file = str(resolve_env_file_path(profile_path))
-        plan, plan_diags = build_plan(profile_path, env_file=env_file, environment=args.environment)
+        plan, plan_diags = build_plan(
+            profile_path, env_file=env_file, environment=args.environment, hardened=args.hardened
+        )
         all_diags = diagnostics + plan_diags
         if has_errors(all_diags):
             print_diagnostics(all_diags)
@@ -1343,7 +1376,9 @@ def main() -> int:
         render_diags: list[Diagnostic] = []
         render_ok = False
         if validate_ok:
-            plan, plan_diags = build_plan(profile_path, env_file=env_file, environment=args.environment)
+            plan, plan_diags = build_plan(
+                profile_path, env_file=env_file, environment=args.environment, hardened=args.hardened
+            )
             plan_ok = not has_errors(diagnostics + plan_diags)
             if plan_ok:
                 compose_yaml, render_diags = render_compose(plan, env_file=env_file)
@@ -1468,7 +1503,7 @@ def main() -> int:
 
         ps_cmd = ["docker", "compose", "-f", str(compose_path), "ps", "-a", "--format", "json"]
         try:
-            ps_result = subprocess.run(ps_cmd, capture_output=True, text=True)  # nosec B603
+            ps_result = subprocess.run(ps_cmd, capture_output=True, text=True)  # nosec B603  # noqa: S603
         except FileNotFoundError:
             print("ERROR docker was not found. Install Docker and ensure it is on your PATH.")
             return 1

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/planner.py

@@ -1,23 +1,24 @@
 # cli/planner.py
 from __future__ import annotations
 
+import os
+import re
 from copy import deepcopy
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
-import os
-import re
 
 from .constants import MAX_NESTING_DEPTH, MaxNestingDepthExceeded
 from .diagnostics import Diagnostic
 from .loader import load_yaml_file, resolve_module_file
-from .resolver import is_secret_ref, parse_contract_ref, resolve_path, secret_name_from_ref
+from .resolver import (
+    is_secret_ref,
+    parse_contract_ref,
+    resolve_path,
+    secret_name_from_ref,
+)
 from .secrets import load_profile_secrets
 
-from dataclasses import dataclass
-
-
-
-
 
 @dataclass
 class SecretRef:
@@ -28,6 +29,7 @@ def build_plan(
     profile_path: str,
     env_file: str | None = None,
     environment: str | None = None,
+    hardened: bool = False,
 ) -> tuple[dict[str, Any] | None, list[Diagnostic]]:
     """
     Build a resolved plan from a profile.
@@ -40,6 +42,14 @@ def build_plan(
             overlay is merged over the base profile before planning; see
             cli.overlay.resolve_profile. Value provenance for the merge is
             recorded on the returned plan under "provenance".
+        hardened: When True, overrides `config.image.variant` to "hardened"
+            for any module instance whose configSchema exposes an
+            `image.variant` property (currently only
+            modules/orchestration/dagster), leaving modules without that
+            config option untouched. This is the CLI-level `--hardened`
+            convenience flag (see cli/main.py's up/render/test subcommands)
+            for switching to the Alpine-hardened image variant without
+            hand-editing the profile YAML.
 
     Returns:
         Tuple of (plan, diagnostics)
@@ -150,6 +160,8 @@ def build_plan(
                 module_instance.get("config", {}),
                 module_def.get("spec", {}).get("configSchema", {})
             )
+            if hardened and _supports_image_variant(module_def):
+                normalized_config.setdefault("image", {})["variant"] = "hardened"
         except MaxNestingDepthExceeded:
             diagnostics.append(Diagnostic(
                 level="error",
@@ -253,6 +265,22 @@ def _substitute_config_env_vars(
         return _CDS_VAR_PATTERN.sub(_replace, obj)
     return obj
 
+def _supports_image_variant(module_def: dict[str, Any]) -> bool:
+    """
+    True if a module's configSchema exposes a `image.variant` property
+    (currently only modules/orchestration/dagster), which is what the
+    `--hardened` CLI flag overrides. Modules without this config option
+    (e.g. warehouse/postgres, bi/superset) are left untouched.
+    """
+    image_schema = (
+        module_def.get("spec", {})
+        .get("configSchema", {})
+        .get("properties", {})
+        .get("image", {})
+    )
+    return "variant" in image_schema.get("properties", {})
+
+
 def apply_defaults(config: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]:
     config_copy = deepcopy(config)
     return _apply_schema_defaults(config_copy, schema)
@@ -307,10 +335,13 @@ def _apply_schema_defaults(value: Any, schema: dict[str, Any], _depth: int = 0)
 
     return value
 
-def resolve_provided_contracts(inst: dict[str, Any], secrets: dict[str, str] = {}) -> dict[str, Any]:
+def resolve_provided_contracts(
+    inst: dict[str, Any], secrets: dict[str, str] | None = None
+) -> dict[str, Any]:
     """
     Resolve contracts provided by a module instance.
     """
+    secrets = secrets if secrets is not None else {}
     provides = inst["module"].get("spec", {}).get("provides", [])
     resolved: dict[str, Any] = {}
     service_name = inst["id"]
@@ -340,8 +371,9 @@ def resolve_consumed_contracts(
     modules_by_id: dict[str, dict[str, Any]],
     resolved_contracts_by_module: dict[str, dict[str, Any]],
     diagnostics: list[Diagnostic],
-    secrets: dict[str, str] = {},
+    secrets: dict[str, str] | None = None,
 ) -> dict[str, Any]:
+    secrets = secrets if secrets is not None else {}
     consumes = inst["module"].get("spec", {}).get("consumes", [])
     resolved: dict[str, Any] = {}
 

{composable_data_stack-0.5.1 → composable_data_stack-0.6.0}/cli/preflight.py

@@ -11,14 +11,17 @@ from typing import Any
 
 import yaml
 
-from .image_verification import default_fixture_path, load_policy_from_env, verify_images
+from .image_verification import (
+    default_fixture_path,
+    load_policy_from_env,
+    verify_images,
+)
 from .security_common import SECRET_KEY_SEGMENT_RE, infer_profile_class
 
-
 _ENV_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)([^}]*)\}")
 # Docker publishes ports on every IPv4 interface when no host IP is specified.
 # Preflight uses this only for a short-lived availability probe.
-_DOCKER_WILDCARD_IPV4 = "0.0.0.0"  # nosec B104
+_DOCKER_WILDCARD_IPV4 = "0.0.0.0"  # nosec B104  # noqa: S104
 
 
 @dataclass(frozen=True)
@@ -95,7 +98,7 @@ def _run_runtime_command(
     failure_message: str,
 ) -> PreflightCheck:
     try:
-        result = subprocess.run(  # nosec B603
+        result = subprocess.run(  # nosec B603  # noqa: S603
             command,
             capture_output=True,
             text=True,
@@ -171,7 +174,7 @@ def _check_environment(compose_yaml: str, env_file: Path) -> list[PreflightCheck
                         PreflightCheck(
                             "WARN",
                             "environment.placeholders",
-                            "Replace placeholder values for: "
+                            "Replace placeholder values for: "  # noqa: S608
                             + ", ".join(placeholders)
                             + ".",
                         )