composable-data-stack 0.5.2__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/PKG-INFO +1 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/getter.py +37 -6
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/image_updates.py +2 -2
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/image_verification.py +1 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/loader.py +0 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/main.py +41 -19
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/planner.py +42 -10
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/preflight.py +8 -5
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/renderer.py +125 -11
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/secrets.py +1 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/security.py +2 -3
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/security_common.py +0 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/up_runner.py +4 -4
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/utils.py +1 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/validator.py +6 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/composable_data_stack.egg-info/PKG-INFO +1 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/pyproject.toml +16 -4
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_environment_classification.py +5 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_getter.py +75 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_image_verification.py +1 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_main.py +89 -14
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_module_isolation.py +43 -18
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_modules_no_committed_secrets.py +3 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_overlay.py +6 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_planner.py +101 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_render_example_profile.py +3 -9
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_renderer.py +214 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_security.py +6 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_state.py +5 -1
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/LICENSE +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/README.md +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/__init__.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/constants.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/diagnostics.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/graph.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/overlay.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resolver.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resources/__init__.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resources/contract.schema.json +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resources/module.schema.json +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resources/profile.schema.json +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resources/rule-schema.json +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/resources/rule-set.json +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/cli/state.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/composable_data_stack.egg-info/SOURCES.txt +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/composable_data_stack.egg-info/dependency_links.txt +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/composable_data_stack.egg-info/entry_points.txt +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/composable_data_stack.egg-info/requires.txt +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/composable_data_stack.egg-info/top_level.txt +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/setup.cfg +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_cds_workflow.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_completion.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_compose_runtime_smoke.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_dagster_config_generation.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_dagster_definitions_uri.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_dagster_entrypoint_backend_guard.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_dagster_hardening.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_environment_cli.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_graph.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_image_security_scan_workflow.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_image_updates.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_mvp_proof_workflow.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_preflight.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_publish_images_workflow.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_python_version_alignment.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_release_version_check.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_resolve_profile_path.py +3 -3
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_resolver.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_secrets.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_smoke_example_profile.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_superset_hardening.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_trivyignore.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_up_runner.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_upstream_image_hardening.py +0 -0
- {composable_data_stack-0.5.2 → composable_data_stack-0.6.0}/tests/test_validator.py +0 -0
|
@@ -20,7 +20,6 @@ from urllib.request import Request, urlopen
|
|
|
20
20
|
from .loader import load_yaml_file, resolve_module_dir
|
|
21
21
|
from .planner import MaxNestingDepthExceeded, apply_defaults, substitute_string
|
|
22
22
|
|
|
23
|
-
|
|
24
23
|
# The upstream repository `cds get` downloads from when no `--remote` is
|
|
25
24
|
# given. Keep in sync with the `Repository` URL in pyproject.toml.
|
|
26
25
|
DEFAULT_REMOTE = "RonaldHensbergen/composable-data-stack"
|
|
@@ -92,9 +91,7 @@ def fetch_profile(
|
|
|
92
91
|
f"{rendered}{extra}"
|
|
93
92
|
)
|
|
94
93
|
|
|
95
|
-
|
|
96
|
-
action.destination.parent.mkdir(parents=True, exist_ok=True)
|
|
97
|
-
shutil.copy2(action.source, action.destination)
|
|
94
|
+
_write_actions(actions)
|
|
98
95
|
|
|
99
96
|
_write_tracking_manifest(
|
|
100
97
|
target_root=target_root,
|
|
@@ -169,7 +166,7 @@ def _download_github_repository(owner: str, repo: str, ref: str, work_dir: Path)
|
|
|
169
166
|
url = f"https://api.github.com/repos/{owner}/{repo}/tarball/{ref}"
|
|
170
167
|
request = Request(url, headers={"User-Agent": "composable-data-stack-cds-get"})
|
|
171
168
|
try:
|
|
172
|
-
with urlopen(request, timeout=30) as response: # nosec B310 - fixed https GitHub API host
|
|
169
|
+
with urlopen(request, timeout=30) as response: # nosec B310 - fixed https GitHub API host # noqa: S310
|
|
173
170
|
archive_bytes = response.read()
|
|
174
171
|
except HTTPError as exc:
|
|
175
172
|
raise GetError(
|
|
@@ -599,7 +596,14 @@ def _add_copy_action(
|
|
|
599
596
|
actions_by_destination: dict[Path, CopyAction],
|
|
600
597
|
) -> None:
|
|
601
598
|
repo_relative = source_file.resolve().relative_to(source_repo.resolve())
|
|
602
|
-
|
|
599
|
+
# Deliberately do NOT call .resolve() on the combined destination path:
|
|
600
|
+
# destination_root is already an absolute, resolved path (see
|
|
601
|
+
# fetch_profile()), and resolving the full path here would follow a
|
|
602
|
+
# symlink planted at the destination leaf, silently swapping the
|
|
603
|
+
# CopyAction's destination for the symlink's target instead of the
|
|
604
|
+
# symlink path itself. That would defeat the is_symlink() conflict/
|
|
605
|
+
# write-through guards in _find_conflicts()/_write_actions() (#474).
|
|
606
|
+
destination = destination_root / repo_relative
|
|
603
607
|
existing = actions_by_destination.get(destination)
|
|
604
608
|
if existing is None:
|
|
605
609
|
actions_by_destination[destination] = CopyAction(
|
|
@@ -618,6 +622,14 @@ def _find_conflicts(actions: list[CopyAction]) -> list[str]:
|
|
|
618
622
|
conflicts: list[str] = []
|
|
619
623
|
for action in actions:
|
|
620
624
|
destination = action.destination
|
|
625
|
+
if destination.is_symlink():
|
|
626
|
+
# A symlink destination is always a conflict, even if it's
|
|
627
|
+
# dangling (Path.exists() follows symlinks and returns False for
|
|
628
|
+
# a broken link, which would otherwise let a pre-planted symlink
|
|
629
|
+
# silently bypass this check). Never write through a symlink:
|
|
630
|
+
# see _write_actions().
|
|
631
|
+
conflicts.append(action.repo_relative_path)
|
|
632
|
+
continue
|
|
621
633
|
if not destination.exists():
|
|
622
634
|
continue
|
|
623
635
|
if destination.is_dir():
|
|
@@ -628,6 +640,21 @@ def _find_conflicts(actions: list[CopyAction]) -> list[str]:
|
|
|
628
640
|
return conflicts
|
|
629
641
|
|
|
630
642
|
|
|
643
|
+
def _write_actions(actions: list[CopyAction]) -> None:
|
|
644
|
+
"""Copy planned actions to their destinations, refusing to write through
|
|
645
|
+
a symlinked destination even when --force allowed the conflict check to
|
|
646
|
+
pass. Any pre-existing symlink at a destination path (dangling or not)
|
|
647
|
+
is removed first so shutil.copy2() always creates/overwrites a regular
|
|
648
|
+
file there instead of following the link to some other location on disk
|
|
649
|
+
(e.g. a symlink planted at profiles/foo/profile.yaml pointing outside
|
|
650
|
+
the destination tree)."""
|
|
651
|
+
for action in actions:
|
|
652
|
+
action.destination.parent.mkdir(parents=True, exist_ok=True)
|
|
653
|
+
if action.destination.is_symlink():
|
|
654
|
+
action.destination.unlink()
|
|
655
|
+
shutil.copy2(action.source, action.destination)
|
|
656
|
+
|
|
657
|
+
|
|
631
658
|
def _write_tracking_manifest(
|
|
632
659
|
*,
|
|
633
660
|
target_root: Path,
|
|
@@ -662,6 +689,10 @@ def _write_tracking_manifest(
|
|
|
662
689
|
profiles[requested_profile] = entry
|
|
663
690
|
|
|
664
691
|
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
|
692
|
+
if manifest_path.is_symlink():
|
|
693
|
+
# Same write-through-symlink guard as _write_actions(): never follow
|
|
694
|
+
# a symlink planted at the tracking manifest's path.
|
|
695
|
+
manifest_path.unlink()
|
|
665
696
|
manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
|
666
697
|
|
|
667
698
|
|
|
@@ -239,10 +239,10 @@ def fetch_dockerhub_tags(namespace: str, repository: str, page_size: int = 100,
|
|
|
239
239
|
|
|
240
240
|
while url and pages_read < page_limit:
|
|
241
241
|
try:
|
|
242
|
-
req = Request(url, headers={"User-Agent": "cds-image-check/1.0"})
|
|
242
|
+
req = Request(url, headers={"User-Agent": "cds-image-check/1.0"}) # noqa: S310
|
|
243
243
|
# url is always http(s), built from the hardcoded DOCKER_HUB_API
|
|
244
244
|
# constant or the scheme-validated pagination url below.
|
|
245
|
-
with urlopen(req, timeout=10) as response: # nosec B310
|
|
245
|
+
with urlopen(req, timeout=10) as response: # nosec B310 # noqa: S310
|
|
246
246
|
data = json.loads(response.read().decode())
|
|
247
247
|
except (HTTPError, URLError, ValueError):
|
|
248
248
|
return None
|
|
@@ -246,7 +246,7 @@ def _verify_with_cosign(
|
|
|
246
246
|
command.append(image_ref)
|
|
247
247
|
|
|
248
248
|
try:
|
|
249
|
-
result = subprocess.run(command, capture_output=True, text=True, timeout=120) # nosec B603
|
|
249
|
+
result = subprocess.run(command, capture_output=True, text=True, timeout=120) # nosec B603 # noqa: S603
|
|
250
250
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
251
251
|
return False, f"cosign invocation failed: {exc}"
|
|
252
252
|
|
|
@@ -7,9 +7,8 @@ import os
|
|
|
7
7
|
import re
|
|
8
8
|
import subprocess # nosec B404
|
|
9
9
|
import sys
|
|
10
|
-
import
|
|
11
|
-
from
|
|
12
|
-
from importlib.metadata import PackageNotFoundError, version as _package_version
|
|
10
|
+
from importlib.metadata import PackageNotFoundError
|
|
11
|
+
from importlib.metadata import version as _package_version
|
|
13
12
|
from pathlib import Path
|
|
14
13
|
from typing import Any
|
|
15
14
|
|
|
@@ -18,22 +17,24 @@ try:
|
|
|
18
17
|
except ImportError:
|
|
19
18
|
argcomplete = None
|
|
20
19
|
|
|
21
|
-
|
|
20
|
+
import yaml
|
|
21
|
+
|
|
22
22
|
from .diagnostics import Diagnostic
|
|
23
|
-
from .
|
|
24
|
-
from .
|
|
25
|
-
from .renderer import render_compose
|
|
26
|
-
from .image_updates import collect_module_images, check_image_update
|
|
27
|
-
from .overlay import resolve_profile
|
|
28
|
-
from .preflight import preflight_passed, run_preflight
|
|
29
|
-
from .security import PrecomputedRender, run_security_validation
|
|
30
|
-
from .security_common import SEVERITY_ORDER, infer_profile_class
|
|
23
|
+
from .getter import GetError, fetch_profile, format_get_plan
|
|
24
|
+
from .image_updates import check_image_update, collect_module_images
|
|
31
25
|
from .image_verification import (
|
|
32
26
|
ImagePolicy,
|
|
33
27
|
default_fixture_path,
|
|
34
28
|
load_policy_from_env,
|
|
35
29
|
verify_images,
|
|
36
30
|
)
|
|
31
|
+
from .loader import load_yaml_file
|
|
32
|
+
from .overlay import resolve_profile
|
|
33
|
+
from .planner import build_plan
|
|
34
|
+
from .preflight import preflight_passed, run_preflight
|
|
35
|
+
from .renderer import render_compose
|
|
36
|
+
from .security import PrecomputedRender, run_security_validation
|
|
37
|
+
from .security_common import SEVERITY_ORDER, infer_profile_class
|
|
37
38
|
from .state import format_state_output, group_services_by_health, parse_compose_ps_json
|
|
38
39
|
from .up_runner import (
|
|
39
40
|
DEFAULT_TIMEOUT_SECONDS,
|
|
@@ -44,9 +45,8 @@ from .up_runner import (
|
|
|
44
45
|
start_up_in_background,
|
|
45
46
|
stop_log_tail,
|
|
46
47
|
)
|
|
47
|
-
from .
|
|
48
|
-
from .
|
|
49
|
-
import yaml
|
|
48
|
+
from .utils import _atomic_write
|
|
49
|
+
from .validator import has_errors, validate_profile
|
|
50
50
|
|
|
51
51
|
|
|
52
52
|
def load_env_file(env_file: str = ".env") -> None:
|
|
@@ -445,6 +445,19 @@ def _add_environment_arg(subparser: argparse.ArgumentParser) -> None:
|
|
|
445
445
|
)
|
|
446
446
|
|
|
447
447
|
|
|
448
|
+
def _add_hardened_arg(subparser: argparse.ArgumentParser) -> None:
|
|
449
|
+
subparser.add_argument(
|
|
450
|
+
"--hardened",
|
|
451
|
+
action="store_true",
|
|
452
|
+
help=(
|
|
453
|
+
"Override config.image.variant to the Alpine-hardened build for any "
|
|
454
|
+
"module whose configSchema exposes an image.variant option (currently "
|
|
455
|
+
"only modules/orchestration/dagster), without editing the profile YAML. "
|
|
456
|
+
"Modules without that config option are left unchanged."
|
|
457
|
+
),
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
|
|
448
461
|
def _collect_profile_env_vars(
|
|
449
462
|
profile_path: str, environment: str | None = None
|
|
450
463
|
) -> tuple[list[str], set[str]]:
|
|
@@ -769,6 +782,7 @@ def main() -> int:
|
|
|
769
782
|
"-o",
|
|
770
783
|
help="Output file path for rendered output (default: <project-root>/docker-compose.yml)",
|
|
771
784
|
)
|
|
785
|
+
_add_hardened_arg(render_parser)
|
|
772
786
|
|
|
773
787
|
up_parser = subparsers.add_parser(
|
|
774
788
|
"up",
|
|
@@ -805,6 +819,7 @@ def main() -> int:
|
|
|
805
819
|
action="store_true",
|
|
806
820
|
help="Disable colored labels in the live state view",
|
|
807
821
|
)
|
|
822
|
+
_add_hardened_arg(up_parser)
|
|
808
823
|
|
|
809
824
|
test_parser = subparsers.add_parser(
|
|
810
825
|
"test",
|
|
@@ -820,6 +835,7 @@ def main() -> int:
|
|
|
820
835
|
"By default, values are redacted to avoid echoing real secrets to stdout/CI logs."
|
|
821
836
|
),
|
|
822
837
|
)
|
|
838
|
+
_add_hardened_arg(test_parser)
|
|
823
839
|
|
|
824
840
|
preflight_parser = subparsers.add_parser(
|
|
825
841
|
"preflight",
|
|
@@ -1104,7 +1120,9 @@ def main() -> int:
|
|
|
1104
1120
|
return 1
|
|
1105
1121
|
|
|
1106
1122
|
env_file = str(resolve_env_file_path(profile_path))
|
|
1107
|
-
plan, plan_diags = build_plan(
|
|
1123
|
+
plan, plan_diags = build_plan(
|
|
1124
|
+
profile_path, env_file=env_file, environment=args.environment, hardened=args.hardened
|
|
1125
|
+
)
|
|
1108
1126
|
all_diags = diagnostics + plan_diags
|
|
1109
1127
|
if has_errors(all_diags):
|
|
1110
1128
|
print_diagnostics(all_diags)
|
|
@@ -1141,7 +1159,9 @@ def main() -> int:
|
|
|
1141
1159
|
return 1
|
|
1142
1160
|
|
|
1143
1161
|
env_file = str(resolve_env_file_path(profile_path))
|
|
1144
|
-
plan, plan_diags = build_plan(
|
|
1162
|
+
plan, plan_diags = build_plan(
|
|
1163
|
+
profile_path, env_file=env_file, environment=args.environment, hardened=args.hardened
|
|
1164
|
+
)
|
|
1145
1165
|
all_diags = diagnostics + plan_diags
|
|
1146
1166
|
if has_errors(all_diags):
|
|
1147
1167
|
print_diagnostics(all_diags)
|
|
@@ -1356,7 +1376,9 @@ def main() -> int:
|
|
|
1356
1376
|
render_diags: list[Diagnostic] = []
|
|
1357
1377
|
render_ok = False
|
|
1358
1378
|
if validate_ok:
|
|
1359
|
-
plan, plan_diags = build_plan(
|
|
1379
|
+
plan, plan_diags = build_plan(
|
|
1380
|
+
profile_path, env_file=env_file, environment=args.environment, hardened=args.hardened
|
|
1381
|
+
)
|
|
1360
1382
|
plan_ok = not has_errors(diagnostics + plan_diags)
|
|
1361
1383
|
if plan_ok:
|
|
1362
1384
|
compose_yaml, render_diags = render_compose(plan, env_file=env_file)
|
|
@@ -1481,7 +1503,7 @@ def main() -> int:
|
|
|
1481
1503
|
|
|
1482
1504
|
ps_cmd = ["docker", "compose", "-f", str(compose_path), "ps", "-a", "--format", "json"]
|
|
1483
1505
|
try:
|
|
1484
|
-
ps_result = subprocess.run(ps_cmd, capture_output=True, text=True) # nosec B603
|
|
1506
|
+
ps_result = subprocess.run(ps_cmd, capture_output=True, text=True) # nosec B603 # noqa: S603
|
|
1485
1507
|
except FileNotFoundError:
|
|
1486
1508
|
print("ERROR docker was not found. Install Docker and ensure it is on your PATH.")
|
|
1487
1509
|
return 1
|
|
@@ -1,23 +1,24 @@
|
|
|
1
1
|
# cli/planner.py
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
4
6
|
from copy import deepcopy
|
|
7
|
+
from dataclasses import dataclass
|
|
5
8
|
from pathlib import Path
|
|
6
9
|
from typing import Any
|
|
7
|
-
import os
|
|
8
|
-
import re
|
|
9
10
|
|
|
10
11
|
from .constants import MAX_NESTING_DEPTH, MaxNestingDepthExceeded
|
|
11
12
|
from .diagnostics import Diagnostic
|
|
12
13
|
from .loader import load_yaml_file, resolve_module_file
|
|
13
|
-
from .resolver import
|
|
14
|
+
from .resolver import (
|
|
15
|
+
is_secret_ref,
|
|
16
|
+
parse_contract_ref,
|
|
17
|
+
resolve_path,
|
|
18
|
+
secret_name_from_ref,
|
|
19
|
+
)
|
|
14
20
|
from .secrets import load_profile_secrets
|
|
15
21
|
|
|
16
|
-
from dataclasses import dataclass
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
22
|
|
|
22
23
|
@dataclass
|
|
23
24
|
class SecretRef:
|
|
@@ -28,6 +29,7 @@ def build_plan(
|
|
|
28
29
|
profile_path: str,
|
|
29
30
|
env_file: str | None = None,
|
|
30
31
|
environment: str | None = None,
|
|
32
|
+
hardened: bool = False,
|
|
31
33
|
) -> tuple[dict[str, Any] | None, list[Diagnostic]]:
|
|
32
34
|
"""
|
|
33
35
|
Build a resolved plan from a profile.
|
|
@@ -40,6 +42,14 @@ def build_plan(
|
|
|
40
42
|
overlay is merged over the base profile before planning; see
|
|
41
43
|
cli.overlay.resolve_profile. Value provenance for the merge is
|
|
42
44
|
recorded on the returned plan under "provenance".
|
|
45
|
+
hardened: When True, overrides `config.image.variant` to "hardened"
|
|
46
|
+
for any module instance whose configSchema exposes an
|
|
47
|
+
`image.variant` property (currently only
|
|
48
|
+
modules/orchestration/dagster), leaving modules without that
|
|
49
|
+
config option untouched. This is the CLI-level `--hardened`
|
|
50
|
+
convenience flag (see cli/main.py's up/render/test subcommands)
|
|
51
|
+
for switching to the Alpine-hardened image variant without
|
|
52
|
+
hand-editing the profile YAML.
|
|
43
53
|
|
|
44
54
|
Returns:
|
|
45
55
|
Tuple of (plan, diagnostics)
|
|
@@ -150,6 +160,8 @@ def build_plan(
|
|
|
150
160
|
module_instance.get("config", {}),
|
|
151
161
|
module_def.get("spec", {}).get("configSchema", {})
|
|
152
162
|
)
|
|
163
|
+
if hardened and _supports_image_variant(module_def):
|
|
164
|
+
normalized_config.setdefault("image", {})["variant"] = "hardened"
|
|
153
165
|
except MaxNestingDepthExceeded:
|
|
154
166
|
diagnostics.append(Diagnostic(
|
|
155
167
|
level="error",
|
|
@@ -253,6 +265,22 @@ def _substitute_config_env_vars(
|
|
|
253
265
|
return _CDS_VAR_PATTERN.sub(_replace, obj)
|
|
254
266
|
return obj
|
|
255
267
|
|
|
268
|
+
def _supports_image_variant(module_def: dict[str, Any]) -> bool:
|
|
269
|
+
"""
|
|
270
|
+
True if a module's configSchema exposes a `image.variant` property
|
|
271
|
+
(currently only modules/orchestration/dagster), which is what the
|
|
272
|
+
`--hardened` CLI flag overrides. Modules without this config option
|
|
273
|
+
(e.g. warehouse/postgres, bi/superset) are left untouched.
|
|
274
|
+
"""
|
|
275
|
+
image_schema = (
|
|
276
|
+
module_def.get("spec", {})
|
|
277
|
+
.get("configSchema", {})
|
|
278
|
+
.get("properties", {})
|
|
279
|
+
.get("image", {})
|
|
280
|
+
)
|
|
281
|
+
return "variant" in image_schema.get("properties", {})
|
|
282
|
+
|
|
283
|
+
|
|
256
284
|
def apply_defaults(config: dict[str, Any], schema: dict[str, Any]) -> dict[str, Any]:
|
|
257
285
|
config_copy = deepcopy(config)
|
|
258
286
|
return _apply_schema_defaults(config_copy, schema)
|
|
@@ -307,10 +335,13 @@ def _apply_schema_defaults(value: Any, schema: dict[str, Any], _depth: int = 0)
|
|
|
307
335
|
|
|
308
336
|
return value
|
|
309
337
|
|
|
310
|
-
def resolve_provided_contracts(
|
|
338
|
+
def resolve_provided_contracts(
|
|
339
|
+
inst: dict[str, Any], secrets: dict[str, str] | None = None
|
|
340
|
+
) -> dict[str, Any]:
|
|
311
341
|
"""
|
|
312
342
|
Resolve contracts provided by a module instance.
|
|
313
343
|
"""
|
|
344
|
+
secrets = secrets if secrets is not None else {}
|
|
314
345
|
provides = inst["module"].get("spec", {}).get("provides", [])
|
|
315
346
|
resolved: dict[str, Any] = {}
|
|
316
347
|
service_name = inst["id"]
|
|
@@ -340,8 +371,9 @@ def resolve_consumed_contracts(
|
|
|
340
371
|
modules_by_id: dict[str, dict[str, Any]],
|
|
341
372
|
resolved_contracts_by_module: dict[str, dict[str, Any]],
|
|
342
373
|
diagnostics: list[Diagnostic],
|
|
343
|
-
secrets: dict[str, str] =
|
|
374
|
+
secrets: dict[str, str] | None = None,
|
|
344
375
|
) -> dict[str, Any]:
|
|
376
|
+
secrets = secrets if secrets is not None else {}
|
|
345
377
|
consumes = inst["module"].get("spec", {}).get("consumes", [])
|
|
346
378
|
resolved: dict[str, Any] = {}
|
|
347
379
|
|
|
@@ -11,14 +11,17 @@ from typing import Any
|
|
|
11
11
|
|
|
12
12
|
import yaml
|
|
13
13
|
|
|
14
|
-
from .image_verification import
|
|
14
|
+
from .image_verification import (
|
|
15
|
+
default_fixture_path,
|
|
16
|
+
load_policy_from_env,
|
|
17
|
+
verify_images,
|
|
18
|
+
)
|
|
15
19
|
from .security_common import SECRET_KEY_SEGMENT_RE, infer_profile_class
|
|
16
20
|
|
|
17
|
-
|
|
18
21
|
_ENV_REFERENCE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)([^}]*)\}")
|
|
19
22
|
# Docker publishes ports on every IPv4 interface when no host IP is specified.
|
|
20
23
|
# Preflight uses this only for a short-lived availability probe.
|
|
21
|
-
_DOCKER_WILDCARD_IPV4 = "0.0.0.0" # nosec B104
|
|
24
|
+
_DOCKER_WILDCARD_IPV4 = "0.0.0.0" # nosec B104 # noqa: S104
|
|
22
25
|
|
|
23
26
|
|
|
24
27
|
@dataclass(frozen=True)
|
|
@@ -95,7 +98,7 @@ def _run_runtime_command(
|
|
|
95
98
|
failure_message: str,
|
|
96
99
|
) -> PreflightCheck:
|
|
97
100
|
try:
|
|
98
|
-
result = subprocess.run( # nosec B603
|
|
101
|
+
result = subprocess.run( # nosec B603 # noqa: S603
|
|
99
102
|
command,
|
|
100
103
|
capture_output=True,
|
|
101
104
|
text=True,
|
|
@@ -171,7 +174,7 @@ def _check_environment(compose_yaml: str, env_file: Path) -> list[PreflightCheck
|
|
|
171
174
|
PreflightCheck(
|
|
172
175
|
"WARN",
|
|
173
176
|
"environment.placeholders",
|
|
174
|
-
"Replace placeholder values for: "
|
|
177
|
+
"Replace placeholder values for: " # noqa: S608
|
|
175
178
|
+ ", ".join(placeholders)
|
|
176
179
|
+ ".",
|
|
177
180
|
)
|
|
@@ -5,22 +5,18 @@ Render docker-compose YAML from a composition plan.
|
|
|
5
5
|
"""
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
|
-
import re
|
|
9
8
|
import os
|
|
10
|
-
import
|
|
11
|
-
import yaml
|
|
9
|
+
import re
|
|
12
10
|
from copy import deepcopy
|
|
13
11
|
from pathlib import Path
|
|
14
12
|
from typing import Any
|
|
15
13
|
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
16
|
from .constants import MAX_NESTING_DEPTH, MaxNestingDepthExceeded
|
|
17
|
-
from .utils import _atomic_write
|
|
18
17
|
from .diagnostics import Diagnostic
|
|
19
18
|
from .loader import resolve_module_dir
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
19
|
+
from .utils import _atomic_write
|
|
24
20
|
|
|
25
21
|
|
|
26
22
|
def render_compose(
|
|
@@ -107,7 +103,7 @@ def render_compose(
|
|
|
107
103
|
volumes = compose_impl.get("volumes", {})
|
|
108
104
|
|
|
109
105
|
try:
|
|
110
|
-
rendered_services = _render_services(
|
|
106
|
+
rendered_services, unsafe_diags = _render_services(
|
|
111
107
|
module,
|
|
112
108
|
services,
|
|
113
109
|
secrets,
|
|
@@ -116,6 +112,7 @@ def render_compose(
|
|
|
116
112
|
compose_dir=compose_dir,
|
|
117
113
|
network_name=default_network_name,
|
|
118
114
|
)
|
|
115
|
+
diagnostics.extend(unsafe_diags)
|
|
119
116
|
rendered_volumes = _render_volumes(module, volumes, secrets)
|
|
120
117
|
except MaxNestingDepthExceeded:
|
|
121
118
|
diagnostics.append(Diagnostic(
|
|
@@ -199,8 +196,9 @@ def _render_services(
|
|
|
199
196
|
project_root: Path | None,
|
|
200
197
|
compose_dir: Path,
|
|
201
198
|
network_name: str | None = None,
|
|
202
|
-
) -> dict[str, Any]:
|
|
199
|
+
) -> tuple[dict[str, Any], list[Diagnostic]]:
|
|
203
200
|
rendered: dict[str, Any] = {}
|
|
201
|
+
diagnostics: list[Diagnostic] = []
|
|
204
202
|
context = _build_context(module, secrets)
|
|
205
203
|
|
|
206
204
|
for service_name, service_def in services.items():
|
|
@@ -227,6 +225,9 @@ def _render_services(
|
|
|
227
225
|
else:
|
|
228
226
|
service_copy["healthcheck"] = _substitute_values(hc_copy, context)
|
|
229
227
|
|
|
228
|
+
diagnostics.extend(
|
|
229
|
+
_check_unsafe_field_type_substitutions(service_copy, module.get("id"), service_name, context)
|
|
230
|
+
)
|
|
230
231
|
service_copy = _substitute_values(service_copy, context)
|
|
231
232
|
service_copy = _rewrite_service_volumes(
|
|
232
233
|
service_copy,
|
|
@@ -250,7 +251,7 @@ def _render_services(
|
|
|
250
251
|
|
|
251
252
|
rendered[service_name] = service_copy
|
|
252
253
|
|
|
253
|
-
return rendered
|
|
254
|
+
return rendered, diagnostics
|
|
254
255
|
|
|
255
256
|
|
|
256
257
|
def _render_volumes(
|
|
@@ -371,6 +372,119 @@ def _substitute_string(value: str, context: dict[str, Any]) -> Any:
|
|
|
371
372
|
return _PATTERN.sub(_replace, value)
|
|
372
373
|
|
|
373
374
|
|
|
375
|
+
# Compose service fields where a module template's pure `${config.*}` /
|
|
376
|
+
# `${bindings.*}` substitution resolving to a non-scalar type (dict/list) is
|
|
377
|
+
# always attacker-relevant: they control what docker-compose executes, sets
|
|
378
|
+
# as environment, or bind-mounts on the host, regardless of the module
|
|
379
|
+
# author's intent. See GHSA-gmc4-jw3j-mqcf.
|
|
380
|
+
_UNSAFE_TYPED_SUBSTITUTION_FIELDS = frozenset({
|
|
381
|
+
"command",
|
|
382
|
+
"entrypoint",
|
|
383
|
+
"environment",
|
|
384
|
+
"volumes",
|
|
385
|
+
"cap_add",
|
|
386
|
+
"cap_drop",
|
|
387
|
+
"devices",
|
|
388
|
+
"security_opt",
|
|
389
|
+
"privileged",
|
|
390
|
+
"network_mode",
|
|
391
|
+
"pid",
|
|
392
|
+
"ports",
|
|
393
|
+
"labels",
|
|
394
|
+
"tmpfs",
|
|
395
|
+
"dns",
|
|
396
|
+
"extra_hosts",
|
|
397
|
+
"logging",
|
|
398
|
+
"expose",
|
|
399
|
+
"depends_on",
|
|
400
|
+
"build",
|
|
401
|
+
})
|
|
402
|
+
|
|
403
|
+
# Fields where the field itself is already scalar (bool/str), so a dict/list
|
|
404
|
+
# resolution is not the only attacker-relevant outcome: a profile-controlled
|
|
405
|
+
# scalar value can *also* grant host-level escalation without ever changing
|
|
406
|
+
# the field's expected shape (e.g. `privileged: "${config.priv}"` resolving
|
|
407
|
+
# to `True`, or `network_mode`/`pid` resolving to `"host"`). These are
|
|
408
|
+
# checked against specific dangerous scalar values, in addition to the
|
|
409
|
+
# dict/list check above.
|
|
410
|
+
_UNSAFE_SCALAR_SUBSTITUTION_VALUES: dict[str, frozenset[Any]] = {
|
|
411
|
+
"privileged": frozenset({True}),
|
|
412
|
+
"network_mode": frozenset({"host"}),
|
|
413
|
+
"pid": frozenset({"host"}),
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def _check_unsafe_field_type_substitutions(
|
|
418
|
+
service_def: dict[str, Any],
|
|
419
|
+
module_id: str | None,
|
|
420
|
+
service_name: str,
|
|
421
|
+
context: dict[str, Any],
|
|
422
|
+
) -> list[Diagnostic]:
|
|
423
|
+
"""
|
|
424
|
+
Detect a module template that uses a *pure* `${config.*}`/`${bindings.*}`
|
|
425
|
+
substitution (i.e. the entire field value is a single expression, not
|
|
426
|
+
string-concatenated) in a compose field position where docker-compose
|
|
427
|
+
expects a fixed shape (list/dict/scalar). Pure substitution preserves the
|
|
428
|
+
resolved value's original Python type, so a profile can inject arbitrary
|
|
429
|
+
lists/dicts (e.g. a `command` list, `environment` map, or `volumes` bind
|
|
430
|
+
mounts) through module config, bypassing the field's intended shape.
|
|
431
|
+
|
|
432
|
+
This does not affect mixed substitution (e.g. "db://${bindings.db.host}"),
|
|
433
|
+
which is always string-concatenated back into a str.
|
|
434
|
+
|
|
435
|
+
Beyond the dict/list check, a small set of fields (`privileged`,
|
|
436
|
+
`network_mode`, `pid`) are already scalar by design, so a resolved value
|
|
437
|
+
is also flagged if it matches a known dangerous scalar (e.g.
|
|
438
|
+
`privileged` resolving to `True`, or `network_mode`/`pid` resolving to
|
|
439
|
+
`"host"`), since a profile can drive host-level escalation through
|
|
440
|
+
module config without ever changing the field's shape.
|
|
441
|
+
"""
|
|
442
|
+
diagnostics: list[Diagnostic] = []
|
|
443
|
+
pure_pattern = re.compile(r"^\$\{([^}]+)\}$")
|
|
444
|
+
|
|
445
|
+
for field in _UNSAFE_TYPED_SUBSTITUTION_FIELDS:
|
|
446
|
+
raw_value = service_def.get(field)
|
|
447
|
+
if not isinstance(raw_value, str):
|
|
448
|
+
continue
|
|
449
|
+
match = pure_pattern.match(raw_value)
|
|
450
|
+
if not match:
|
|
451
|
+
continue
|
|
452
|
+
|
|
453
|
+
resolved = _resolve_expr(match.group(1), context)
|
|
454
|
+
unsafe_scalars = _UNSAFE_SCALAR_SUBSTITUTION_VALUES.get(field, frozenset())
|
|
455
|
+
if isinstance(resolved, (dict, list)):
|
|
456
|
+
diagnostics.append(Diagnostic(
|
|
457
|
+
level="error",
|
|
458
|
+
code="E072",
|
|
459
|
+
message=(
|
|
460
|
+
f'Module "{module_id}" service "{service_name}" field "{field}" resolves '
|
|
461
|
+
f'"${{{match.group(1)}}}" to a {type(resolved).__name__}, but pure '
|
|
462
|
+
"substitution in this field position is not permitted: a profile-supplied "
|
|
463
|
+
"config value would be spliced verbatim into the rendered Compose file, "
|
|
464
|
+
"letting an untrusted profile inject arbitrary command/environment/volumes "
|
|
465
|
+
"entries. Restructure the module template to interpolate scalar leaf values "
|
|
466
|
+
"individually instead of substituting the whole field."
|
|
467
|
+
),
|
|
468
|
+
path=f"module:{module_id}.implementation.compose.services.{service_name}.{field}",
|
|
469
|
+
))
|
|
470
|
+
elif resolved in unsafe_scalars:
|
|
471
|
+
diagnostics.append(Diagnostic(
|
|
472
|
+
level="error",
|
|
473
|
+
code="E072",
|
|
474
|
+
message=(
|
|
475
|
+
f'Module "{module_id}" service "{service_name}" field "{field}" resolves '
|
|
476
|
+
f'"${{{match.group(1)}}}" to {resolved!r}, a host-escalating value for this '
|
|
477
|
+
"field. A profile-supplied config value would grant host-level privileges "
|
|
478
|
+
"(privileged mode, or host network/PID namespace sharing) through what "
|
|
479
|
+
"should be module-controlled configuration. Do not let profile-controlled "
|
|
480
|
+
"config drive this field."
|
|
481
|
+
),
|
|
482
|
+
path=f"module:{module_id}.implementation.compose.services.{service_name}.{field}",
|
|
483
|
+
))
|
|
484
|
+
|
|
485
|
+
return diagnostics
|
|
486
|
+
|
|
487
|
+
|
|
374
488
|
def _resolve_expr(expr: str, context: dict[str, Any]) -> Any:
|
|
375
489
|
"""
|
|
376
490
|
Resolve a dot-notation expression against context.
|
|
@@ -9,6 +9,7 @@ from __future__ import annotations
|
|
|
9
9
|
import json
|
|
10
10
|
import os
|
|
11
11
|
import re
|
|
12
|
+
from dataclasses import dataclass
|
|
12
13
|
from importlib.resources import files
|
|
13
14
|
from importlib.resources.abc import Traversable
|
|
14
15
|
from pathlib import Path
|
|
@@ -17,8 +18,6 @@ from typing import Any
|
|
|
17
18
|
import yaml
|
|
18
19
|
from jsonschema import Draft202012Validator
|
|
19
20
|
|
|
20
|
-
from dataclasses import dataclass
|
|
21
|
-
|
|
22
21
|
from .diagnostics import Diagnostic
|
|
23
22
|
from .loader import load_yaml_file, resolve_module_file
|
|
24
23
|
from .planner import build_plan
|
|
@@ -375,7 +374,7 @@ def _eval_condition(
|
|
|
375
374
|
if "portExposure" in cond:
|
|
376
375
|
exposure = cond["portExposure"]
|
|
377
376
|
# Comparing a scanned config's string, not binding to an interface.
|
|
378
|
-
if exposure == "0.0.0.0" and "0.0.0.0:" not in sval: # nosec B104
|
|
377
|
+
if exposure == "0.0.0.0" and "0.0.0.0:" not in sval: # nosec B104 # noqa: S104
|
|
379
378
|
return False
|
|
380
379
|
if exposure == "host-published" and ":" not in sval:
|
|
381
380
|
return False
|