composable-data-stack 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/image_verification.py +24 -3
- cli/k8s_renderer.py +5 -0
- cli/k8s_runner.py +79 -2
- cli/k8s_security.py +13 -0
- cli/main.py +145 -20
- cli/resources/compatibility-registry.json +31 -0
- cli/resources/compatibility-registry.schema.json +70 -0
- cli/resources/profile.schema.json +28 -0
- cli/resources/rule-schema.json +24 -0
- cli/resources/rule-set.json +55 -0
- cli/security.py +317 -18
- cli/security_common.py +61 -0
- cli/validator.py +108 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.10.0.dist-info}/METADATA +138 -103
- composable_data_stack-0.10.0.dist-info/RECORD +38 -0
- composable_data_stack-0.9.0.dist-info/RECORD +0 -36
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.10.0.dist-info}/WHEEL +0 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.10.0.dist-info}/entry_points.txt +0 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.10.0.dist-info}/licenses/LICENSE +0 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.10.0.dist-info}/top_level.txt +0 -0
cli/image_verification.py
CHANGED
|
@@ -156,6 +156,18 @@ def _finding(
|
|
|
156
156
|
return {
|
|
157
157
|
"rule_id": rule_id,
|
|
158
158
|
"severity": severity,
|
|
159
|
+
# These findings aren't declared in cli/resources/rule-set.json (see
|
|
160
|
+
# docs/image-signing.md for why), so their compliance category can't
|
|
161
|
+
# be looked up the way cli.security.run_security_validation() does
|
|
162
|
+
# for rule-set findings. Every check here is about image tag/digest
|
|
163
|
+
# pinning, registry trust, or signature/provenance verification --
|
|
164
|
+
# i.e. keeping deployed images current, authentic, and traceable to
|
|
165
|
+
# a trusted build -- which is exactly the use case
|
|
166
|
+
# docs/security-compliance-categories.md documents "patching" for,
|
|
167
|
+
# so tag them directly with that
|
|
168
|
+
# cli.security_common.COMPLIANCE_CATEGORIES value instead of
|
|
169
|
+
# leaving them uncategorized.
|
|
170
|
+
"category": "patching",
|
|
159
171
|
"module": service,
|
|
160
172
|
"message": message,
|
|
161
173
|
"path": f"services.{service}.image",
|
|
@@ -280,13 +292,14 @@ def _fixture_entry(
|
|
|
280
292
|
fixture: dict[str, Any] | None,
|
|
281
293
|
image_ref: str,
|
|
282
294
|
) -> dict[str, Any] | None:
|
|
283
|
-
"""Return the
|
|
295
|
+
"""Return the repository entry matching image_ref, preferring its digest."""
|
|
284
296
|
if fixture is None:
|
|
285
297
|
return None
|
|
286
298
|
images = fixture.get("images", {})
|
|
287
299
|
if not isinstance(images, dict):
|
|
288
300
|
return None
|
|
289
301
|
folded_image_ref = image_ref.casefold()
|
|
302
|
+
matches: list[dict[str, Any]] = []
|
|
290
303
|
for entry in images.values():
|
|
291
304
|
if not isinstance(entry, dict):
|
|
292
305
|
continue
|
|
@@ -297,8 +310,16 @@ def _fixture_entry(
|
|
|
297
310
|
if folded_image_ref == folded_repository or folded_image_ref.startswith(
|
|
298
311
|
(folded_repository + "@", folded_repository + ":")
|
|
299
312
|
):
|
|
300
|
-
|
|
301
|
-
|
|
313
|
+
matches.append(entry)
|
|
314
|
+
if not matches:
|
|
315
|
+
return None
|
|
316
|
+
if "@sha256:" in folded_image_ref:
|
|
317
|
+
ref_digest = folded_image_ref.rsplit("@", 1)[1]
|
|
318
|
+
for entry in matches:
|
|
319
|
+
digest = entry.get("digest")
|
|
320
|
+
if isinstance(digest, str) and digest.casefold() == ref_digest:
|
|
321
|
+
return entry
|
|
322
|
+
return matches[0]
|
|
302
323
|
|
|
303
324
|
|
|
304
325
|
def _verification_findings(
|
cli/k8s_renderer.py
CHANGED
|
@@ -1374,6 +1374,11 @@ def _values_config(value: Any) -> Any:
|
|
|
1374
1374
|
|
|
1375
1375
|
|
|
1376
1376
|
def _write_chart(root: Path, files: dict[str, str]) -> None:
|
|
1377
|
+
# Canonicalize before touching the filesystem (same convention as
|
|
1378
|
+
# `_atomic_write()`), so a caller-supplied `--output`/`--chart-dir`
|
|
1379
|
+
# value is resolved to a single, unambiguous absolute path rather than
|
|
1380
|
+
# acted on as a raw, possibly relative/traversal-bearing string.
|
|
1381
|
+
root = root.resolve()
|
|
1377
1382
|
root.parent.mkdir(parents=True, exist_ok=True)
|
|
1378
1383
|
staging = Path(tempfile.mkdtemp(prefix=f".{root.name}.tmp-", dir=root.parent))
|
|
1379
1384
|
backup = staging.with_name(staging.name.replace(".tmp-", ".previous-"))
|
cli/k8s_runner.py
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import json
|
|
5
|
+
import math
|
|
5
6
|
import os
|
|
7
|
+
import re
|
|
6
8
|
import subprocess # nosec B404
|
|
7
9
|
import tempfile
|
|
8
10
|
from pathlib import Path
|
|
@@ -13,6 +15,70 @@ import yaml
|
|
|
13
15
|
from .state import parse_k8s_workloads_json
|
|
14
16
|
from .up_runner import _validate_command, run_streamed
|
|
15
17
|
|
|
18
|
+
# RFC 1123 DNS label: lowercase alphanumeric and `-`, not leading/trailing
|
|
19
|
+
# with `-`, 1-63 chars. Both Kubernetes namespaces and Helm release names
|
|
20
|
+
# must already satisfy this, so enforcing it here rejects nothing legitimate.
|
|
21
|
+
_K8S_NAME_RE = re.compile(r"[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?")
|
|
22
|
+
|
|
23
|
+
# Kube contexts allow a wider character set (e.g. ARNs), but must not start
|
|
24
|
+
# with `-` (which would be parsed as a flag) or contain control characters.
|
|
25
|
+
_KUBE_CONTEXT_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:@/-]*")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _validate_k8s_name(value: str, label: str) -> str:
|
|
29
|
+
"""
|
|
30
|
+
Reject a `namespace`/`release` value that isn't a valid Kubernetes
|
|
31
|
+
DNS-1123 label before it reaches `helm`/`kubectl` as a command
|
|
32
|
+
argument. Without this, a value starting with `-` (e.g. supplied by a
|
|
33
|
+
misbehaving caller or automation) could be parsed as an extra flag by
|
|
34
|
+
the downstream binary instead of a plain name (CWE-88 argument
|
|
35
|
+
injection).
|
|
36
|
+
|
|
37
|
+
Returns a new string built only from the characters matched by the
|
|
38
|
+
allowlist regex (rather than the original object) so nothing beyond a
|
|
39
|
+
validated Kubernetes name can flow to the command it's used to build.
|
|
40
|
+
"""
|
|
41
|
+
match = _K8S_NAME_RE.fullmatch(value) if isinstance(value, str) else None
|
|
42
|
+
if match is None:
|
|
43
|
+
raise ValueError(f"invalid {label}: {value!r} is not a valid Kubernetes name")
|
|
44
|
+
return match.group(0)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _validate_kube_context(value: str | None) -> str | None:
|
|
48
|
+
"""
|
|
49
|
+
Reject a `kube_context` that looks like a command-line flag (starts
|
|
50
|
+
with `-`) or contains characters outside the safe allowlist, so it
|
|
51
|
+
can't be smuggled to `helm`/`kubectl` as an extra argument (CWE-88
|
|
52
|
+
argument injection).
|
|
53
|
+
|
|
54
|
+
Returns a new string built only from the allowlisted characters
|
|
55
|
+
(rather than the original object) so nothing beyond a validated
|
|
56
|
+
context name can flow to the command it's used to build.
|
|
57
|
+
"""
|
|
58
|
+
if value is None:
|
|
59
|
+
return None
|
|
60
|
+
match = _KUBE_CONTEXT_RE.fullmatch(value) if isinstance(value, str) else None
|
|
61
|
+
if match is None:
|
|
62
|
+
raise ValueError(f"invalid kube context: {value!r}")
|
|
63
|
+
return match.group(0)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _validate_timeout(value: float) -> int:
|
|
67
|
+
"""
|
|
68
|
+
Reject a `timeout` that isn't a finite, positive number before it is
|
|
69
|
+
interpolated into a `--timeout=<n>s` argument for `helm`/`kubectl`. A
|
|
70
|
+
non-finite or non-positive value (e.g. NaN, infinity, or a negative
|
|
71
|
+
number) could otherwise render as a string that isn't a plain
|
|
72
|
+
numeric suffix, which risks being parsed as an extra flag by the
|
|
73
|
+
downstream binary (CWE-88 argument injection).
|
|
74
|
+
|
|
75
|
+
Returns a newly constructed ``int`` (rather than the original tainted
|
|
76
|
+
object) clamped to at least 1 second.
|
|
77
|
+
"""
|
|
78
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value):
|
|
79
|
+
raise ValueError(f"invalid timeout: {value!r} is not a finite number")
|
|
80
|
+
return max(1, int(value))
|
|
81
|
+
|
|
16
82
|
|
|
17
83
|
def helm_up(
|
|
18
84
|
plan: dict[str, Any],
|
|
@@ -26,10 +92,14 @@ def helm_up(
|
|
|
26
92
|
log_file: IO[str],
|
|
27
93
|
) -> int:
|
|
28
94
|
"""Install or upgrade a rendered chart without persisting secret values."""
|
|
95
|
+
namespace = _validate_k8s_name(namespace, "namespace")
|
|
96
|
+
release = _validate_k8s_name(release, "release")
|
|
97
|
+
kube_context = _validate_kube_context(kube_context)
|
|
98
|
+
timeout = _validate_timeout(timeout)
|
|
29
99
|
secret_values = _secret_values(plan)
|
|
30
100
|
secret_path = _write_secret_values(secret_values)
|
|
31
101
|
context_args = ["--kube-context", kube_context] if kube_context else []
|
|
32
|
-
timeout_arg = f"{
|
|
102
|
+
timeout_arg = f"{timeout}s"
|
|
33
103
|
command = [
|
|
34
104
|
"helm",
|
|
35
105
|
*context_args,
|
|
@@ -90,6 +160,10 @@ def helm_down(
|
|
|
90
160
|
delete_pvcs: bool,
|
|
91
161
|
log_file: IO[str],
|
|
92
162
|
) -> int:
|
|
163
|
+
namespace = _validate_k8s_name(namespace, "namespace")
|
|
164
|
+
release = _validate_k8s_name(release, "release")
|
|
165
|
+
kube_context = _validate_kube_context(kube_context)
|
|
166
|
+
timeout = _validate_timeout(timeout)
|
|
93
167
|
pvc_names: list[str] = []
|
|
94
168
|
if delete_pvcs:
|
|
95
169
|
pvc_names = _stateful_pvc_names(
|
|
@@ -105,7 +179,7 @@ def helm_down(
|
|
|
105
179
|
namespace,
|
|
106
180
|
"--ignore-not-found",
|
|
107
181
|
"--timeout",
|
|
108
|
-
f"{
|
|
182
|
+
f"{timeout}s",
|
|
109
183
|
]
|
|
110
184
|
result = run_streamed(command, log_file, timeout=timeout + 30)
|
|
111
185
|
if result != 0 or not delete_pvcs or not pvc_names:
|
|
@@ -121,6 +195,9 @@ def helm_down(
|
|
|
121
195
|
def get_k8s_workloads(
|
|
122
196
|
namespace: str, release: str, kube_context: str | None
|
|
123
197
|
) -> list[dict[str, Any]]:
|
|
198
|
+
namespace = _validate_k8s_name(namespace, "namespace")
|
|
199
|
+
release = _validate_k8s_name(release, "release")
|
|
200
|
+
kube_context = _validate_kube_context(kube_context)
|
|
124
201
|
command = _kubectl_command(kube_context, namespace) + [
|
|
125
202
|
"get",
|
|
126
203
|
"deployment,statefulset,job",
|
cli/k8s_security.py
CHANGED
|
@@ -76,6 +76,7 @@ def scan_k8s_security(plan: dict[str, Any]) -> list[dict[str, Any]]:
|
|
|
76
76
|
f"{path}.resources.{container_name}",
|
|
77
77
|
module_id,
|
|
78
78
|
["Declare both requests and limits for the container."],
|
|
79
|
+
category="configuration-management",
|
|
79
80
|
))
|
|
80
81
|
return findings
|
|
81
82
|
|
|
@@ -113,10 +114,22 @@ def _finding(
|
|
|
113
114
|
path: str,
|
|
114
115
|
module: str,
|
|
115
116
|
recommendation: list[str],
|
|
117
|
+
*,
|
|
118
|
+
category: str = "system-hardening",
|
|
116
119
|
) -> dict[str, Any]:
|
|
117
120
|
return {
|
|
118
121
|
"rule_id": rule_id,
|
|
119
122
|
"severity": severity,
|
|
123
|
+
# These findings aren't declared in cli/resources/rule-set.json (the
|
|
124
|
+
# declarative rule-set engine has no scope for the plan's
|
|
125
|
+
# Kubernetes-specific implementation), so their compliance category
|
|
126
|
+
# can't be looked up the way cli.security.run_security_validation()
|
|
127
|
+
# does for rule-set findings. CDS-K8S-001-004 are container/pod
|
|
128
|
+
# runtime-hardening posture checks, so they default to the matching
|
|
129
|
+
# cli.security_common.COMPLIANCE_CATEGORIES value; CDS-K8S-005
|
|
130
|
+
# (resource requests/limits) overrides this below since it's a
|
|
131
|
+
# capacity-governance concern rather than runtime-hardening posture.
|
|
132
|
+
"category": category,
|
|
120
133
|
"message": message,
|
|
121
134
|
"path": path,
|
|
122
135
|
"module": module,
|
cli/main.py
CHANGED
|
@@ -38,7 +38,7 @@ from .planner import build_plan
|
|
|
38
38
|
from .preflight import preflight_passed, run_preflight
|
|
39
39
|
from .renderer import render_compose
|
|
40
40
|
from .security import PrecomputedRender, run_security_validation
|
|
41
|
-
from .security_common import SEVERITY_ORDER, infer_profile_class
|
|
41
|
+
from .security_common import COMPLIANCE_CATEGORIES, SEVERITY_ORDER, infer_profile_class
|
|
42
42
|
from .state import format_state_output, group_services_by_health, parse_compose_ps_json
|
|
43
43
|
from .up_runner import (
|
|
44
44
|
DEFAULT_TIMEOUT_SECONDS,
|
|
@@ -875,6 +875,58 @@ def _unverifiable_image_finding(message: str) -> dict[str, Any]:
|
|
|
875
875
|
}
|
|
876
876
|
|
|
877
877
|
|
|
878
|
+
_UNCATEGORIZED_LABEL = "uncategorized"
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _filter_findings_by_category(
|
|
882
|
+
findings: list[dict[str, Any]], categories: list[str] | None
|
|
883
|
+
) -> list[dict[str, Any]]:
|
|
884
|
+
"""
|
|
885
|
+
Restrict findings to the given compliance control categories.
|
|
886
|
+
|
|
887
|
+
Every finding source (the declarative rule-set engine, scan_k8s_security,
|
|
888
|
+
image verification) tags its findings with a "category" at generation
|
|
889
|
+
time (see cli.security.run_security_validation(), cli.k8s_security,
|
|
890
|
+
cli.image_verification), so this is a uniform key lookup; there is no
|
|
891
|
+
remaining source of genuinely uncategorized findings today, but a
|
|
892
|
+
missing "category" is still treated as excluded rather than raising, in
|
|
893
|
+
case a future finding source doesn't set one.
|
|
894
|
+
"""
|
|
895
|
+
if not categories:
|
|
896
|
+
return findings
|
|
897
|
+
wanted = set(categories)
|
|
898
|
+
return [f for f in findings if f.get("category") in wanted]
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
def _group_findings_by_category(
|
|
902
|
+
findings: list[dict[str, Any]],
|
|
903
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
904
|
+
"""
|
|
905
|
+
Group findings by their compliance control category, preserving each
|
|
906
|
+
group's existing (severity/rule_id/...) ordering. A finding without a
|
|
907
|
+
"category" is grouped under "uncategorized" rather than dropped, as a
|
|
908
|
+
defensive fallback (see _filter_findings_by_category()); in practice
|
|
909
|
+
every current finding source sets one.
|
|
910
|
+
"""
|
|
911
|
+
grouped: dict[str, list[dict[str, Any]]] = {}
|
|
912
|
+
for finding in findings:
|
|
913
|
+
grouped.setdefault(finding.get("category") or _UNCATEGORIZED_LABEL, []).append(finding)
|
|
914
|
+
return dict(sorted(grouped.items()))
|
|
915
|
+
|
|
916
|
+
|
|
917
|
+
def _category_heading(category: str) -> str:
|
|
918
|
+
"""
|
|
919
|
+
Render a "== <label> (<category>) ==" heading for --group-by-category,
|
|
920
|
+
falling back to just "== <category> ==" for the uncategorized bucket,
|
|
921
|
+
which has no COMPLIANCE_CATEGORIES entry.
|
|
922
|
+
"""
|
|
923
|
+
entry = COMPLIANCE_CATEGORIES.get(category)
|
|
924
|
+
if entry is None:
|
|
925
|
+
return f"== {category} =="
|
|
926
|
+
label, _rationale = entry
|
|
927
|
+
return f"== {label} ({category}) =="
|
|
928
|
+
|
|
929
|
+
|
|
878
930
|
def _run_image_verification(
|
|
879
931
|
profile_path: str,
|
|
880
932
|
environment: str | None,
|
|
@@ -961,7 +1013,7 @@ def _render_helm_chart(plan, output_dir, force):
|
|
|
961
1013
|
compose target this refuses to write over existing unrelated content unless
|
|
962
1014
|
the caller passes --force.
|
|
963
1015
|
"""
|
|
964
|
-
target_dir = Path(output_dir)
|
|
1016
|
+
target_dir = Path(output_dir).resolve()
|
|
965
1017
|
if target_dir.exists() and not target_dir.is_dir():
|
|
966
1018
|
if not force:
|
|
967
1019
|
print(f"ERROR Helm output path {target_dir} exists and is not a directory. Pass --force to replace it.")
|
|
@@ -1159,6 +1211,21 @@ def main() -> int:
|
|
|
1159
1211
|
"By default, values are redacted to avoid echoing real secrets to stdout/CI logs."
|
|
1160
1212
|
),
|
|
1161
1213
|
)
|
|
1214
|
+
test_parser.add_argument(
|
|
1215
|
+
"--category",
|
|
1216
|
+
action="append",
|
|
1217
|
+
choices=COMPLIANCE_CATEGORIES,
|
|
1218
|
+
help=(
|
|
1219
|
+
"Only print security findings tagged with this compliance control "
|
|
1220
|
+
"category (repeatable); does not affect the security stage's "
|
|
1221
|
+
"pass/fail outcome. See `cds security --help` for details."
|
|
1222
|
+
),
|
|
1223
|
+
)
|
|
1224
|
+
test_parser.add_argument(
|
|
1225
|
+
"--group-by-category",
|
|
1226
|
+
action="store_true",
|
|
1227
|
+
help="Print security findings grouped by compliance control category instead of one flat list.",
|
|
1228
|
+
)
|
|
1162
1229
|
_add_hardened_arg(test_parser)
|
|
1163
1230
|
_add_image_source_arg(test_parser)
|
|
1164
1231
|
|
|
@@ -1283,6 +1350,25 @@ def main() -> int:
|
|
|
1283
1350
|
"tests/fixtures/signed-images.json) for offline verification."
|
|
1284
1351
|
),
|
|
1285
1352
|
)
|
|
1353
|
+
security_parser.add_argument(
|
|
1354
|
+
"--category",
|
|
1355
|
+
action="append",
|
|
1356
|
+
choices=COMPLIANCE_CATEGORIES,
|
|
1357
|
+
help=(
|
|
1358
|
+
"Only print findings tagged with this compliance control category "
|
|
1359
|
+
"(repeatable); does not change which rules run or the command's "
|
|
1360
|
+
"exit code -- a high-severity finding outside the requested "
|
|
1361
|
+
"category still fails the scan. Categories are an informational "
|
|
1362
|
+
"readiness aid for organizing findings against a user's own risk "
|
|
1363
|
+
"assessment (e.g. NIS2/Cyberbeveiligingswet), not a compliance "
|
|
1364
|
+
"certification -- see docs/security-compliance-categories.md."
|
|
1365
|
+
),
|
|
1366
|
+
)
|
|
1367
|
+
security_parser.add_argument(
|
|
1368
|
+
"--group-by-category",
|
|
1369
|
+
action="store_true",
|
|
1370
|
+
help="Print findings grouped by compliance control category instead of one flat list.",
|
|
1371
|
+
)
|
|
1286
1372
|
|
|
1287
1373
|
diff_parser = subparsers.add_parser(
|
|
1288
1374
|
"diff",
|
|
@@ -1610,7 +1696,7 @@ def main() -> int:
|
|
|
1610
1696
|
|
|
1611
1697
|
if args.target == "helm":
|
|
1612
1698
|
project_root = resolve_project_root(profile_path)
|
|
1613
|
-
chart_dir = Path(args.chart_dir) if args.chart_dir else project_root / "chart"
|
|
1699
|
+
chart_dir = (Path(args.chart_dir) if args.chart_dir else project_root / "chart").resolve()
|
|
1614
1700
|
code, render_diags = _render_helm_chart(plan, chart_dir, force=True)
|
|
1615
1701
|
all_diags.extend(render_diags)
|
|
1616
1702
|
if code != 0 or has_errors(all_diags):
|
|
@@ -1620,9 +1706,9 @@ def main() -> int:
|
|
|
1620
1706
|
|
|
1621
1707
|
namespace = args.namespace or plan.get("runtime", {}).get("namespace") or "cds-local"
|
|
1622
1708
|
release = args.release or plan.get("metadata", {}).get("name") or "cds"
|
|
1623
|
-
log_path =
|
|
1624
|
-
Path(profile_path).parent.name
|
|
1625
|
-
)
|
|
1709
|
+
log_path = (
|
|
1710
|
+
Path(args.log_file) if args.log_file else default_log_path(Path(profile_path).parent.name)
|
|
1711
|
+
).resolve()
|
|
1626
1712
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1627
1713
|
try:
|
|
1628
1714
|
with open(log_path, "a", encoding="utf-8") as log_file:
|
|
@@ -1678,6 +1764,7 @@ def main() -> int:
|
|
|
1678
1764
|
log_path = Path(args.log_file)
|
|
1679
1765
|
else:
|
|
1680
1766
|
log_path = default_log_path(Path(profile_path).parent.name)
|
|
1767
|
+
log_path = log_path.resolve()
|
|
1681
1768
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1682
1769
|
|
|
1683
1770
|
log_tail_process = None
|
|
@@ -1837,9 +1924,9 @@ def main() -> int:
|
|
|
1837
1924
|
return 1
|
|
1838
1925
|
project_root = resolve_project_root(profile_path)
|
|
1839
1926
|
profile_release, profile_namespace = _k8s_runtime_defaults(profile_path)
|
|
1840
|
-
log_path =
|
|
1841
|
-
f"down-{Path(profile_path).parent.name}"
|
|
1842
|
-
)
|
|
1927
|
+
log_path = (
|
|
1928
|
+
Path(args.log_file) if args.log_file else default_log_path(f"down-{Path(profile_path).parent.name}")
|
|
1929
|
+
).resolve()
|
|
1843
1930
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1844
1931
|
try:
|
|
1845
1932
|
with open(log_path, "a", encoding="utf-8") as log_file:
|
|
@@ -1940,14 +2027,29 @@ def main() -> int:
|
|
|
1940
2027
|
))
|
|
1941
2028
|
for diag in sec_diags:
|
|
1942
2029
|
print(diag.format(), file=sys.stderr)
|
|
1943
|
-
|
|
1944
|
-
|
|
2030
|
+
displayed_findings = _filter_findings_by_category(findings, args.category)
|
|
2031
|
+
if args.group_by_category:
|
|
2032
|
+
for category, group in _group_findings_by_category(displayed_findings).items():
|
|
2033
|
+
print(_category_heading(category))
|
|
2034
|
+
for f in group:
|
|
2035
|
+
print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
|
|
2036
|
+
else:
|
|
2037
|
+
for f in displayed_findings:
|
|
2038
|
+
print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
|
|
1945
2039
|
# A W096 warning means rendered-compose-scoped rules (e.g.
|
|
1946
2040
|
# CDS-SEC-070) were silently skipped due to an unexpected
|
|
1947
2041
|
# error during rendering. Treat that the same as a failed
|
|
1948
2042
|
# security stage rather than letting any non-high finding
|
|
1949
2043
|
# (or no finding at all) mask the fact that some checks
|
|
1950
2044
|
# never ran (GHSA-mx5p-cv63-6829).
|
|
2045
|
+
#
|
|
2046
|
+
# security_ok is intentionally computed from the full,
|
|
2047
|
+
# unfiltered `findings`, not `displayed_findings`: --category
|
|
2048
|
+
# is a display-only filter (see docs/security-compliance-
|
|
2049
|
+
# categories.md), so narrowing what's printed must never
|
|
2050
|
+
# narrow what counts as a failing scan -- that would let a
|
|
2051
|
+
# real high-severity finding outside the requested category
|
|
2052
|
+
# silently report success.
|
|
1951
2053
|
render_scan_skipped = any(d.code == "W096" for d in sec_diags)
|
|
1952
2054
|
security_ok = (
|
|
1953
2055
|
not render_scan_skipped
|
|
@@ -2292,15 +2394,38 @@ def main() -> int:
|
|
|
2292
2394
|
print("No security findings.")
|
|
2293
2395
|
return 0
|
|
2294
2396
|
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2397
|
+
# --category is a display-only filter (see docs/security-compliance-
|
|
2398
|
+
# categories.md): it must never narrow what counts as a failing scan,
|
|
2399
|
+
# only what gets printed. The exit code below is deliberately
|
|
2400
|
+
# computed from the full, unfiltered `findings`, not
|
|
2401
|
+
# `displayed_findings` -- otherwise a real high-severity finding
|
|
2402
|
+
# outside the requested category would silently report success.
|
|
2403
|
+
displayed_findings = _filter_findings_by_category(findings, args.category)
|
|
2404
|
+
|
|
2405
|
+
if not displayed_findings:
|
|
2406
|
+
print(f"No security findings in category: {', '.join(sorted(set(args.category)))}.")
|
|
2407
|
+
elif args.group_by_category:
|
|
2408
|
+
for category, group in _group_findings_by_category(displayed_findings).items():
|
|
2409
|
+
print(_category_heading(category))
|
|
2410
|
+
for f in group:
|
|
2411
|
+
print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
|
|
2412
|
+
print(f" object: {f['path']}")
|
|
2413
|
+
print(f" module: {f['module']}")
|
|
2414
|
+
if f["value"] is not None:
|
|
2415
|
+
print(f" value: {f['value']}")
|
|
2416
|
+
for rec in f["recommendation"]:
|
|
2417
|
+
print(f" fix: {rec}")
|
|
2418
|
+
print()
|
|
2419
|
+
else:
|
|
2420
|
+
for f in displayed_findings:
|
|
2421
|
+
print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
|
|
2422
|
+
print(f" object: {f['path']}")
|
|
2423
|
+
print(f" module: {f['module']}")
|
|
2424
|
+
if f["value"] is not None:
|
|
2425
|
+
print(f" value: {f['value']}")
|
|
2426
|
+
for rec in f["recommendation"]:
|
|
2427
|
+
print(f" fix: {rec}")
|
|
2428
|
+
print()
|
|
2304
2429
|
|
|
2305
2430
|
# As above: a W096 warning means some rendered-compose-scoped rules
|
|
2306
2431
|
# were skipped, so even when only non-high findings are present the
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "compatibility-registry.schema.json",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"metadata": {
|
|
5
|
+
"name": "cds-core-compatibility-registry",
|
|
6
|
+
"description": "Central record of module pairings known to have been tested together (\"tested\"), or known NOT to work together despite matching a contract's kind (\"unsupported\"). Kept as a single file rather than scattered across individual module.yaml documents so composition knowledge about the whole stack stays in one place (see issue #350). A pairing with no entry here is only structurally compatible (same contract kind); it is neither vouched for nor ruled out."
|
|
7
|
+
},
|
|
8
|
+
"pairings": [
|
|
9
|
+
{
|
|
10
|
+
"contract": "sql-database",
|
|
11
|
+
"provider": "warehouse/postgres",
|
|
12
|
+
"consumer": "orchestration/dagster",
|
|
13
|
+
"status": "tested",
|
|
14
|
+
"notes": "Exercised end-to-end by profiles/local-dagster-postgres-superset/profile.yaml, covered by the \"CDS Test (Stable Profile)\" CI workflow."
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"contract": "sql-database",
|
|
18
|
+
"provider": "warehouse/postgres",
|
|
19
|
+
"consumer": "bi/superset",
|
|
20
|
+
"status": "tested",
|
|
21
|
+
"notes": "Exercised end-to-end by profiles/local-dagster-postgres-superset/profile.yaml, covered by the \"CDS Test (Stable Profile)\" CI workflow."
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"contract": "cache-service",
|
|
25
|
+
"provider": "cache/keydb",
|
|
26
|
+
"consumer": "bi/superset",
|
|
27
|
+
"status": "tested",
|
|
28
|
+
"notes": "Exercised end-to-end by profiles/local-dagster-postgres-superset/profile.yaml, covered by the \"CDS Test (Stable Profile)\" CI workflow."
|
|
29
|
+
}
|
|
30
|
+
]
|
|
31
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
3
|
+
"$id": "./compatibility-registry.schema.json",
|
|
4
|
+
"title": "Composable Data Stack Compatibility Registry",
|
|
5
|
+
"description": "Central, non-scattered record of module pairings for a given contract that are known to have been tested together, or explicitly known NOT to work together. This is deliberately separate from plain contract `kind` matching (structural compatibility): an entry here expresses practical, known-good/known-bad compatibility, not just shape compatibility. Absence of an entry for a pairing means 'not yet recorded' -- it is neither tested nor unsupported.",
|
|
6
|
+
"type": "object",
|
|
7
|
+
"additionalProperties": false,
|
|
8
|
+
"required": ["version", "metadata", "pairings"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"$schema": {
|
|
11
|
+
"type": "string"
|
|
12
|
+
},
|
|
13
|
+
"version": {
|
|
14
|
+
"type": "string"
|
|
15
|
+
},
|
|
16
|
+
"metadata": {
|
|
17
|
+
"type": "object",
|
|
18
|
+
"additionalProperties": false,
|
|
19
|
+
"required": ["name"],
|
|
20
|
+
"properties": {
|
|
21
|
+
"name": {
|
|
22
|
+
"type": "string",
|
|
23
|
+
"minLength": 1
|
|
24
|
+
},
|
|
25
|
+
"description": {
|
|
26
|
+
"type": "string"
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"pairings": {
|
|
31
|
+
"type": "array",
|
|
32
|
+
"items": {
|
|
33
|
+
"$ref": "#/$defs/pairing"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
"$defs": {
|
|
38
|
+
"pairing": {
|
|
39
|
+
"type": "object",
|
|
40
|
+
"additionalProperties": false,
|
|
41
|
+
"required": ["contract", "provider", "consumer", "status"],
|
|
42
|
+
"properties": {
|
|
43
|
+
"contract": {
|
|
44
|
+
"type": "string",
|
|
45
|
+
"pattern": "^[a-z0-9-]+$",
|
|
46
|
+
"description": "Contract kind this pairing applies to (matches a module's provides[].contract.kind / consumes[].contract.kind), not a specific provide-name. A provider can expose several provide-names under the same kind (e.g. warehouse/postgres provides \"sql-database\", \"dagster-database\", and \"superset-database\", all kind \"sql-database\"); one row here is intended to cover the provider/consumer pair for all of them at this kind granularity. A row also carries no module version, so a \"tested\" status remains valid across version changes on either side (tracked as a follow-up, see issue #750)."
|
|
47
|
+
},
|
|
48
|
+
"provider": {
|
|
49
|
+
"type": "string",
|
|
50
|
+
"pattern": "^[a-z0-9-]+/[a-z0-9-]+$",
|
|
51
|
+
"description": "Providing module's identity, as \"<metadata.category>/<metadata.name>\" (e.g. \"warehouse/postgres\"). Stable across profiles, unlike a profile-scoped module instance id or a source path."
|
|
52
|
+
},
|
|
53
|
+
"consumer": {
|
|
54
|
+
"type": "string",
|
|
55
|
+
"pattern": "^[a-z0-9-]+/[a-z0-9-]+$",
|
|
56
|
+
"description": "Consuming module's identity, as \"<metadata.category>/<metadata.name>\" (e.g. \"orchestration/dagster\")."
|
|
57
|
+
},
|
|
58
|
+
"status": {
|
|
59
|
+
"type": "string",
|
|
60
|
+
"enum": ["tested", "unsupported"],
|
|
61
|
+
"description": "\"tested\": this exact provider/consumer pairing for this contract has been exercised together (e.g. by a profile under active CI coverage) and is known-good. \"unsupported\": this pairing is structurally compatible (same contract kind) but has been found NOT to work together in practice; validation reports this as an error."
|
|
62
|
+
},
|
|
63
|
+
"notes": {
|
|
64
|
+
"type": "string",
|
|
65
|
+
"description": "Free-text context, e.g. which profile/test exercises a \"tested\" pairing, or why an \"unsupported\" pairing fails."
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
@@ -154,6 +154,21 @@
|
|
|
154
154
|
}
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
|
+
},
|
|
158
|
+
"security": {
|
|
159
|
+
"type": "object",
|
|
160
|
+
"additionalProperties": false,
|
|
161
|
+
"properties": {
|
|
162
|
+
"waivers": {
|
|
163
|
+
"type": "object",
|
|
164
|
+
"additionalProperties": false,
|
|
165
|
+
"properties": {
|
|
166
|
+
"plaintextEndpointExposure": {
|
|
167
|
+
"$ref": "#/$defs/plaintextEndpointExposureWaiver"
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
157
172
|
}
|
|
158
173
|
}
|
|
159
174
|
}
|
|
@@ -231,6 +246,19 @@
|
|
|
231
246
|
"default": "bridge"
|
|
232
247
|
}
|
|
233
248
|
}
|
|
249
|
+
},
|
|
250
|
+
"plaintextEndpointExposureWaiver": {
|
|
251
|
+
"type": "object",
|
|
252
|
+
"additionalProperties": false,
|
|
253
|
+
"required": ["reason"],
|
|
254
|
+
"properties": {
|
|
255
|
+
"reason": {
|
|
256
|
+
"type": "string",
|
|
257
|
+
"minLength": 1,
|
|
258
|
+
"pattern": "\\S",
|
|
259
|
+
"description": "Must contain at least one non-whitespace character; cli/security.py strips and treats an all-whitespace reason as unset (no waiver applied, so CDS-SEC-074 still fires), so a schema that only enforced minLength: 1 would silently accept a value the code then discards."
|
|
260
|
+
}
|
|
261
|
+
}
|
|
234
262
|
}
|
|
235
263
|
}
|
|
236
264
|
}
|
cli/resources/rule-schema.json
CHANGED
|
@@ -49,6 +49,20 @@
|
|
|
49
49
|
"observability-leakage"
|
|
50
50
|
]
|
|
51
51
|
},
|
|
52
|
+
"complianceCategory": {
|
|
53
|
+
"type": "string",
|
|
54
|
+
"description": "Informational compliance control category this rule's finding maps onto (e.g. for a user's own NIS2/Cyberbeveiligingswet risk assessment). This is a readiness aid, not a legal or regulatory certification of the rule or category -- see docs/security-compliance-categories.md.",
|
|
55
|
+
"enum": [
|
|
56
|
+
"access-control",
|
|
57
|
+
"secrets-management",
|
|
58
|
+
"network-exposure",
|
|
59
|
+
"encryption-in-transit",
|
|
60
|
+
"configuration-management",
|
|
61
|
+
"system-hardening",
|
|
62
|
+
"logging-monitoring",
|
|
63
|
+
"patching"
|
|
64
|
+
]
|
|
65
|
+
},
|
|
52
66
|
"scopeItem": {
|
|
53
67
|
"type": "string",
|
|
54
68
|
"enum": [
|
|
@@ -183,6 +197,7 @@
|
|
|
183
197
|
"title",
|
|
184
198
|
"severity",
|
|
185
199
|
"category",
|
|
200
|
+
"complianceCategory",
|
|
186
201
|
"scope",
|
|
187
202
|
"match",
|
|
188
203
|
"message",
|
|
@@ -207,6 +222,9 @@
|
|
|
207
222
|
"category": {
|
|
208
223
|
"$ref": "#/$defs/category"
|
|
209
224
|
},
|
|
225
|
+
"complianceCategory": {
|
|
226
|
+
"$ref": "#/$defs/complianceCategory"
|
|
227
|
+
},
|
|
210
228
|
"scope": {
|
|
211
229
|
"type": "array",
|
|
212
230
|
"minItems": 1,
|
|
@@ -231,6 +249,11 @@
|
|
|
231
249
|
"enabled": {
|
|
232
250
|
"type": "boolean",
|
|
233
251
|
"default": true
|
|
252
|
+
},
|
|
253
|
+
"codeEnforced": {
|
|
254
|
+
"type": "boolean",
|
|
255
|
+
"default": true,
|
|
256
|
+
"description": "For a scope: [\"none\"] rule that is actually enforced entirely in code (not by the declarative match engine -- see the rule's $comment), this is the real on/off switch a custom rule set can flip to disable that code path. `enabled` stays false on such rules per the scope-none convention (it can never produce a declarative-engine finding), so `enabled` alone cannot represent whether the code-level check is active."
|
|
234
257
|
}
|
|
235
258
|
}
|
|
236
259
|
}
|
|
@@ -249,6 +272,7 @@
|
|
|
249
272
|
"title": "Hardcoded secret-like value",
|
|
250
273
|
"severity": "high",
|
|
251
274
|
"category": "secrets",
|
|
275
|
+
"complianceCategory": "secrets-management",
|
|
252
276
|
"scope": ["profile-raw", "module-values", "service-env"],
|
|
253
277
|
"match": {
|
|
254
278
|
"all": [
|