composable-data-stack 0.9.1__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {composable_data_stack-0.9.1/composable_data_stack.egg-info → composable_data_stack-0.11.0}/PKG-INFO +8 -6
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/README.md +6 -4
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/getter.py +3 -1
- composable_data_stack-0.11.0/cli/image_digest_check.py +171 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/image_verification.py +28 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/k8s_runner.py +100 -29
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/k8s_security.py +13 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/loader.py +2 -2
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/main.py +479 -30
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/overlay.py +139 -24
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/planner.py +144 -7
- composable_data_stack-0.11.0/cli/report.py +253 -0
- composable_data_stack-0.11.0/cli/resources/compatibility-registry.json +31 -0
- composable_data_stack-0.11.0/cli/resources/compatibility-registry.schema.json +70 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resources/profile.schema.json +28 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resources/rule-schema.json +27 -1
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resources/rule-set.json +103 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/security.py +455 -18
- composable_data_stack-0.11.0/cli/security_common.py +156 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/up_runner.py +88 -36
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/validator.py +159 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0/composable_data_stack.egg-info}/PKG-INFO +8 -6
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/composable_data_stack.egg-info/SOURCES.txt +11 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/composable_data_stack.egg-info/requires.txt +1 -1
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/pyproject.toml +2 -2
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_cds_workflow.py +85 -0
- composable_data_stack-0.11.0/tests/test_check_roadmap_freshness.py +203 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_compose_to_module.py +11 -0
- composable_data_stack-0.11.0/tests/test_generate_release_inventory.py +197 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_getter.py +16 -0
- composable_data_stack-0.11.0/tests/test_image_digest_check.py +225 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_image_verification.py +13 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_k8s_runner.py +61 -1
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_k8s_security.py +28 -0
- composable_data_stack-0.11.0/tests/test_keycloak_postgres_wiring.py +169 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_loader.py +13 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_main.py +519 -1
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_overlay.py +184 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_planner.py +194 -0
- composable_data_stack-0.11.0/tests/test_release_notes_support.py +79 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_renderer.py +26 -0
- composable_data_stack-0.11.0/tests/test_report.py +525 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_security.py +559 -1
- composable_data_stack-0.11.0/tests/test_validate_cli_sbom.py +145 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_validator.py +238 -1
- composable_data_stack-0.9.1/cli/security_common.py +0 -40
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/LICENSE +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/__init__.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/constants.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/diagnostics.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/graph.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/image_updates.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/k8s_renderer.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/preflight.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/renderer.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resolver.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resources/__init__.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resources/contract.schema.json +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/resources/module.schema.json +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/secrets.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/state.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/cli/utils.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/composable_data_stack.egg-info/dependency_links.txt +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/composable_data_stack.egg-info/entry_points.txt +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/composable_data_stack.egg-info/top_level.txt +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/setup.cfg +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_ai_profile_review.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_bump_changelog.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_completion.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_compose_runtime_smoke.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_dagster_config_generation.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_dagster_definitions_uri.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_dagster_entrypoint_backend_guard.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_dagster_hardening.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_dbt_hardening.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_dlt_postgres_wiring.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_duckdb_hardening.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_environment_classification.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_environment_cli.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_graph.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_image_security_scan_workflow.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_image_updates.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_k3d_e2e_profile.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_k3d_exposure.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_k8s_renderer.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_markdown_mermaid_blocks.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_module_isolation.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_modules_no_committed_secrets.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_mvp_proof_workflow.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_preflight.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_publish_images_workflow.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_python_version_alignment.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_release_version_check.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_render_example_profile.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_resolve_profile_path.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_resolver.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_secrets.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_smoke_example_profile.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_standalone_module_profile.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_state.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_superset_hardening.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_tender_analytics.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_tender_dashboard.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_test_runner_timeout.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_trivyignore.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_up_runner.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_upstream_image_hardening.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_utils.py +0 -0
- {composable_data_stack-0.9.1 → composable_data_stack-0.11.0}/tests/test_vendored_llm_client.py +0 -0
{composable_data_stack-0.9.1/composable_data_stack.egg-info → composable_data_stack-0.11.0}/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: composable-data-stack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: CLI for validating and composing composable data stack profiles
|
|
5
5
|
Author: Ronald Hensbergen
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -32,7 +32,7 @@ Provides-Extra: dev
|
|
|
32
32
|
Requires-Dist: coverage>=7.6; extra == "dev"
|
|
33
33
|
Requires-Dist: bandit>=1.7; extra == "dev"
|
|
34
34
|
Requires-Dist: pip-audit>=2.7; extra == "dev"
|
|
35
|
-
Requires-Dist: ruff==0.16.
|
|
35
|
+
Requires-Dist: ruff==0.16.10; extra == "dev"
|
|
36
36
|
Dynamic: license-file
|
|
37
37
|
|
|
38
38
|
# 🚀 Composable Data Stack (CDS)
|
|
@@ -892,6 +892,7 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
|
|
|
892
892
|
| `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
|
|
893
893
|
| `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
|
|
894
894
|
| `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
|
|
895
|
+
| `[E043] ... recorded as unsupported in the compatibility registry` | The binding's `kind` matches, but this exact provider/consumer pairing is explicitly recorded as `unsupported` in `cli/resources/compatibility-registry.json` -- it's known NOT to work together despite matching shape. | Use a different provider/consumer pairing for this contract, or if the pairing has since been fixed/verified, update its `status` to `tested` in the registry (see [docs/architecture.md § Secrets and contract resolution](docs/architecture.md#secrets-and-contract-resolution)). |
|
|
895
896
|
| `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
|
|
896
897
|
| `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
|
|
897
898
|
|
|
@@ -1033,9 +1034,9 @@ Please read these first:
|
|
|
1033
1034
|
|
|
1034
1035
|
- [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
1035
1036
|
- [docs/maintainer-merge-policy.md](docs/maintainer-merge-policy.md)
|
|
1036
|
-
- [
|
|
1037
|
-
- [
|
|
1038
|
-
-
|
|
1037
|
+
- [SECURITY.md](SECURITY.md) — how to report a vulnerability
|
|
1038
|
+
- [LEGAL.md](LEGAL.md) — license, code of conduct, security, support, and
|
|
1039
|
+
regulatory-compliance documentation index
|
|
1039
1040
|
- [CHANGELOG.md](CHANGELOG.md)
|
|
1040
1041
|
- [RELEASE.md](RELEASE.md)
|
|
1041
1042
|
|
|
@@ -1058,4 +1059,5 @@ Development helper tools are located in the gitignored `tools/` directory. See
|
|
|
1058
1059
|
|
|
1059
1060
|
## 📜 License
|
|
1060
1061
|
|
|
1061
|
-
See `LICENSE`.
|
|
1062
|
+
See `LICENSE`. For security, support, and regulatory-compliance
|
|
1063
|
+
documentation, see [LEGAL.md](LEGAL.md).
|
|
@@ -855,6 +855,7 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
|
|
|
855
855
|
| `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
|
|
856
856
|
| `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
|
|
857
857
|
| `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
|
|
858
|
+
| `[E043] ... recorded as unsupported in the compatibility registry` | The binding's `kind` matches, but this exact provider/consumer pairing is explicitly recorded as `unsupported` in `cli/resources/compatibility-registry.json` -- it's known NOT to work together despite matching shape. | Use a different provider/consumer pairing for this contract, or if the pairing has since been fixed/verified, update its `status` to `tested` in the registry (see [docs/architecture.md § Secrets and contract resolution](docs/architecture.md#secrets-and-contract-resolution)). |
|
|
858
859
|
| `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
|
|
859
860
|
| `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
|
|
860
861
|
|
|
@@ -996,9 +997,9 @@ Please read these first:
|
|
|
996
997
|
|
|
997
998
|
- [CONTRIBUTING.md](CONTRIBUTING.md)
|
|
998
999
|
- [docs/maintainer-merge-policy.md](docs/maintainer-merge-policy.md)
|
|
999
|
-
- [
|
|
1000
|
-
- [
|
|
1001
|
-
-
|
|
1000
|
+
- [SECURITY.md](SECURITY.md) — how to report a vulnerability
|
|
1001
|
+
- [LEGAL.md](LEGAL.md) — license, code of conduct, security, support, and
|
|
1002
|
+
regulatory-compliance documentation index
|
|
1002
1003
|
- [CHANGELOG.md](CHANGELOG.md)
|
|
1003
1004
|
- [RELEASE.md](RELEASE.md)
|
|
1004
1005
|
|
|
@@ -1021,4 +1022,5 @@ Development helper tools are located in the gitignored `tools/` directory. See
|
|
|
1021
1022
|
|
|
1022
1023
|
## 📜 License
|
|
1023
1024
|
|
|
1024
|
-
See `LICENSE`.
|
|
1025
|
+
See `LICENSE`. For security, support, and regulatory-compliance
|
|
1026
|
+
documentation, see [LEGAL.md](LEGAL.md).
|
|
@@ -154,11 +154,13 @@ def _prepare_source_repository(
|
|
|
154
154
|
defaults to this project's upstream repository, and any `owner/repo` or
|
|
155
155
|
`github.com/...` value is fetched as a tarball for `ref`. Pass `local` to
|
|
156
156
|
explicitly use an existing local directory instead (e.g. an offline/dev
|
|
157
|
-
checkout) -- `remote
|
|
157
|
+
checkout) -- `remote` and a non-default `ref` are rejected in that case.
|
|
158
158
|
"""
|
|
159
159
|
if local is not None:
|
|
160
160
|
if remote is not None:
|
|
161
161
|
raise GetError("Specify only one of --remote and --local")
|
|
162
|
+
if ref != DEFAULT_REF:
|
|
163
|
+
raise GetError(f"--ref {ref!r} is ignored with --local: specify only one of --local and --ref")
|
|
162
164
|
yield _validate_source_repository(Path(local).expanduser())
|
|
163
165
|
return
|
|
164
166
|
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# cli/image_digest_check.py
|
|
2
|
+
"""
|
|
3
|
+
Pinned image digest staleness check (issue #736).
|
|
4
|
+
|
|
5
|
+
Modules pin third-party base images by digest (e.g.
|
|
6
|
+
`postgres:18@sha256:...`) so deploys are reproducible. When the upstream
|
|
7
|
+
registry republishes the same tag with a new digest (a patched rebuild),
|
|
8
|
+
that pin silently goes stale until someone notices out-of-band. This module
|
|
9
|
+
adds an opt-in, network-gated check that compares a module's pinned digest
|
|
10
|
+
against the digest the registry currently serves for that tag, and reports
|
|
11
|
+
a stable warning diagnostic (W100) when they differ.
|
|
12
|
+
|
|
13
|
+
The check is off by default so `cds validate`/`cds up` stay usable offline
|
|
14
|
+
and in CI without network access; it is enabled explicitly via
|
|
15
|
+
`--check-image-digests` or the CDS_CHECK_IMAGE_DIGESTS=1 environment
|
|
16
|
+
variable. Any lookup failure (offline, auth failure, unknown tag, ...) is
|
|
17
|
+
reported as "skipped", never as an error, per the issue's acceptance
|
|
18
|
+
criteria.
|
|
19
|
+
"""
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
from typing import Any
|
|
26
|
+
from urllib.error import HTTPError, URLError
|
|
27
|
+
from urllib.request import Request, urlopen
|
|
28
|
+
|
|
29
|
+
from .image_updates import find_images_in_compose, parse_image_reference
|
|
30
|
+
|
|
31
|
+
ENV_VAR = "CDS_CHECK_IMAGE_DIGESTS"
|
|
32
|
+
_TIMEOUT = 10
|
|
33
|
+
_MANIFEST_ACCEPT = ", ".join(
|
|
34
|
+
[
|
|
35
|
+
"application/vnd.oci.image.index.v1+json",
|
|
36
|
+
"application/vnd.oci.image.manifest.v1+json",
|
|
37
|
+
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
38
|
+
"application/vnd.docker.distribution.manifest.v2+json",
|
|
39
|
+
]
|
|
40
|
+
)
|
|
41
|
+
_USER_AGENT = "cds-digest-check/1.0"
|
|
42
|
+
|
|
43
|
+
_PINNED_DIGEST_PATTERN = re.compile(r"^([^@]+)@(sha256:[a-f0-9]{64})$")
|
|
44
|
+
_AUTH_PARAM_PATTERN = re.compile(r'(\w+)="([^"]*)"')
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def is_enabled(explicit: bool | None = None) -> bool:
|
|
48
|
+
"""
|
|
49
|
+
Resolve whether the staleness check should run: an explicit CLI flag
|
|
50
|
+
wins, otherwise CDS_CHECK_IMAGE_DIGESTS=1/true/yes/on opts in. Off by
|
|
51
|
+
default so validate/up/render stay network-free.
|
|
52
|
+
"""
|
|
53
|
+
if explicit:
|
|
54
|
+
return True
|
|
55
|
+
return os.getenv(ENV_VAR, "").strip().lower() in ("1", "true", "yes", "on")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def parse_pinned_reference(image: str) -> tuple[str, str] | None:
|
|
59
|
+
"""Split "repo:tag@sha256:..." into ("repo:tag", "sha256:...")."""
|
|
60
|
+
match = _PINNED_DIGEST_PATTERN.match(image)
|
|
61
|
+
if not match:
|
|
62
|
+
return None
|
|
63
|
+
return match.group(1), match.group(2)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _registry_base_url(registry: str) -> str:
|
|
67
|
+
if registry == "docker.io":
|
|
68
|
+
return "https://registry-1.docker.io"
|
|
69
|
+
return f"https://{registry}"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _fetch_bearer_token(www_authenticate: str) -> str | None:
|
|
73
|
+
"""Follow the OAuth2 token challenge described by a 401 response's
|
|
74
|
+
WWW-Authenticate header (realm/service/scope), per the Docker Registry
|
|
75
|
+
v2 auth spec."""
|
|
76
|
+
if not www_authenticate.lower().startswith("bearer"):
|
|
77
|
+
return None
|
|
78
|
+
params = dict(_AUTH_PARAM_PATTERN.findall(www_authenticate))
|
|
79
|
+
realm = params.get("realm")
|
|
80
|
+
if not realm:
|
|
81
|
+
return None
|
|
82
|
+
query = "&".join(f"{key}={value}" for key, value in params.items() if key != "realm")
|
|
83
|
+
url = f"{realm}?{query}" if query else realm
|
|
84
|
+
if not url.startswith(("http://", "https://")):
|
|
85
|
+
return None
|
|
86
|
+
try:
|
|
87
|
+
req = Request(url, headers={"User-Agent": _USER_AGENT}) # nosec B310 # noqa: S310
|
|
88
|
+
# This check is opt-in and only ever targets registries already
|
|
89
|
+
# referenced by a module's own pinned image (not arbitrary
|
|
90
|
+
# user/CLI input). The scheme is restricted to http(s) above, but
|
|
91
|
+
# the realm host itself is not otherwise restricted: a
|
|
92
|
+
# compromised/malicious registry could point "realm" at an
|
|
93
|
+
# internal address. The request carries no credentials and only
|
|
94
|
+
# a bearer token string is read back from the (JSON-parsed)
|
|
95
|
+
# response, bounding the impact to an anonymous GET against a
|
|
96
|
+
# registry-controlled URL rather than to full SSRF.
|
|
97
|
+
with urlopen(req, timeout=_TIMEOUT) as response: # nosec B310 # noqa: S310
|
|
98
|
+
data = json.loads(response.read().decode())
|
|
99
|
+
except (HTTPError, URLError, ValueError):
|
|
100
|
+
return None
|
|
101
|
+
token = data.get("token") or data.get("access_token")
|
|
102
|
+
return token if isinstance(token, str) else None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def fetch_registry_digest(registry: str, repository: str, tag: str) -> str | None:
|
|
106
|
+
"""
|
|
107
|
+
Return the manifest digest the registry currently serves for
|
|
108
|
+
repository:tag, or None if it cannot be determined (offline, auth
|
|
109
|
+
failure, unknown tag, unsupported registry response, ...).
|
|
110
|
+
"""
|
|
111
|
+
base = _registry_base_url(registry)
|
|
112
|
+
url = f"{base}/v2/{repository}/manifests/{tag}"
|
|
113
|
+
headers = {"Accept": _MANIFEST_ACCEPT, "User-Agent": _USER_AGENT}
|
|
114
|
+
|
|
115
|
+
for attempt in range(2):
|
|
116
|
+
try:
|
|
117
|
+
req = Request(url, headers=headers, method="HEAD") # nosec B310 # noqa: S310
|
|
118
|
+
# url is built from a schema-validated registry/repository/tag
|
|
119
|
+
# triple, always http(s).
|
|
120
|
+
with urlopen(req, timeout=_TIMEOUT) as response: # nosec B310 # noqa: S310
|
|
121
|
+
digest = response.headers.get("Docker-Content-Digest")
|
|
122
|
+
return digest if isinstance(digest, str) and digest else None
|
|
123
|
+
except HTTPError as exc:
|
|
124
|
+
if exc.code == 401 and attempt == 0:
|
|
125
|
+
www_auth = exc.headers.get("WWW-Authenticate", "") if exc.headers else ""
|
|
126
|
+
token = _fetch_bearer_token(www_auth)
|
|
127
|
+
if token:
|
|
128
|
+
headers["Authorization"] = "Bearer " + token
|
|
129
|
+
continue
|
|
130
|
+
return None
|
|
131
|
+
except URLError:
|
|
132
|
+
return None
|
|
133
|
+
return None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def check_digest_staleness(image: str) -> dict[str, Any]:
|
|
137
|
+
"""
|
|
138
|
+
Compare a pinned "repo:tag@sha256:..." reference against the digest the
|
|
139
|
+
registry currently publishes for that same tag.
|
|
140
|
+
|
|
141
|
+
Returns a dict with "image", "status" (one of "not-pinned",
|
|
142
|
+
"lookup-failed", "up-to-date", "stale"), and "latest_digest".
|
|
143
|
+
"""
|
|
144
|
+
parsed = parse_pinned_reference(image)
|
|
145
|
+
if parsed is None:
|
|
146
|
+
return {"image": image, "status": "not-pinned", "latest_digest": None}
|
|
147
|
+
|
|
148
|
+
tag_ref, pinned_digest = parsed
|
|
149
|
+
info = parse_image_reference(tag_ref)
|
|
150
|
+
repository = info["repository"]
|
|
151
|
+
if info["namespace"]:
|
|
152
|
+
repository = f"{info['namespace']}/{repository}"
|
|
153
|
+
if not repository or not info["registry"]:
|
|
154
|
+
return {"image": image, "status": "lookup-failed", "latest_digest": None}
|
|
155
|
+
|
|
156
|
+
latest_digest = fetch_registry_digest(info["registry"], repository, info["tag"])
|
|
157
|
+
if latest_digest is None:
|
|
158
|
+
return {"image": image, "status": "lookup-failed", "latest_digest": None}
|
|
159
|
+
if latest_digest == pinned_digest:
|
|
160
|
+
return {"image": image, "status": "up-to-date", "latest_digest": latest_digest}
|
|
161
|
+
return {"image": image, "status": "stale", "latest_digest": latest_digest}
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def collect_pinned_images_for_module(module_def: dict[str, Any]) -> list[dict[str, str]]:
|
|
165
|
+
"""Return {service, image} entries for a module's digest-pinned images."""
|
|
166
|
+
compose = module_def.get("spec", {}).get("implementation", {}).get("compose", {})
|
|
167
|
+
entries: list[dict[str, str]] = []
|
|
168
|
+
for service_name, image, _dockerfile in find_images_in_compose(compose):
|
|
169
|
+
if parse_pinned_reference(image) is not None:
|
|
170
|
+
entries.append({"service": service_name, "image": image})
|
|
171
|
+
return entries
|
|
@@ -156,6 +156,18 @@ def _finding(
|
|
|
156
156
|
return {
|
|
157
157
|
"rule_id": rule_id,
|
|
158
158
|
"severity": severity,
|
|
159
|
+
# These findings aren't declared in cli/resources/rule-set.json (see
|
|
160
|
+
# docs/image-signing.md for why), so their compliance category can't
|
|
161
|
+
# be looked up the way cli.security.run_security_validation() does
|
|
162
|
+
# for rule-set findings. Every check here is about image tag/digest
|
|
163
|
+
# pinning, registry trust, or signature/provenance verification --
|
|
164
|
+
# i.e. keeping deployed images current, authentic, and traceable to
|
|
165
|
+
# a trusted build -- which is exactly the use case
|
|
166
|
+
# docs/security-compliance-categories.md documents "patching" for,
|
|
167
|
+
# so tag them directly with that
|
|
168
|
+
# cli.security_common.COMPLIANCE_CATEGORIES value instead of
|
|
169
|
+
# leaving them uncategorized.
|
|
170
|
+
"category": "patching",
|
|
159
171
|
"module": service,
|
|
160
172
|
"message": message,
|
|
161
173
|
"path": f"services.{service}.image",
|
|
@@ -310,6 +322,22 @@ def _fixture_entry(
|
|
|
310
322
|
return matches[0]
|
|
311
323
|
|
|
312
324
|
|
|
325
|
+
def lookup_image_evidence(
|
|
326
|
+
image_ref: str, fixture_path: Path | None
|
|
327
|
+
) -> dict[str, Any] | None:
|
|
328
|
+
"""
|
|
329
|
+
Return the signed-images fixture entry (repository, digest, signed,
|
|
330
|
+
provenanceAttested, sbomAttested) matching image_ref, or None when no
|
|
331
|
+
fixture is configured/loadable or no matching entry exists.
|
|
332
|
+
|
|
333
|
+
Public wrapper around `_load_fixture`/`_fixture_entry` for callers
|
|
334
|
+
outside this module (e.g. `cli.report`'s compliance report) that need
|
|
335
|
+
read-only evidence lookup without pulling in cosign/live-verification.
|
|
336
|
+
"""
|
|
337
|
+
fixture, _error = _load_fixture(fixture_path)
|
|
338
|
+
return _fixture_entry(fixture, image_ref)
|
|
339
|
+
|
|
340
|
+
|
|
313
341
|
def _verification_findings(
|
|
314
342
|
images: list[tuple[str, str, bool]],
|
|
315
343
|
policy: ImagePolicy,
|
|
@@ -7,13 +7,20 @@ import os
|
|
|
7
7
|
import re
|
|
8
8
|
import subprocess # nosec B404
|
|
9
9
|
import tempfile
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Callable
|
|
10
12
|
from pathlib import Path
|
|
11
13
|
from typing import IO, Any
|
|
12
14
|
|
|
13
15
|
import yaml
|
|
14
16
|
|
|
15
|
-
from .state import parse_k8s_workloads_json
|
|
16
|
-
from .up_runner import
|
|
17
|
+
from .state import group_services_by_health, parse_k8s_workloads_json
|
|
18
|
+
from .up_runner import (
|
|
19
|
+
DEFAULT_POLL_INTERVAL_SECONDS,
|
|
20
|
+
_poll_until_settled,
|
|
21
|
+
_validate_command,
|
|
22
|
+
run_streamed,
|
|
23
|
+
)
|
|
17
24
|
|
|
18
25
|
# RFC 1123 DNS label: lowercase alphanumeric and `-`, not leading/trailing
|
|
19
26
|
# with `-`, 1-63 chars. Both Kubernetes namespaces and Helm release names
|
|
@@ -90,16 +97,37 @@ def helm_up(
|
|
|
90
97
|
timeout: float,
|
|
91
98
|
detach: bool,
|
|
92
99
|
log_file: IO[str],
|
|
100
|
+
poll_interval: float = DEFAULT_POLL_INTERVAL_SECONDS,
|
|
101
|
+
use_color: bool = False,
|
|
102
|
+
sleep_fn: Callable[[float], None] = time.sleep,
|
|
103
|
+
now_fn: Callable[[], float] = time.monotonic,
|
|
104
|
+
redraw_fn: Callable[[str], None] | None = None,
|
|
93
105
|
) -> int:
|
|
94
|
-
"""
|
|
106
|
+
"""
|
|
107
|
+
Install or upgrade a rendered chart without persisting secret values.
|
|
108
|
+
|
|
109
|
+
`timeout` is one overall budget shared across the whole call, the way
|
|
110
|
+
Compose's `--timeout` already is: it bounds `helm upgrade --install`
|
|
111
|
+
itself, then whatever remains bounds the readiness poll below, rather
|
|
112
|
+
than being reused in full for each step (which could let total
|
|
113
|
+
wall-clock time exceed `timeout` several times over on a
|
|
114
|
+
multi-workload release).
|
|
115
|
+
|
|
116
|
+
Readiness is decided by polling `get_k8s_state()` /
|
|
117
|
+
`group_services_by_health()` — the exact same functions `cds state
|
|
118
|
+
--target helm` uses — instead of a separate `helm --wait` +
|
|
119
|
+
`kubectl rollout status`/`wait` implementation, so "ready" here and
|
|
120
|
+
"healthy" in `cds state` can never disagree.
|
|
121
|
+
"""
|
|
95
122
|
namespace = _validate_k8s_name(namespace, "namespace")
|
|
96
123
|
release = _validate_k8s_name(release, "release")
|
|
97
124
|
kube_context = _validate_kube_context(kube_context)
|
|
98
125
|
timeout = _validate_timeout(timeout)
|
|
126
|
+
deadline = now_fn() + timeout
|
|
99
127
|
secret_values = _secret_values(plan)
|
|
100
128
|
secret_path = _write_secret_values(secret_values)
|
|
101
129
|
context_args = ["--kube-context", kube_context] if kube_context else []
|
|
102
|
-
|
|
130
|
+
apply_timeout_arg = f"{timeout}s"
|
|
103
131
|
command = [
|
|
104
132
|
"helm",
|
|
105
133
|
*context_args,
|
|
@@ -113,10 +141,8 @@ def helm_up(
|
|
|
113
141
|
"--values",
|
|
114
142
|
str(secret_path),
|
|
115
143
|
"--timeout",
|
|
116
|
-
|
|
144
|
+
apply_timeout_arg,
|
|
117
145
|
]
|
|
118
|
-
if not detach:
|
|
119
|
-
command.append("--wait")
|
|
120
146
|
|
|
121
147
|
try:
|
|
122
148
|
result = run_streamed(command, log_file, timeout=timeout + 30)
|
|
@@ -126,31 +152,76 @@ def helm_up(
|
|
|
126
152
|
return result
|
|
127
153
|
|
|
128
154
|
workloads = get_k8s_workloads(namespace, release, kube_context)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
155
|
+
remaining = max(0.0, deadline - now_fn())
|
|
156
|
+
settled, grouped = poll_k8s_state_until_settled(
|
|
157
|
+
namespace,
|
|
158
|
+
release,
|
|
159
|
+
kube_context,
|
|
160
|
+
expected_service_count=len(workloads),
|
|
161
|
+
poll_interval=poll_interval,
|
|
162
|
+
timeout=remaining,
|
|
163
|
+
use_color=use_color,
|
|
164
|
+
sleep_fn=sleep_fn,
|
|
165
|
+
now_fn=now_fn,
|
|
166
|
+
redraw_fn=redraw_fn,
|
|
167
|
+
)
|
|
168
|
+
if not settled:
|
|
169
|
+
unhealthy = [name for bucket in ("UNHEALTHY", "UNHEALTHY EXIT") for name in grouped.get(bucket, [])]
|
|
170
|
+
log_file.write(
|
|
171
|
+
f"helm release {release} did not settle within {timeout:.0f}s"
|
|
172
|
+
+ (f"; unhealthy: {', '.join(unhealthy)}" if unhealthy else "")
|
|
173
|
+
+ "\n"
|
|
174
|
+
)
|
|
175
|
+
log_file.flush()
|
|
176
|
+
return 1
|
|
151
177
|
return 0
|
|
152
178
|
|
|
153
179
|
|
|
180
|
+
def poll_k8s_state_until_settled(
|
|
181
|
+
namespace: str,
|
|
182
|
+
release: str,
|
|
183
|
+
kube_context: str | None,
|
|
184
|
+
*,
|
|
185
|
+
expected_service_count: int | None = None,
|
|
186
|
+
poll_interval: float = DEFAULT_POLL_INTERVAL_SECONDS,
|
|
187
|
+
timeout: float = 180.0,
|
|
188
|
+
use_color: bool = False,
|
|
189
|
+
sleep_fn: Callable[[float], None] = time.sleep,
|
|
190
|
+
now_fn: Callable[[], float] = time.monotonic,
|
|
191
|
+
fetch_fn: Callable[[], list[dict[str, Any]]] | None = None,
|
|
192
|
+
redraw_fn: Callable[[str], None] | None = None,
|
|
193
|
+
) -> tuple[bool, dict[str, list[str]]]:
|
|
194
|
+
"""
|
|
195
|
+
Polls `get_k8s_state()` every `poll_interval` seconds, grouping with
|
|
196
|
+
`group_services_by_health()` — the same pair `cds state --target
|
|
197
|
+
helm` calls — until every workload settles into a terminal bucket or
|
|
198
|
+
`timeout` seconds elapse. Shares its settle/failure rule with
|
|
199
|
+
Compose's `poll_state_until_settled()` via `_poll_until_settled()`.
|
|
200
|
+
|
|
201
|
+
`fetch_fn` is injectable for tests; defaults to a real
|
|
202
|
+
`get_k8s_state(namespace, release, kube_context)` call.
|
|
203
|
+
"""
|
|
204
|
+
if fetch_fn is None:
|
|
205
|
+
def fetch_fn() -> list[dict[str, Any]]:
|
|
206
|
+
return get_k8s_state(namespace, release, kube_context)
|
|
207
|
+
|
|
208
|
+
def fetch_grouped() -> dict[str, list[str]]:
|
|
209
|
+
return group_services_by_health(fetch_fn())
|
|
210
|
+
|
|
211
|
+
return _poll_until_settled(
|
|
212
|
+
fetch_grouped,
|
|
213
|
+
expected_service_count=expected_service_count,
|
|
214
|
+
poll_interval=poll_interval,
|
|
215
|
+
timeout=timeout,
|
|
216
|
+
use_color=use_color,
|
|
217
|
+
sleep_fn=sleep_fn,
|
|
218
|
+
now_fn=now_fn,
|
|
219
|
+
redraw_fn=redraw_fn,
|
|
220
|
+
up_done_fn=None,
|
|
221
|
+
on_up_finished=None,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
154
225
|
def helm_down(
|
|
155
226
|
*,
|
|
156
227
|
namespace: str,
|
|
@@ -76,6 +76,7 @@ def scan_k8s_security(plan: dict[str, Any]) -> list[dict[str, Any]]:
|
|
|
76
76
|
f"{path}.resources.{container_name}",
|
|
77
77
|
module_id,
|
|
78
78
|
["Declare both requests and limits for the container."],
|
|
79
|
+
category="configuration-management",
|
|
79
80
|
))
|
|
80
81
|
return findings
|
|
81
82
|
|
|
@@ -113,10 +114,22 @@ def _finding(
|
|
|
113
114
|
path: str,
|
|
114
115
|
module: str,
|
|
115
116
|
recommendation: list[str],
|
|
117
|
+
*,
|
|
118
|
+
category: str = "system-hardening",
|
|
116
119
|
) -> dict[str, Any]:
|
|
117
120
|
return {
|
|
118
121
|
"rule_id": rule_id,
|
|
119
122
|
"severity": severity,
|
|
123
|
+
# These findings aren't declared in cli/resources/rule-set.json (the
|
|
124
|
+
# declarative rule-set engine has no scope for the plan's
|
|
125
|
+
# Kubernetes-specific implementation), so their compliance category
|
|
126
|
+
# can't be looked up the way cli.security.run_security_validation()
|
|
127
|
+
# does for rule-set findings. CDS-K8S-001-004 are container/pod
|
|
128
|
+
# runtime-hardening posture checks, so they default to the matching
|
|
129
|
+
# cli.security_common.COMPLIANCE_CATEGORIES value; CDS-K8S-005
|
|
130
|
+
# (resource requests/limits) overrides this below since it's a
|
|
131
|
+
# capacity-governance concern rather than runtime-hardening posture.
|
|
132
|
+
"category": category,
|
|
120
133
|
"message": message,
|
|
121
134
|
"path": path,
|
|
122
135
|
"module": module,
|
|
@@ -252,12 +252,12 @@ def save_generated_profile(
|
|
|
252
252
|
]
|
|
253
253
|
|
|
254
254
|
name_path = Path(name)
|
|
255
|
-
if name_path.is_absolute() or ".." in name_path.parts:
|
|
255
|
+
if name_path.is_absolute() or ".." in name_path.parts or len(name_path.parts) != 1:
|
|
256
256
|
return None, [
|
|
257
257
|
Diagnostic(
|
|
258
258
|
level="error",
|
|
259
259
|
code="E115",
|
|
260
|
-
message=f'Generated profile name "{name}" must be a relative
|
|
260
|
+
message=f'Generated profile name "{name}" must be a single relative path segment.',
|
|
261
261
|
path="metadata.name",
|
|
262
262
|
)
|
|
263
263
|
]
|