composable-data-stack 0.9.1__tar.gz → 0.10.0__tar.gz

{composable_data_stack-0.9.1/composable_data_stack.egg-info → composable_data_stack-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: composable-data-stack
-Version: 0.9.1
+Version: 0.10.0
 Summary: CLI for validating and composing composable data stack profiles
 Author: Ronald Hensbergen
 License-Expression: Apache-2.0
@@ -32,7 +32,7 @@ Provides-Extra: dev
 Requires-Dist: coverage>=7.6; extra == "dev"
 Requires-Dist: bandit>=1.7; extra == "dev"
 Requires-Dist: pip-audit>=2.7; extra == "dev"
-Requires-Dist: ruff==0.16.8; extra == "dev"
+Requires-Dist: ruff==0.16.9; extra == "dev"
 Dynamic: license-file
 
 # 🚀 Composable Data Stack (CDS)
@@ -892,6 +892,7 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
 | `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
 | `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
 | `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
+| `[E043] ... recorded as unsupported in the compatibility registry` | The binding's `kind` matches, but this exact provider/consumer pairing is explicitly recorded as `unsupported` in `cli/resources/compatibility-registry.json` -- it's known NOT to work together despite matching shape. | Use a different provider/consumer pairing for this contract, or if the pairing has since been fixed/verified, update its `status` to `tested` in the registry (see [docs/architecture.md § Secrets and contract resolution](docs/architecture.md#secrets-and-contract-resolution)). |
 | `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
 | `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
 
@@ -1036,6 +1037,7 @@ Please read these first:
 - [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
 - [SECURITY.md](SECURITY.md)
 - [SUPPORT.md](SUPPORT.md)
+- [docs/security-support-policy.md](docs/security-support-policy.md)
 - [CHANGELOG.md](CHANGELOG.md)
 - [RELEASE.md](RELEASE.md)
 

{composable_data_stack-0.9.1 → composable_data_stack-0.10.0}/README.md

@@ -855,6 +855,7 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
 | `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
 | `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
 | `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
+| `[E043] ... recorded as unsupported in the compatibility registry` | The binding's `kind` matches, but this exact provider/consumer pairing is explicitly recorded as `unsupported` in `cli/resources/compatibility-registry.json` -- it's known NOT to work together despite matching shape. | Use a different provider/consumer pairing for this contract, or if the pairing has since been fixed/verified, update its `status` to `tested` in the registry (see [docs/architecture.md § Secrets and contract resolution](docs/architecture.md#secrets-and-contract-resolution)). |
 | `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
 | `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
 
@@ -999,6 +1000,7 @@ Please read these first:
 - [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md)
 - [SECURITY.md](SECURITY.md)
 - [SUPPORT.md](SUPPORT.md)
+- [docs/security-support-policy.md](docs/security-support-policy.md)
 - [CHANGELOG.md](CHANGELOG.md)
 - [RELEASE.md](RELEASE.md)
 

{composable_data_stack-0.9.1 → composable_data_stack-0.10.0}/cli/image_verification.py

@@ -156,6 +156,18 @@ def _finding(
     return {
         "rule_id": rule_id,
         "severity": severity,
+        # These findings aren't declared in cli/resources/rule-set.json (see
+        # docs/image-signing.md for why), so their compliance category can't
+        # be looked up the way cli.security.run_security_validation() does
+        # for rule-set findings. Every check here is about image tag/digest
+        # pinning, registry trust, or signature/provenance verification --
+        # i.e. keeping deployed images current, authentic, and traceable to
+        # a trusted build -- which is exactly the use case
+        # docs/security-compliance-categories.md documents "patching" for,
+        # so tag them directly with that
+        # cli.security_common.COMPLIANCE_CATEGORIES value instead of
+        # leaving them uncategorized.
+        "category": "patching",
         "module": service,
         "message": message,
         "path": f"services.{service}.image",

{composable_data_stack-0.9.1 → composable_data_stack-0.10.0}/cli/k8s_security.py

@@ -76,6 +76,7 @@ def scan_k8s_security(plan: dict[str, Any]) -> list[dict[str, Any]]:
                         f"{path}.resources.{container_name}",
                         module_id,
                         ["Declare both requests and limits for the container."],
+                        category="configuration-management",
                     ))
     return findings
 
@@ -113,10 +114,22 @@ def _finding(
     path: str,
     module: str,
     recommendation: list[str],
+    *,
+    category: str = "system-hardening",
 ) -> dict[str, Any]:
     return {
         "rule_id": rule_id,
         "severity": severity,
+        # These findings aren't declared in cli/resources/rule-set.json (the
+        # declarative rule-set engine has no scope for the plan's
+        # Kubernetes-specific implementation), so their compliance category
+        # can't be looked up the way cli.security.run_security_validation()
+        # does for rule-set findings. CDS-K8S-001-004 are container/pod
+        # runtime-hardening posture checks, so they default to the matching
+        # cli.security_common.COMPLIANCE_CATEGORIES value; CDS-K8S-005
+        # (resource requests/limits) overrides this below since it's a
+        # capacity-governance concern rather than runtime-hardening posture.
+        "category": category,
         "message": message,
         "path": path,
         "module": module,

{composable_data_stack-0.9.1 → composable_data_stack-0.10.0}/cli/main.py

@@ -38,7 +38,7 @@ from .planner import build_plan
 from .preflight import preflight_passed, run_preflight
 from .renderer import render_compose
 from .security import PrecomputedRender, run_security_validation
-from .security_common import SEVERITY_ORDER, infer_profile_class
+from .security_common import COMPLIANCE_CATEGORIES, SEVERITY_ORDER, infer_profile_class
 from .state import format_state_output, group_services_by_health, parse_compose_ps_json
 from .up_runner import (
     DEFAULT_TIMEOUT_SECONDS,
@@ -875,6 +875,58 @@ def _unverifiable_image_finding(message: str) -> dict[str, Any]:
     }
 
 
+_UNCATEGORIZED_LABEL = "uncategorized"
+
+
+def _filter_findings_by_category(
+    findings: list[dict[str, Any]], categories: list[str] | None
+) -> list[dict[str, Any]]:
+    """
+    Restrict findings to the given compliance control categories.
+
+    Every finding source (the declarative rule-set engine, scan_k8s_security,
+    image verification) tags its findings with a "category" at generation
+    time (see cli.security.run_security_validation(), cli.k8s_security,
+    cli.image_verification), so this is a uniform key lookup; there is no
+    remaining source of genuinely uncategorized findings today, but a
+    missing "category" is still treated as excluded rather than raising, in
+    case a future finding source doesn't set one.
+    """
+    if not categories:
+        return findings
+    wanted = set(categories)
+    return [f for f in findings if f.get("category") in wanted]
+
+
+def _group_findings_by_category(
+    findings: list[dict[str, Any]],
+) -> dict[str, list[dict[str, Any]]]:
+    """
+    Group findings by their compliance control category, preserving each
+    group's existing (severity/rule_id/...) ordering. A finding without a
+    "category" is grouped under "uncategorized" rather than dropped, as a
+    defensive fallback (see _filter_findings_by_category()); in practice
+    every current finding source sets one.
+    """
+    grouped: dict[str, list[dict[str, Any]]] = {}
+    for finding in findings:
+        grouped.setdefault(finding.get("category") or _UNCATEGORIZED_LABEL, []).append(finding)
+    return dict(sorted(grouped.items()))
+
+
+def _category_heading(category: str) -> str:
+    """
+    Render a "== <label> (<category>) ==" heading for --group-by-category,
+    falling back to just "== <category> ==" for the uncategorized bucket,
+    which has no COMPLIANCE_CATEGORIES entry.
+    """
+    entry = COMPLIANCE_CATEGORIES.get(category)
+    if entry is None:
+        return f"== {category} =="
+    label, _rationale = entry
+    return f"== {label} ({category}) =="
+
+
 def _run_image_verification(
     profile_path: str,
     environment: str | None,
@@ -1159,6 +1211,21 @@ def main() -> int:
             "By default, values are redacted to avoid echoing real secrets to stdout/CI logs."
         ),
     )
+    test_parser.add_argument(
+        "--category",
+        action="append",
+        choices=COMPLIANCE_CATEGORIES,
+        help=(
+            "Only print security findings tagged with this compliance control "
+            "category (repeatable); does not affect the security stage's "
+            "pass/fail outcome. See `cds security --help` for details."
+        ),
+    )
+    test_parser.add_argument(
+        "--group-by-category",
+        action="store_true",
+        help="Print security findings grouped by compliance control category instead of one flat list.",
+    )
     _add_hardened_arg(test_parser)
     _add_image_source_arg(test_parser)
 
@@ -1283,6 +1350,25 @@ def main() -> int:
             "tests/fixtures/signed-images.json) for offline verification."
         ),
     )
+    security_parser.add_argument(
+        "--category",
+        action="append",
+        choices=COMPLIANCE_CATEGORIES,
+        help=(
+            "Only print findings tagged with this compliance control category "
+            "(repeatable); does not change which rules run or the command's "
+            "exit code -- a high-severity finding outside the requested "
+            "category still fails the scan. Categories are an informational "
+            "readiness aid for organizing findings against a user's own risk "
+            "assessment (e.g. NIS2/Cyberbeveiligingswet), not a compliance "
+            "certification -- see docs/security-compliance-categories.md."
+        ),
+    )
+    security_parser.add_argument(
+        "--group-by-category",
+        action="store_true",
+        help="Print findings grouped by compliance control category instead of one flat list.",
+    )
 
     diff_parser = subparsers.add_parser(
         "diff",
@@ -1941,14 +2027,29 @@ def main() -> int:
                     ))
                 for diag in sec_diags:
                     print(diag.format(), file=sys.stderr)
-                for f in findings:
-                    print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
+                displayed_findings = _filter_findings_by_category(findings, args.category)
+                if args.group_by_category:
+                    for category, group in _group_findings_by_category(displayed_findings).items():
+                        print(_category_heading(category))
+                        for f in group:
+                            print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
+                else:
+                    for f in displayed_findings:
+                        print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
                 # A W096 warning means rendered-compose-scoped rules (e.g.
                 # CDS-SEC-070) were silently skipped due to an unexpected
                 # error during rendering. Treat that the same as a failed
                 # security stage rather than letting any non-high finding
                 # (or no finding at all) mask the fact that some checks
                 # never ran (GHSA-mx5p-cv63-6829).
+                #
+                # security_ok is intentionally computed from the full,
+                # unfiltered `findings`, not `displayed_findings`: --category
+                # is a display-only filter (see docs/security-compliance-
+                # categories.md), so narrowing what's printed must never
+                # narrow what counts as a failing scan -- that would let a
+                # real high-severity finding outside the requested category
+                # silently report success.
                 render_scan_skipped = any(d.code == "W096" for d in sec_diags)
                 security_ok = (
                     not render_scan_skipped
@@ -2293,15 +2394,38 @@ def main() -> int:
             print("No security findings.")
             return 0
 
-        for f in findings:
-            print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
-            print(f"  object: {f['path']}")
-            print(f"  module: {f['module']}")
-            if f["value"] is not None:
-                print(f"  value: {f['value']}")
-            for rec in f["recommendation"]:
-                print(f"  fix: {rec}")
-            print()
+        # --category is a display-only filter (see docs/security-compliance-
+        # categories.md): it must never narrow what counts as a failing scan,
+        # only what gets printed. The exit code below is deliberately
+        # computed from the full, unfiltered `findings`, not
+        # `displayed_findings` -- otherwise a real high-severity finding
+        # outside the requested category would silently report success.
+        displayed_findings = _filter_findings_by_category(findings, args.category)
+
+        if not displayed_findings:
+            print(f"No security findings in category: {', '.join(sorted(set(args.category)))}.")
+        elif args.group_by_category:
+            for category, group in _group_findings_by_category(displayed_findings).items():
+                print(_category_heading(category))
+                for f in group:
+                    print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
+                    print(f"  object: {f['path']}")
+                    print(f"  module: {f['module']}")
+                    if f["value"] is not None:
+                        print(f"  value: {f['value']}")
+                    for rec in f["recommendation"]:
+                        print(f"  fix: {rec}")
+                    print()
+        else:
+            for f in displayed_findings:
+                print(f"[{f['severity'].upper()}] {f['rule_id']} {f['message']}")
+                print(f"  object: {f['path']}")
+                print(f"  module: {f['module']}")
+                if f["value"] is not None:
+                    print(f"  value: {f['value']}")
+                for rec in f["recommendation"]:
+                    print(f"  fix: {rec}")
+                print()
 
         # As above: a W096 warning means some rendered-compose-scoped rules
         # were skipped, so even when only non-high findings are present the

composable_data_stack-0.10.0/cli/resources/compatibility-registry.json

@@ -0,0 +1,31 @@
+{
+  "$schema": "compatibility-registry.schema.json",
+  "version": "1.0.0",
+  "metadata": {
+    "name": "cds-core-compatibility-registry",
+    "description": "Central record of module pairings known to have been tested together (\"tested\"), or known NOT to work together despite matching a contract's kind (\"unsupported\"). Kept as a single file rather than scattered across individual module.yaml documents so composition knowledge about the whole stack stays in one place (see issue #350). A pairing with no entry here is only structurally compatible (same contract kind); it is neither vouched for nor ruled out."
+  },
+  "pairings": [
+    {
+      "contract": "sql-database",
+      "provider": "warehouse/postgres",
+      "consumer": "orchestration/dagster",
+      "status": "tested",
+      "notes": "Exercised end-to-end by profiles/local-dagster-postgres-superset/profile.yaml, covered by the \"CDS Test (Stable Profile)\" CI workflow."
+    },
+    {
+      "contract": "sql-database",
+      "provider": "warehouse/postgres",
+      "consumer": "bi/superset",
+      "status": "tested",
+      "notes": "Exercised end-to-end by profiles/local-dagster-postgres-superset/profile.yaml, covered by the \"CDS Test (Stable Profile)\" CI workflow."
+    },
+    {
+      "contract": "cache-service",
+      "provider": "cache/keydb",
+      "consumer": "bi/superset",
+      "status": "tested",
+      "notes": "Exercised end-to-end by profiles/local-dagster-postgres-superset/profile.yaml, covered by the \"CDS Test (Stable Profile)\" CI workflow."
+    }
+  ]
+}

composable_data_stack-0.10.0/cli/resources/compatibility-registry.schema.json

@@ -0,0 +1,70 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "./compatibility-registry.schema.json",
+  "title": "Composable Data Stack Compatibility Registry",
+  "description": "Central, non-scattered record of module pairings for a given contract that are known to have been tested together, or explicitly known NOT to work together. This is deliberately separate from plain contract `kind` matching (structural compatibility): an entry here expresses practical, known-good/known-bad compatibility, not just shape compatibility. Absence of an entry for a pairing means 'not yet recorded' -- it is neither tested nor unsupported.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["version", "metadata", "pairings"],
+  "properties": {
+    "$schema": {
+      "type": "string"
+    },
+    "version": {
+      "type": "string"
+    },
+    "metadata": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name"],
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1
+        },
+        "description": {
+          "type": "string"
+        }
+      }
+    },
+    "pairings": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/pairing"
+      }
+    }
+  },
+  "$defs": {
+    "pairing": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["contract", "provider", "consumer", "status"],
+      "properties": {
+        "contract": {
+          "type": "string",
+          "pattern": "^[a-z0-9-]+$",
+          "description": "Contract kind this pairing applies to (matches a module's provides[].contract.kind / consumes[].contract.kind), not a specific provide-name. A provider can expose several provide-names under the same kind (e.g. warehouse/postgres provides \"sql-database\", \"dagster-database\", and \"superset-database\", all kind \"sql-database\"); one row here is intended to cover the provider/consumer pair for all of them at this kind granularity. A row also carries no module version, so a \"tested\" status remains valid across version changes on either side (tracked as a follow-up, see issue #750)."
+        },
+        "provider": {
+          "type": "string",
+          "pattern": "^[a-z0-9-]+/[a-z0-9-]+$",
+          "description": "Providing module's identity, as \"<metadata.category>/<metadata.name>\" (e.g. \"warehouse/postgres\"). Stable across profiles, unlike a profile-scoped module instance id or a source path."
+        },
+        "consumer": {
+          "type": "string",
+          "pattern": "^[a-z0-9-]+/[a-z0-9-]+$",
+          "description": "Consuming module's identity, as \"<metadata.category>/<metadata.name>\" (e.g. \"orchestration/dagster\")."
+        },
+        "status": {
+          "type": "string",
+          "enum": ["tested", "unsupported"],
+          "description": "\"tested\": this exact provider/consumer pairing for this contract has been exercised together (e.g. by a profile under active CI coverage) and is known-good. \"unsupported\": this pairing is structurally compatible (same contract kind) but has been found NOT to work together in practice; validation reports this as an error."
+        },
+        "notes": {
+          "type": "string",
+          "description": "Free-text context, e.g. which profile/test exercises a \"tested\" pairing, or why an \"unsupported\" pairing fails."
+        }
+      }
+    }
+  }
+}

{composable_data_stack-0.9.1 → composable_data_stack-0.10.0}/cli/resources/profile.schema.json

@@ -154,6 +154,21 @@
               }
             }
           }
+        },
+        "security": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "waivers": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "plaintextEndpointExposure": {
+                  "$ref": "#/$defs/plaintextEndpointExposureWaiver"
+                }
+              }
+            }
+          }
         }
       }
     }
@@ -231,6 +246,19 @@
           "default": "bridge"
         }
       }
+    },
+    "plaintextEndpointExposureWaiver": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["reason"],
+      "properties": {
+        "reason": {
+          "type": "string",
+          "minLength": 1,
+          "pattern": "\\S",
+          "description": "Must contain at least one non-whitespace character; cli/security.py strips and treats an all-whitespace reason as unset (no waiver applied, so CDS-SEC-074 still fires), so a schema that only enforced minLength: 1 would silently accept a value the code then discards."
+        }
+      }
     }
   }
 }

{composable_data_stack-0.9.1 → composable_data_stack-0.10.0}/cli/resources/rule-schema.json

@@ -49,6 +49,20 @@
         "observability-leakage"
       ]
     },
+    "complianceCategory": {
+      "type": "string",
+      "description": "Informational compliance control category this rule's finding maps onto (e.g. for a user's own NIS2/Cyberbeveiligingswet risk assessment). This is a readiness aid, not a legal or regulatory certification of the rule or category -- see docs/security-compliance-categories.md.",
+      "enum": [
+        "access-control",
+        "secrets-management",
+        "network-exposure",
+        "encryption-in-transit",
+        "configuration-management",
+        "system-hardening",
+        "logging-monitoring",
+        "patching"
+      ]
+    },
     "scopeItem": {
       "type": "string",
       "enum": [
@@ -183,6 +197,7 @@
         "title",
         "severity",
         "category",
+        "complianceCategory",
         "scope",
         "match",
         "message",
@@ -207,6 +222,9 @@
         "category": {
           "$ref": "#/$defs/category"
         },
+        "complianceCategory": {
+          "$ref": "#/$defs/complianceCategory"
+        },
         "scope": {
           "type": "array",
           "minItems": 1,
@@ -231,6 +249,11 @@
         "enabled": {
           "type": "boolean",
           "default": true
+        },
+        "codeEnforced": {
+          "type": "boolean",
+          "default": true,
+          "description": "For a scope: [\"none\"] rule that is actually enforced entirely in code (not by the declarative match engine -- see the rule's $comment), this is the real on/off switch a custom rule set can flip to disable that code path. `enabled` stays false on such rules per the scope-none convention (it can never produce a declarative-engine finding), so `enabled` alone cannot represent whether the code-level check is active."
         }
       }
     }
@@ -249,6 +272,7 @@
           "title": "Hardcoded secret-like value",
           "severity": "high",
           "category": "secrets",
+          "complianceCategory": "secrets-management",
           "scope": ["profile-raw", "module-values", "service-env"],
           "match": {
             "all": [