composable-data-stack 0.9.0__py3-none-any.whl → 0.9.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cli/image_verification.py +12 -3
- cli/k8s_renderer.py +5 -0
- cli/k8s_runner.py +79 -2
- cli/main.py +9 -8
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/METADATA +136 -103
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/RECORD +10 -10
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/WHEEL +0 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/entry_points.txt +0 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/licenses/LICENSE +0 -0
- {composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/top_level.txt +0 -0
cli/image_verification.py
CHANGED
|
@@ -280,13 +280,14 @@ def _fixture_entry(
|
|
|
280
280
|
fixture: dict[str, Any] | None,
|
|
281
281
|
image_ref: str,
|
|
282
282
|
) -> dict[str, Any] | None:
|
|
283
|
-
"""Return the
|
|
283
|
+
"""Return the repository entry matching image_ref, preferring its digest."""
|
|
284
284
|
if fixture is None:
|
|
285
285
|
return None
|
|
286
286
|
images = fixture.get("images", {})
|
|
287
287
|
if not isinstance(images, dict):
|
|
288
288
|
return None
|
|
289
289
|
folded_image_ref = image_ref.casefold()
|
|
290
|
+
matches: list[dict[str, Any]] = []
|
|
290
291
|
for entry in images.values():
|
|
291
292
|
if not isinstance(entry, dict):
|
|
292
293
|
continue
|
|
@@ -297,8 +298,16 @@ def _fixture_entry(
|
|
|
297
298
|
if folded_image_ref == folded_repository or folded_image_ref.startswith(
|
|
298
299
|
(folded_repository + "@", folded_repository + ":")
|
|
299
300
|
):
|
|
300
|
-
|
|
301
|
-
|
|
301
|
+
matches.append(entry)
|
|
302
|
+
if not matches:
|
|
303
|
+
return None
|
|
304
|
+
if "@sha256:" in folded_image_ref:
|
|
305
|
+
ref_digest = folded_image_ref.rsplit("@", 1)[1]
|
|
306
|
+
for entry in matches:
|
|
307
|
+
digest = entry.get("digest")
|
|
308
|
+
if isinstance(digest, str) and digest.casefold() == ref_digest:
|
|
309
|
+
return entry
|
|
310
|
+
return matches[0]
|
|
302
311
|
|
|
303
312
|
|
|
304
313
|
def _verification_findings(
|
cli/k8s_renderer.py
CHANGED
|
@@ -1374,6 +1374,11 @@ def _values_config(value: Any) -> Any:
|
|
|
1374
1374
|
|
|
1375
1375
|
|
|
1376
1376
|
def _write_chart(root: Path, files: dict[str, str]) -> None:
|
|
1377
|
+
# Canonicalize before touching the filesystem (same convention as
|
|
1378
|
+
# `_atomic_write()`), so a caller-supplied `--output`/`--chart-dir`
|
|
1379
|
+
# value is resolved to a single, unambiguous absolute path rather than
|
|
1380
|
+
# acted on as a raw, possibly relative/traversal-bearing string.
|
|
1381
|
+
root = root.resolve()
|
|
1377
1382
|
root.parent.mkdir(parents=True, exist_ok=True)
|
|
1378
1383
|
staging = Path(tempfile.mkdtemp(prefix=f".{root.name}.tmp-", dir=root.parent))
|
|
1379
1384
|
backup = staging.with_name(staging.name.replace(".tmp-", ".previous-"))
|
cli/k8s_runner.py
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
from __future__ import annotations
|
|
3
3
|
|
|
4
4
|
import json
|
|
5
|
+
import math
|
|
5
6
|
import os
|
|
7
|
+
import re
|
|
6
8
|
import subprocess # nosec B404
|
|
7
9
|
import tempfile
|
|
8
10
|
from pathlib import Path
|
|
@@ -13,6 +15,70 @@ import yaml
|
|
|
13
15
|
from .state import parse_k8s_workloads_json
|
|
14
16
|
from .up_runner import _validate_command, run_streamed
|
|
15
17
|
|
|
18
|
+
# RFC 1123 DNS label: lowercase alphanumeric and `-`, not leading/trailing
|
|
19
|
+
# with `-`, 1-63 chars. Both Kubernetes namespaces and Helm release names
|
|
20
|
+
# must already satisfy this, so enforcing it here rejects nothing legitimate.
|
|
21
|
+
_K8S_NAME_RE = re.compile(r"[a-z0-9]([-a-z0-9]{0,61}[a-z0-9])?")
|
|
22
|
+
|
|
23
|
+
# Kube contexts allow a wider character set (e.g. ARNs), but must not start
|
|
24
|
+
# with `-` (which would be parsed as a flag) or contain control characters.
|
|
25
|
+
_KUBE_CONTEXT_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:@/-]*")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _validate_k8s_name(value: str, label: str) -> str:
|
|
29
|
+
"""
|
|
30
|
+
Reject a `namespace`/`release` value that isn't a valid Kubernetes
|
|
31
|
+
DNS-1123 label before it reaches `helm`/`kubectl` as a command
|
|
32
|
+
argument. Without this, a value starting with `-` (e.g. supplied by a
|
|
33
|
+
misbehaving caller or automation) could be parsed as an extra flag by
|
|
34
|
+
the downstream binary instead of a plain name (CWE-88 argument
|
|
35
|
+
injection).
|
|
36
|
+
|
|
37
|
+
Returns a new string built only from the characters matched by the
|
|
38
|
+
allowlist regex (rather than the original object) so nothing beyond a
|
|
39
|
+
validated Kubernetes name can flow to the command it's used to build.
|
|
40
|
+
"""
|
|
41
|
+
match = _K8S_NAME_RE.fullmatch(value) if isinstance(value, str) else None
|
|
42
|
+
if match is None:
|
|
43
|
+
raise ValueError(f"invalid {label}: {value!r} is not a valid Kubernetes name")
|
|
44
|
+
return match.group(0)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _validate_kube_context(value: str | None) -> str | None:
|
|
48
|
+
"""
|
|
49
|
+
Reject a `kube_context` that looks like a command-line flag (starts
|
|
50
|
+
with `-`) or contains characters outside the safe allowlist, so it
|
|
51
|
+
can't be smuggled to `helm`/`kubectl` as an extra argument (CWE-88
|
|
52
|
+
argument injection).
|
|
53
|
+
|
|
54
|
+
Returns a new string built only from the allowlisted characters
|
|
55
|
+
(rather than the original object) so nothing beyond a validated
|
|
56
|
+
context name can flow to the command it's used to build.
|
|
57
|
+
"""
|
|
58
|
+
if value is None:
|
|
59
|
+
return None
|
|
60
|
+
match = _KUBE_CONTEXT_RE.fullmatch(value) if isinstance(value, str) else None
|
|
61
|
+
if match is None:
|
|
62
|
+
raise ValueError(f"invalid kube context: {value!r}")
|
|
63
|
+
return match.group(0)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _validate_timeout(value: float) -> int:
|
|
67
|
+
"""
|
|
68
|
+
Reject a `timeout` that isn't a finite, positive number before it is
|
|
69
|
+
interpolated into a `--timeout=<n>s` argument for `helm`/`kubectl`. A
|
|
70
|
+
non-finite or non-positive value (e.g. NaN, infinity, or a negative
|
|
71
|
+
number) could otherwise render as a string that isn't a plain
|
|
72
|
+
numeric suffix, which risks being parsed as an extra flag by the
|
|
73
|
+
downstream binary (CWE-88 argument injection).
|
|
74
|
+
|
|
75
|
+
Returns a newly constructed ``int`` (rather than the original tainted
|
|
76
|
+
object) clamped to at least 1 second.
|
|
77
|
+
"""
|
|
78
|
+
if not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value):
|
|
79
|
+
raise ValueError(f"invalid timeout: {value!r} is not a finite number")
|
|
80
|
+
return max(1, int(value))
|
|
81
|
+
|
|
16
82
|
|
|
17
83
|
def helm_up(
|
|
18
84
|
plan: dict[str, Any],
|
|
@@ -26,10 +92,14 @@ def helm_up(
|
|
|
26
92
|
log_file: IO[str],
|
|
27
93
|
) -> int:
|
|
28
94
|
"""Install or upgrade a rendered chart without persisting secret values."""
|
|
95
|
+
namespace = _validate_k8s_name(namespace, "namespace")
|
|
96
|
+
release = _validate_k8s_name(release, "release")
|
|
97
|
+
kube_context = _validate_kube_context(kube_context)
|
|
98
|
+
timeout = _validate_timeout(timeout)
|
|
29
99
|
secret_values = _secret_values(plan)
|
|
30
100
|
secret_path = _write_secret_values(secret_values)
|
|
31
101
|
context_args = ["--kube-context", kube_context] if kube_context else []
|
|
32
|
-
timeout_arg = f"{
|
|
102
|
+
timeout_arg = f"{timeout}s"
|
|
33
103
|
command = [
|
|
34
104
|
"helm",
|
|
35
105
|
*context_args,
|
|
@@ -90,6 +160,10 @@ def helm_down(
|
|
|
90
160
|
delete_pvcs: bool,
|
|
91
161
|
log_file: IO[str],
|
|
92
162
|
) -> int:
|
|
163
|
+
namespace = _validate_k8s_name(namespace, "namespace")
|
|
164
|
+
release = _validate_k8s_name(release, "release")
|
|
165
|
+
kube_context = _validate_kube_context(kube_context)
|
|
166
|
+
timeout = _validate_timeout(timeout)
|
|
93
167
|
pvc_names: list[str] = []
|
|
94
168
|
if delete_pvcs:
|
|
95
169
|
pvc_names = _stateful_pvc_names(
|
|
@@ -105,7 +179,7 @@ def helm_down(
|
|
|
105
179
|
namespace,
|
|
106
180
|
"--ignore-not-found",
|
|
107
181
|
"--timeout",
|
|
108
|
-
f"{
|
|
182
|
+
f"{timeout}s",
|
|
109
183
|
]
|
|
110
184
|
result = run_streamed(command, log_file, timeout=timeout + 30)
|
|
111
185
|
if result != 0 or not delete_pvcs or not pvc_names:
|
|
@@ -121,6 +195,9 @@ def helm_down(
|
|
|
121
195
|
def get_k8s_workloads(
|
|
122
196
|
namespace: str, release: str, kube_context: str | None
|
|
123
197
|
) -> list[dict[str, Any]]:
|
|
198
|
+
namespace = _validate_k8s_name(namespace, "namespace")
|
|
199
|
+
release = _validate_k8s_name(release, "release")
|
|
200
|
+
kube_context = _validate_kube_context(kube_context)
|
|
124
201
|
command = _kubectl_command(kube_context, namespace) + [
|
|
125
202
|
"get",
|
|
126
203
|
"deployment,statefulset,job",
|
cli/main.py
CHANGED
|
@@ -961,7 +961,7 @@ def _render_helm_chart(plan, output_dir, force):
|
|
|
961
961
|
compose target this refuses to write over existing unrelated content unless
|
|
962
962
|
the caller passes --force.
|
|
963
963
|
"""
|
|
964
|
-
target_dir = Path(output_dir)
|
|
964
|
+
target_dir = Path(output_dir).resolve()
|
|
965
965
|
if target_dir.exists() and not target_dir.is_dir():
|
|
966
966
|
if not force:
|
|
967
967
|
print(f"ERROR Helm output path {target_dir} exists and is not a directory. Pass --force to replace it.")
|
|
@@ -1610,7 +1610,7 @@ def main() -> int:
|
|
|
1610
1610
|
|
|
1611
1611
|
if args.target == "helm":
|
|
1612
1612
|
project_root = resolve_project_root(profile_path)
|
|
1613
|
-
chart_dir = Path(args.chart_dir) if args.chart_dir else project_root / "chart"
|
|
1613
|
+
chart_dir = (Path(args.chart_dir) if args.chart_dir else project_root / "chart").resolve()
|
|
1614
1614
|
code, render_diags = _render_helm_chart(plan, chart_dir, force=True)
|
|
1615
1615
|
all_diags.extend(render_diags)
|
|
1616
1616
|
if code != 0 or has_errors(all_diags):
|
|
@@ -1620,9 +1620,9 @@ def main() -> int:
|
|
|
1620
1620
|
|
|
1621
1621
|
namespace = args.namespace or plan.get("runtime", {}).get("namespace") or "cds-local"
|
|
1622
1622
|
release = args.release or plan.get("metadata", {}).get("name") or "cds"
|
|
1623
|
-
log_path =
|
|
1624
|
-
Path(profile_path).parent.name
|
|
1625
|
-
)
|
|
1623
|
+
log_path = (
|
|
1624
|
+
Path(args.log_file) if args.log_file else default_log_path(Path(profile_path).parent.name)
|
|
1625
|
+
).resolve()
|
|
1626
1626
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1627
1627
|
try:
|
|
1628
1628
|
with open(log_path, "a", encoding="utf-8") as log_file:
|
|
@@ -1678,6 +1678,7 @@ def main() -> int:
|
|
|
1678
1678
|
log_path = Path(args.log_file)
|
|
1679
1679
|
else:
|
|
1680
1680
|
log_path = default_log_path(Path(profile_path).parent.name)
|
|
1681
|
+
log_path = log_path.resolve()
|
|
1681
1682
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1682
1683
|
|
|
1683
1684
|
log_tail_process = None
|
|
@@ -1837,9 +1838,9 @@ def main() -> int:
|
|
|
1837
1838
|
return 1
|
|
1838
1839
|
project_root = resolve_project_root(profile_path)
|
|
1839
1840
|
profile_release, profile_namespace = _k8s_runtime_defaults(profile_path)
|
|
1840
|
-
log_path =
|
|
1841
|
-
f"down-{Path(profile_path).parent.name}"
|
|
1842
|
-
)
|
|
1841
|
+
log_path = (
|
|
1842
|
+
Path(args.log_file) if args.log_file else default_log_path(f"down-{Path(profile_path).parent.name}")
|
|
1843
|
+
).resolve()
|
|
1843
1844
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
1844
1845
|
try:
|
|
1845
1846
|
with open(log_path, "a", encoding="utf-8") as log_file:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: composable-data-stack
|
|
3
|
-
Version: 0.9.
|
|
3
|
+
Version: 0.9.1
|
|
4
4
|
Summary: CLI for validating and composing composable data stack profiles
|
|
5
5
|
Author: Ronald Hensbergen
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -32,15 +32,15 @@ Provides-Extra: dev
|
|
|
32
32
|
Requires-Dist: coverage>=7.6; extra == "dev"
|
|
33
33
|
Requires-Dist: bandit>=1.7; extra == "dev"
|
|
34
34
|
Requires-Dist: pip-audit>=2.7; extra == "dev"
|
|
35
|
-
Requires-Dist: ruff==0.16.
|
|
35
|
+
Requires-Dist: ruff==0.16.8; extra == "dev"
|
|
36
36
|
Dynamic: license-file
|
|
37
37
|
|
|
38
38
|
# 🚀 Composable Data Stack (CDS)
|
|
39
39
|
|
|
40
|
-
> **
|
|
41
|
-
>
|
|
40
|
+
> **The modern data stack has a composition problem. CDS is an open framework
|
|
41
|
+
> for building data platforms that can evolve without being rebuilt.**
|
|
42
42
|
|
|
43
|
-

|
|
43
|
+

|
|
44
44
|
|
|
45
45
|
Project:
|
|
46
46
|
|
|
@@ -69,43 +69,122 @@ SonarCloud — Metrics:
|
|
|
69
69
|
|
|
70
70
|
---
|
|
71
71
|
|
|
72
|
-
|
|
72
|
+
You start with a straightforward goal: orchestrate some pipelines, store the
|
|
73
|
+
data, and make it useful.
|
|
73
74
|
|
|
74
|
-
|
|
75
|
+
Then the tool decisions begin.
|
|
75
76
|
|
|
76
|
-
|
|
77
|
+
Airflow, Dagster, or Prefect? Iceberg, Delta, or Hudi? Superset or Metabase?
|
|
78
|
+
Every choice opens another comparison, another integration, and another
|
|
79
|
+
architecture diagram. Every diagram looks different. Every tool has its own
|
|
80
|
+
configuration model, assumptions, and lifecycle. Before long, the platform is
|
|
81
|
+
less a designed system than a collection of decisions held together by custom
|
|
82
|
+
code and institutional memory.
|
|
77
83
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
- **Proof it**: if you run it in a real workflow, share your feedback — good or bad
|
|
84
|
+
The hardest question is no longer **"Which tool is best?"** It is **"How do
|
|
85
|
+
these tools fit together, and can we change our minds later?"**
|
|
81
86
|
|
|
82
|
-
|
|
87
|
+
## Why Data Platforms Have a Composition Problem
|
|
83
88
|
|
|
84
|
-
|
|
89
|
+
The modern data ecosystem is excellent at producing capable tools and terrible
|
|
90
|
+
at making them work as interchangeable parts.
|
|
85
91
|
|
|
86
|
-
|
|
92
|
+
Teams are expected to assemble orchestration, storage, transformation,
|
|
93
|
+
cataloging, observability, security, and BI into one dependable platform. But
|
|
94
|
+
the boundaries between those tools are usually implicit. Connection details
|
|
95
|
+
leak into environment variables. Service names become dependencies. Deployment
|
|
96
|
+
logic assumes a specific vendor. Compatibility lives in a staff engineer's
|
|
97
|
+
head—or is discovered during an incident.
|
|
87
98
|
|
|
88
|
-
|
|
99
|
+
That fragmentation creates a familiar cycle:
|
|
89
100
|
|
|
90
|
-
-
|
|
91
|
-
-
|
|
92
|
-
-
|
|
101
|
+
- evaluate an overwhelming number of tools
|
|
102
|
+
- debate architecture instead of delivering data products
|
|
103
|
+
- hand-build integrations that only one team understands
|
|
104
|
+
- duplicate stacks across development, CI, and production
|
|
105
|
+
- postpone upgrades because nobody knows what they will break
|
|
106
|
+
- accept lock-in because replacing one component feels like replacing the
|
|
107
|
+
platform
|
|
93
108
|
|
|
94
|
-
|
|
109
|
+
This is not primarily a tooling problem. **It is a composition problem.** We
|
|
110
|
+
have many good components, but no shared way to declare what they provide,
|
|
111
|
+
what they need, and whether a particular assembly is valid.
|
|
95
112
|
|
|
96
|
-
|
|
113
|
+
## The Cost of Choosing Wrong
|
|
97
114
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
115
|
+
A technology choice that looks local rarely stays local.
|
|
116
|
+
|
|
117
|
+
Choose an orchestrator and its concepts spread into deployment, secrets,
|
|
118
|
+
logging, and team workflows. Choose a table format and it constrains engines,
|
|
119
|
+
catalogs, and migration paths. Choose a BI layer and dashboards become coupled
|
|
120
|
+
to authentication, database behavior, and operational assumptions.
|
|
121
|
+
|
|
122
|
+
The bill arrives later:
|
|
123
|
+
|
|
124
|
+
- migrations become multi-quarter programs
|
|
125
|
+
- upgrades become risky, manual compatibility exercises
|
|
126
|
+
- platform teams maintain glue code instead of platform capabilities
|
|
127
|
+
- product teams wait while engineers renegotiate architecture
|
|
128
|
+
- vendors gain leverage because the cost of leaving keeps rising
|
|
129
|
+
|
|
130
|
+
There is no universally correct stack, and pretending otherwise is expensive.
|
|
131
|
+
The right choice today may be the wrong choice at a different scale, under a
|
|
132
|
+
new regulatory constraint, or after the team learns more. A healthy platform
|
|
133
|
+
must make change routine—not heroic.
|
|
134
|
+
|
|
135
|
+
## What If Data Platforms Were Composable?
|
|
136
|
+
|
|
137
|
+
What if a data platform described capabilities and interfaces before naming
|
|
138
|
+
products?
|
|
139
|
+
|
|
140
|
+
What if an orchestrator could request a database contract without hardcoding a
|
|
141
|
+
specific database? What if a profile could declare a supported combination,
|
|
142
|
+
validate it before deployment, and produce the runtime configuration? What if
|
|
143
|
+
replacing a component meant satisfying the same contract instead of rebuilding
|
|
144
|
+
every integration around it?
|
|
145
|
+
|
|
146
|
+
That is the goal of **Composable Data Stack (CDS)**.
|
|
147
|
+
|
|
148
|
+
CDS is an open-source framework for defining, validating, securing, and
|
|
149
|
+
assembling data platforms from reusable components. Think of it as
|
|
150
|
+
**Infrastructure as Code for the shape of a data platform**: the components,
|
|
151
|
+
the interfaces between them, and the rules that make the whole stack coherent.
|
|
152
|
+
|
|
153
|
+
CDS introduces three building blocks:
|
|
154
|
+
|
|
155
|
+
- 🔧 **Modules** package reusable capabilities such as orchestration, storage,
|
|
156
|
+
BI, and secret management.
|
|
157
|
+
- 🔗 **Contracts** define explicit, testable interfaces between those
|
|
158
|
+
capabilities.
|
|
159
|
+
- 🧩 **Profiles** compose modules into supported, reproducible stacks.
|
|
102
160
|
|
|
103
|
-
|
|
161
|
+
Instead of burying platform architecture in scripts and tribal knowledge, CDS
|
|
162
|
+
makes it declarative. It answers:
|
|
104
163
|
|
|
105
|
-
-
|
|
106
|
-
|
|
107
|
-
-
|
|
108
|
-
|
|
164
|
+
- **How do these tools fit together?** Through explicit contracts and declared
|
|
165
|
+
bindings.
|
|
166
|
+
- **What happens if I replace a component later?** CDS validates the new
|
|
167
|
+
composition against the same requirements before runtime.
|
|
168
|
+
- **How do I avoid vendor and technology lock-in?** Consumers depend on
|
|
169
|
+
capabilities, not hardcoded product identities.
|
|
170
|
+
|
|
171
|
+
The result is composability without chaos, flexibility with guarantees, and a
|
|
172
|
+
platform that can evolve without starting over.
|
|
173
|
+
|
|
174
|
+
## ⚡ Why CDS
|
|
175
|
+
|
|
176
|
+
Most teams are forced into one of two bad options:
|
|
177
|
+
|
|
178
|
+
|Approach|What You Gain|What It Costs|
|
|
179
|
+
|---|---|---|
|
|
180
|
+
|Monolithic stack|A consistent starting point|Rigid boundaries and an expensive exit|
|
|
181
|
+
|Custom integrations|Freedom to choose every component|Fragile glue, inconsistent stacks, and hidden coupling|
|
|
182
|
+
|CDS|Explicit composition and replaceable components|A contract and profile model the community maintains together|
|
|
183
|
+
|
|
184
|
+
CDS does not claim that every tool is interchangeable or that integration
|
|
185
|
+
complexity disappears. It makes compatibility explicit, validates known
|
|
186
|
+
constraints early, and gives the ecosystem a shared place to improve how tools
|
|
187
|
+
compose.
|
|
109
188
|
|
|
110
189
|
---
|
|
111
190
|
|
|
@@ -113,10 +192,12 @@ CDS gives you the best of both:
|
|
|
113
192
|
|
|
114
193
|
Use CDS if you:
|
|
115
194
|
|
|
195
|
+
- expect your architecture to evolve as requirements change
|
|
116
196
|
- want to swap tools (Airflow ↔ Dagster, Superset ↔ Metabase)
|
|
117
197
|
- need reproducible environments across dev, CI, and prod
|
|
118
198
|
- are building a platform for multiple teams
|
|
119
199
|
- want contract-driven integration instead of implicit coupling
|
|
200
|
+
- want to contribute to an open interoperability model for data tooling
|
|
120
201
|
|
|
121
202
|
CDS may be overkill if:
|
|
122
203
|
|
|
@@ -125,7 +206,7 @@ CDS may be overkill if:
|
|
|
125
206
|
|
|
126
207
|
---
|
|
127
208
|
|
|
128
|
-
## 🏗️
|
|
209
|
+
## 🏗️ See Composition in Practice
|
|
129
210
|
|
|
130
211
|
The `local-dagster-postgres-superset` profile defines:
|
|
131
212
|
|
|
@@ -133,7 +214,7 @@ The `local-dagster-postgres-superset` profile defines:
|
|
|
133
214
|
- Postgres -> storage
|
|
134
215
|
- Superset -> BI
|
|
135
216
|
|
|
136
|
-
|
|
217
|
+
CDS then:
|
|
137
218
|
|
|
138
219
|
1. Validates module definitions
|
|
139
220
|
2. Resolves contract bindings
|
|
@@ -142,7 +223,8 @@ The `local-dagster-postgres-superset` profile defines:
|
|
|
142
223
|
|
|
143
224
|
`cds plan` resolves the full dependency graph before runtime configuration is generated, ensuring all module interactions are valid and predictable.
|
|
144
225
|
|
|
145
|
-
|
|
226
|
+
The composition model creates a path to replace components while preserving
|
|
227
|
+
the contracts the rest of the platform depends on:
|
|
146
228
|
|
|
147
229
|
```text
|
|
148
230
|
Dagster -> Airflow
|
|
@@ -160,23 +242,7 @@ CDS wires modules through **contracts**, not direct dependencies. This section h
|
|
|
160
242
|
|
|
161
243
|
Below, `local-dagster-postgres-superset` wires Dagster to Postgres to Superset through contracts:
|
|
162
244
|
|
|
163
|
-
|
|
164
|
-
flowchart TD
|
|
165
|
-
Dagster[Dagster]
|
|
166
|
-
Postgres[(Postgres)]
|
|
167
|
-
Superset[Superset]
|
|
168
|
-
|
|
169
|
-
Dagster -->|transformation-runner| Postgres
|
|
170
|
-
Postgres -->|warehouse-query| Superset
|
|
171
|
-
|
|
172
|
-
classDef tool stroke:#818cf8,fill:#eef2ff
|
|
173
|
-
classDef database stroke:#2dd4bf,fill:#f0fdfa
|
|
174
|
-
classDef viz stroke:#a78bfa,fill:#f5f3ff
|
|
175
|
-
|
|
176
|
-
class Dagster tool
|
|
177
|
-
class Postgres database
|
|
178
|
-
class Superset viz
|
|
179
|
-
```
|
|
245
|
+

|
|
180
246
|
|
|
181
247
|
### Internal Flow
|
|
182
248
|
|
|
@@ -198,51 +264,7 @@ consumption in Superset), see
|
|
|
198
264
|
[docs/profile-testing/test-plan.md](docs/profile-testing/test-plan.md) and
|
|
199
265
|
[docs/profile-testing/failure-path-and-ci.md](docs/profile-testing/failure-path-and-ci.md).
|
|
200
266
|
|
|
201
|
-
|
|
202
|
-
flowchart TD
|
|
203
|
-
subgraph compile["Compile-time (cds)"]
|
|
204
|
-
direction TB
|
|
205
|
-
Profile[/profile.yaml/]
|
|
206
|
-
Validate[Validate]
|
|
207
|
-
Security["Security checks<br/>(cds test only)"]
|
|
208
|
-
Plan[Plan]
|
|
209
|
-
Render[Render]
|
|
210
|
-
Compose[/docker-compose.yaml/]
|
|
211
|
-
Stop1((stops here))
|
|
212
|
-
|
|
213
|
-
Profile --> Validate
|
|
214
|
-
Validate -->|structural + contract checks| Security
|
|
215
|
-
Security -->|rule-based checks| Plan
|
|
216
|
-
Plan -->|resolve + substitute| Render
|
|
217
|
-
Render --> Compose
|
|
218
|
-
|
|
219
|
-
Validate -.->|E020, E041, E042, E081| Stop1
|
|
220
|
-
end
|
|
221
|
-
|
|
222
|
-
subgraph runtime["Runtime (docker compose, cds up only)"]
|
|
223
|
-
direction TB
|
|
224
|
-
Build["docker compose build<br/>(skip with --no-build)"]
|
|
225
|
-
Up["docker compose up"]
|
|
226
|
-
Env[(".env file")]
|
|
227
|
-
Containers["running containers,<br/>real secret values injected"]
|
|
228
|
-
|
|
229
|
-
Build --> Up
|
|
230
|
-
Env -.->|resolves CDS_VAR| Up
|
|
231
|
-
Up --> Containers
|
|
232
|
-
end
|
|
233
|
-
|
|
234
|
-
Compose --> Build
|
|
235
|
-
|
|
236
|
-
classDef stage stroke:#818cf8,fill:#eef2ff
|
|
237
|
-
classDef artifact stroke:#2dd4bf,fill:#f0fdfa
|
|
238
|
-
classDef stop stroke:#f87171,fill:#fef2f2,stroke-dasharray: 3 3
|
|
239
|
-
classDef runtimeNode stroke:#a78bfa,fill:#f5f3ff
|
|
240
|
-
|
|
241
|
-
class Validate,Security,Plan,Render stage
|
|
242
|
-
class Profile,Compose artifact
|
|
243
|
-
class Stop1 stop
|
|
244
|
-
class Build,Up,Env,Containers runtimeNode
|
|
245
|
-
```
|
|
267
|
+

|
|
246
268
|
|
|
247
269
|
This mirrors the [`cds` command table](#️-cli) below: `validate`, `plan`, and `render` are each callable on their own; `security` only runs as part of `cds test`, not `cds up`. Module and contract definitions follow the [Contract-First](#contract-first) design principle, so most of what "Validate" and "Plan" check comes directly from `module.yaml` and `profile.yaml`.
|
|
248
270
|
|
|
@@ -272,7 +294,7 @@ cds security local-dagster-postgres-superset
|
|
|
272
294
|
|
|
273
295
|
## 📦 What You Get
|
|
274
296
|
|
|
275
|
-
When
|
|
297
|
+
When a profile compiles, CDS produces:
|
|
276
298
|
|
|
277
299
|
- validated module graph
|
|
278
300
|
- resolved contract bindings
|
|
@@ -281,7 +303,9 @@ When you run CDS:
|
|
|
281
303
|
- generated Helm chart for Kubernetes
|
|
282
304
|
- reproducible stack definition
|
|
283
305
|
|
|
284
|
-
This
|
|
306
|
+
This turns a declarative platform design into a reproducible, runnable local
|
|
307
|
+
data stack—and turns architectural assumptions into things the project can
|
|
308
|
+
inspect, validate, and improve.
|
|
285
309
|
|
|
286
310
|
---
|
|
287
311
|
|
|
@@ -990,7 +1014,20 @@ See [docs/support-policy.md](docs/support-policy.md) for OS support policy and p
|
|
|
990
1014
|
|
|
991
1015
|
## 🤝 Contributing
|
|
992
1016
|
|
|
993
|
-
|
|
1017
|
+
CDS will only become a useful interoperability layer if the people building and
|
|
1018
|
+
operating data platforms shape it together.
|
|
1019
|
+
|
|
1020
|
+
- **Run a profile** and report what works, what breaks, and what is missing.
|
|
1021
|
+
- **Document compatibility** between providers and consumers with evidence.
|
|
1022
|
+
- **Add or improve modules** without coupling them to a specific stack.
|
|
1023
|
+
- **Strengthen the model** through contracts, validation, security rules, and
|
|
1024
|
+
representative profiles.
|
|
1025
|
+
- **Star and follow** the
|
|
1026
|
+
[repository](https://github.com/RonaldHensbergen/composable-data-stack) if
|
|
1027
|
+
you want to track the project.
|
|
1028
|
+
|
|
1029
|
+
Real-world feedback is especially valuable. CDS should reflect how platforms
|
|
1030
|
+
actually fail and evolve, not how architecture diagrams pretend they do.
|
|
994
1031
|
|
|
995
1032
|
Please read these first:
|
|
996
1033
|
|
|
@@ -1002,12 +1039,8 @@ Please read these first:
|
|
|
1002
1039
|
- [CHANGELOG.md](CHANGELOG.md)
|
|
1003
1040
|
- [RELEASE.md](RELEASE.md)
|
|
1004
1041
|
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
- adding new modules
|
|
1008
|
-
- improving profile examples
|
|
1009
|
-
- extending contract definitions
|
|
1010
|
-
- adding validation or security rules
|
|
1042
|
+
Development helper tools are located in the gitignored `tools/` directory. See
|
|
1043
|
+
`tools/pr-cli/README.md` for PR creation scripts.
|
|
1011
1044
|
|
|
1012
1045
|
---
|
|
1013
1046
|
|
|
@@ -4,12 +4,12 @@ cli/diagnostics.py,sha256=Vu_uY7OPJqrAvrG-DAcIjQVMiQ4WcGWSl32UYCtOCmA,260
|
|
|
4
4
|
cli/getter.py,sha256=Q7YqG7KtcxJw8QP9dcnPU49GK6XWpEX07cUlMteMXZc,30243
|
|
5
5
|
cli/graph.py,sha256=TNx0EuTVxN4G0EIs_9eX0_x7-So3XVMh6AYjgukMMZs,1828
|
|
6
6
|
cli/image_updates.py,sha256=WCA5YNRJ2ctWJPI7xmja5T7qKJTndqpJ0qVp8dbvR8Q,11833
|
|
7
|
-
cli/image_verification.py,sha256=
|
|
8
|
-
cli/k8s_renderer.py,sha256=
|
|
9
|
-
cli/k8s_runner.py,sha256=
|
|
7
|
+
cli/image_verification.py,sha256=qNqH1IUji5LZgEgB7Qq7M1_lBzHzJCTT01aPWwSyKaM,18342
|
|
8
|
+
cli/k8s_renderer.py,sha256=GiNn_Yu1IeFba9Rz5oFnr80QWYhqWtaf7hOOZ_Y0J8I,52063
|
|
9
|
+
cli/k8s_runner.py,sha256=2zdYho-SpEhTNYxqzsUpb3J83OiBm6s0K7ASg9y4-4s,9983
|
|
10
10
|
cli/k8s_security.py,sha256=VmQ_D6UkrgLcYSum4M6TPsRisHZNMFoqKtm682wz1Ps,5768
|
|
11
11
|
cli/loader.py,sha256=KdP3sZTz8Ob5H97yE5h1tsCc5wtwZPjR7iFugxP1aPk,11825
|
|
12
|
-
cli/main.py,sha256=
|
|
12
|
+
cli/main.py,sha256=WZfFVlTJNTOgjFxrkZCZBwAYAogu6SWhHV5faAGYzp4,98725
|
|
13
13
|
cli/overlay.py,sha256=t4G9Qj9n6z2ETC47ipjU78lfYHAJOFctMkYJP7aBOC4,15315
|
|
14
14
|
cli/planner.py,sha256=gl2FGLXoGW0YzMUvCKlUjlylGHZ3jQ1WS_kYcoHuL0U,27866
|
|
15
15
|
cli/preflight.py,sha256=Juby_q6mHghwwGRFvVdpf0AkPKHxR0JtoEo73neRWCQ,14089
|
|
@@ -28,9 +28,9 @@ cli/resources/module.schema.json,sha256=gehPtHEyfWz-Pcm9UavJOF7S3ctDvllZgi8cKolx
|
|
|
28
28
|
cli/resources/profile.schema.json,sha256=fMN8dMyb-wHk8esimvopHlWIEEnozhVcCtBV-PM7j-s,6654
|
|
29
29
|
cli/resources/rule-schema.json,sha256=uRhBGm7l18ey8lxEZsdJAOSrxUgcZuYUEvXi_f2bqyQ,6840
|
|
30
30
|
cli/resources/rule-set.json,sha256=hHSJrJlCGh6gkh07KmG2qaN3k3pRSYowEbEeCAGOkXA,29411
|
|
31
|
-
composable_data_stack-0.9.
|
|
32
|
-
composable_data_stack-0.9.
|
|
33
|
-
composable_data_stack-0.9.
|
|
34
|
-
composable_data_stack-0.9.
|
|
35
|
-
composable_data_stack-0.9.
|
|
36
|
-
composable_data_stack-0.9.
|
|
31
|
+
composable_data_stack-0.9.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
32
|
+
composable_data_stack-0.9.1.dist-info/METADATA,sha256=5wJ04Nj0gY2JIO1ke1Tcfp8J4tY4uPBNOxRIjb3oQjo,41015
|
|
33
|
+
composable_data_stack-0.9.1.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
34
|
+
composable_data_stack-0.9.1.dist-info/entry_points.txt,sha256=GBhnbsVzmlnBKLC5Do-tirjz3hTzOfgGXGuZNQxb5ak,38
|
|
35
|
+
composable_data_stack-0.9.1.dist-info/top_level.txt,sha256=2ImG917oaVHlm0nP9oJE-Qrgs-fq_fGWgba2H1f8fpE,4
|
|
36
|
+
composable_data_stack-0.9.1.dist-info/RECORD,,
|
|
File without changes
|
{composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/licenses/LICENSE
RENAMED
|
File without changes
|
{composable_data_stack-0.9.0.dist-info → composable_data_stack-0.9.1.dist-info}/top_level.txt
RENAMED
|
File without changes
|