composable-data-stack 0.6.1__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/PKG-INFO +109 -7
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/README.md +107 -5
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/getter.py +114 -18
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/main.py +364 -116
- composable_data_stack-0.8.0/cli/overlay.py +427 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/planner.py +81 -7
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/preflight.py +10 -4
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/renderer.py +39 -1
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/profile.schema.json +9 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/security.py +13 -8
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/state.py +5 -2
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/validator.py +62 -14
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/PKG-INFO +109 -7
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/SOURCES.txt +3 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/requires.txt +1 -1
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/pyproject.toml +2 -2
- composable_data_stack-0.8.0/tests/test_duckdb_hardening.py +123 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_environment_cli.py +39 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_getter.py +185 -8
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_image_verification.py +25 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_main.py +280 -0
- composable_data_stack-0.8.0/tests/test_markdown_mermaid_blocks.py +56 -0
- composable_data_stack-0.8.0/tests/test_overlay.py +732 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_planner.py +235 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_publish_images_workflow.py +59 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_renderer.py +167 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_security.py +27 -0
- composable_data_stack-0.8.0/tests/test_standalone_module_profile.py +102 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_state.py +4 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_validator.py +102 -0
- composable_data_stack-0.6.1/cli/overlay.py +0 -239
- composable_data_stack-0.6.1/tests/test_overlay.py +0 -280
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/LICENSE +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/__init__.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/constants.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/diagnostics.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/graph.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/image_updates.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/image_verification.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/loader.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resolver.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/__init__.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/contract.schema.json +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/module.schema.json +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/rule-schema.json +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/rule-set.json +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/secrets.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/security_common.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/up_runner.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/utils.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/dependency_links.txt +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/entry_points.txt +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/top_level.txt +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/setup.cfg +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_cds_workflow.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_completion.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_compose_runtime_smoke.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_config_generation.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_definitions_uri.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_entrypoint_backend_guard.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_hardening.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_environment_classification.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_graph.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_image_security_scan_workflow.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_image_updates.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_module_isolation.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_modules_no_committed_secrets.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_mvp_proof_workflow.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_preflight.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_python_version_alignment.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_release_version_check.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_render_example_profile.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_resolve_profile_path.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_resolver.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_secrets.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_smoke_example_profile.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_superset_hardening.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_trivyignore.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_up_runner.py +0 -0
- {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_upstream_image_hardening.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: composable-data-stack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: CLI for validating and composing composable data stack profiles
|
|
5
5
|
Author: Ronald Hensbergen
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -31,7 +31,7 @@ Provides-Extra: dev
|
|
|
31
31
|
Requires-Dist: coverage>=7.6; extra == "dev"
|
|
32
32
|
Requires-Dist: bandit>=1.7; extra == "dev"
|
|
33
33
|
Requires-Dist: pip-audit>=2.7; extra == "dev"
|
|
34
|
-
Requires-Dist: ruff==0.16.
|
|
34
|
+
Requires-Dist: ruff==0.16.6; extra == "dev"
|
|
35
35
|
Dynamic: license-file
|
|
36
36
|
|
|
37
37
|
# 🚀 Composable Data Stack (CDS)
|
|
@@ -135,7 +135,6 @@ CDS wires modules through **contracts**, not direct dependencies. This section h
|
|
|
135
135
|
Below, `local-dagster-postgres-superset` wires Dagster to Postgres to Superset through contracts:
|
|
136
136
|
|
|
137
137
|
```mermaid
|
|
138
|
-
---
|
|
139
138
|
flowchart TD
|
|
140
139
|
Dagster[Dagster]
|
|
141
140
|
Postgres[(Postgres)]
|
|
@@ -174,7 +173,6 @@ consumption in Superset), see
|
|
|
174
173
|
[docs/profile-testing/failure-path-and-ci.md](docs/profile-testing/failure-path-and-ci.md).
|
|
175
174
|
|
|
176
175
|
```mermaid
|
|
177
|
-
---
|
|
178
176
|
flowchart TD
|
|
179
177
|
subgraph compile["Compile-time (cds)"]
|
|
180
178
|
direction TB
|
|
@@ -633,8 +631,79 @@ Differences from 'dev' to 'prod':
|
|
|
633
631
|
~ spec.modules[postgres].config.storage.size: "2Gi" -> "20Gi"
|
|
634
632
|
```
|
|
635
633
|
|
|
636
|
-
|
|
637
|
-
|
|
634
|
+
Persist an environment default for a project with `cds config set environment
|
|
635
|
+
prod`; commands without `--environment` then use that overlay. An explicit
|
|
636
|
+
`--environment` always takes precedence.
|
|
637
|
+
|
|
638
|
+
---
|
|
639
|
+
|
|
640
|
+
### Profile Composition (`extends`)
|
|
641
|
+
|
|
642
|
+
A profile can also factor out shared configuration into one or more parent
|
|
643
|
+
profiles instead of duplicating it, using a top-level `extends` field:
|
|
644
|
+
|
|
645
|
+
```yaml
|
|
646
|
+
# profiles/analytics-prod/profile.yaml
|
|
647
|
+
apiVersion: cds/v1alpha1
|
|
648
|
+
kind: Profile
|
|
649
|
+
metadata:
|
|
650
|
+
name: analytics-prod
|
|
651
|
+
environment: production
|
|
652
|
+
extends:
|
|
653
|
+
- analytics-base # a bare name resolves to profiles/analytics-base/profile.yaml
|
|
654
|
+
spec:
|
|
655
|
+
modules:
|
|
656
|
+
- id: postgres
|
|
657
|
+
config:
|
|
658
|
+
storage:
|
|
659
|
+
size: 20Gi
|
|
660
|
+
```
|
|
661
|
+
|
|
662
|
+
`extends` accepts a non-empty list of parent references, each either a bare
|
|
663
|
+
profile name (resolved under the profiles root) or a path relative to the
|
|
664
|
+
child profile's directory (e.g. `../shared/profile.yaml`). Parents are
|
|
665
|
+
resolved and merged left-to-right — later parents win over earlier ones —
|
|
666
|
+
and then the child profile's own document is merged on top of all parents.
|
|
667
|
+
Composition uses the exact same deep-merge/module-merge-by-id engine as
|
|
668
|
+
environment overlays: mappings merge recursively, `spec.modules` entries
|
|
669
|
+
merge by stable `id`, and any other array is replaced wholesale rather than
|
|
670
|
+
concatenated. Parent profiles may themselves use `extends` (chains are
|
|
671
|
+
resolved transitively).
|
|
672
|
+
|
|
673
|
+
`extends` is not a CLI flag — it's read directly from `profile.yaml`, so
|
|
674
|
+
every profile-consuming command resolves it automatically, with no new
|
|
675
|
+
syntax to learn:
|
|
676
|
+
|
|
677
|
+
```bash
|
|
678
|
+
cds validate analytics-prod
|
|
679
|
+
cds plan analytics-prod
|
|
680
|
+
cds up analytics-prod
|
|
681
|
+
```
|
|
682
|
+
|
|
683
|
+
A profile can extend more than one parent, which is merged in the order
|
|
684
|
+
listed (later entries win over earlier ones):
|
|
685
|
+
|
|
686
|
+
```yaml
|
|
687
|
+
extends:
|
|
688
|
+
- networking-base
|
|
689
|
+
- observability-base
|
|
690
|
+
```
|
|
691
|
+
|
|
692
|
+
`extends` and `--environment` compose together: parents are merged first,
|
|
693
|
+
then the child, then the selected environment overlay is applied on top of
|
|
694
|
+
that fully-composed result — so a shared base profile and environment
|
|
695
|
+
promotion can both be used without duplicating configuration in either
|
|
696
|
+
dimension.
|
|
697
|
+
|
|
698
|
+
A malformed `extends` chain fails validation/planning before anything else
|
|
699
|
+
runs, with a dedicated diagnostic code:
|
|
700
|
+
|
|
701
|
+
|Code|Meaning|
|
|
702
|
+
|---|---|
|
|
703
|
+
|E110|`extends` is missing, not a list, empty, or contains a non-string/empty entry|
|
|
704
|
+
|E111|A parent reference resolves outside the profiles root|
|
|
705
|
+
|E112|A referenced parent profile does not exist|
|
|
706
|
+
|E113|A cycle was detected in the `extends` chain|
|
|
638
707
|
|
|
639
708
|
---
|
|
640
709
|
|
|
@@ -643,6 +712,7 @@ before and never look for an `environments/` directory.
|
|
|
643
712
|
|Command|Description|
|
|
644
713
|
|---|---|
|
|
645
714
|
|cds get \<profile\> [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>] [--into \<dir\>]|Fetch a profile plus its dependent module/runtime assets from GitHub into a local CDS layout|
|
|
715
|
+
|cds list profiles\|modules\|images [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>]|List available profiles, module sources, or module images and check for newer versions; add `--remote`/`--local` to inspect another repository before fetching from it|
|
|
646
716
|
|cds init [profile]|Generate a project `.env` template from profile secret definitions|
|
|
647
717
|
|cds validate [profile]|Validate modules and contracts|
|
|
648
718
|
|cds preflight [profile]|Check runtime tools, required environment values, and host ports without starting services|
|
|
@@ -654,6 +724,7 @@ before and never look for an `environments/` directory.
|
|
|
654
724
|
|cds security [profile]|Run rule-based security validation on a profile|
|
|
655
725
|
|cds diff [profile] --from \<env\> --to \<env\>|Show effective configuration differences between two environment overlays, secrets never included|
|
|
656
726
|
|cds use [profile] [--clear]|Save (show/clear) a default profile so it doesn't have to be passed to other commands|
|
|
727
|
+
|cds config get\|set\|unset\|list|Manage persisted project defaults in `.cds/config.json`|
|
|
657
728
|
|cds completion \<bash\|zsh\|powershell\>|Print shell setup instructions for tab-completion|
|
|
658
729
|
|
|
659
730
|
`init`, `validate`, `preflight`, `plan`, `render`, `up`, `test`, and `security`
|
|
@@ -674,6 +745,35 @@ destination root, `--dry-run` to inspect the copy plan first, and `--force` to
|
|
|
674
745
|
replace conflicting local files. Successful fetches record tracking metadata
|
|
675
746
|
in `.cds/get-manifest.json` for future update workflows.
|
|
676
747
|
|
|
748
|
+
`cds list profiles`, `cds list modules`, and `cds list images` accept the same
|
|
749
|
+
`--remote`/`--ref`/`--local` source-repository selection as `cds get`, so you
|
|
750
|
+
can discover what's available in another repository (a fork, or an existing
|
|
751
|
+
local checkout) before running `cds get` against it. Without these flags,
|
|
752
|
+
`cds list` inspects the local project as before.
|
|
753
|
+
|
|
754
|
+
### Project defaults
|
|
755
|
+
|
|
756
|
+
`cds config` manages the gitignored `.cds/config.json` file (or the path in
|
|
757
|
+
`CDS_CONFIG_PATH`). Supported settings are `profile`, `environment`, and
|
|
758
|
+
`security.strict`:
|
|
759
|
+
|
|
760
|
+
```bash
|
|
761
|
+
cds config set profile my-profile
|
|
762
|
+
cds config set environment prod
|
|
763
|
+
cds config set security.strict true
|
|
764
|
+
cds config list
|
|
765
|
+
```
|
|
766
|
+
|
|
767
|
+
`profile` is stored as its resolved path, and `environment` is validated
|
|
768
|
+
against that profile's `environments/` directory. `security.strict true`
|
|
769
|
+
applies the existing production security rules even if the profile declares a
|
|
770
|
+
local environment. `cds use` remains a shortcut for setting, showing, or
|
|
771
|
+
clearing `profile`.
|
|
772
|
+
|
|
773
|
+
CLI flags take precedence over these defaults. In particular,
|
|
774
|
+
`--environment` overrides `config environment`; `CDS_PROFILE_PATH` continues
|
|
775
|
+
to override the saved profile.
|
|
776
|
+
|
|
677
777
|
`[profile]` accepts:
|
|
678
778
|
|
|
679
779
|
| Form | Example |
|
|
@@ -682,7 +782,7 @@ in `.cds/get-manifest.json` for future update workflows.
|
|
|
682
782
|
| Path to a `profile.yaml` file | `profiles/local-dagster-postgres-superset/profile.yaml` |
|
|
683
783
|
| Path to a profiles root directory | `profiles/` |
|
|
684
784
|
|
|
685
|
-
When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds use <profile
|
|
785
|
+
When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds config set profile` (or `cds use <profile>`), then the single profile under `profiles/` if there is exactly one. An explicitly-set env var takes precedence over the persisted project default, matching common CLI convention (env vars are per-invocation and reflect the current session more reliably than a saved, gitignored default that's easy to forget about).
|
|
686
786
|
|
|
687
787
|
To view the full list of options for any command, use the `--help` flag:
|
|
688
788
|
|
|
@@ -730,6 +830,8 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
|
|
|
730
830
|
| `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
|
|
731
831
|
| `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
|
|
732
832
|
| `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
|
|
833
|
+
| `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
|
|
834
|
+
| `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
|
|
733
835
|
|
|
734
836
|
All diagnostics print with their error code and YAML path (e.g. `spec.modules[1].config`), so search the profile file for that path to find the exact line to fix.
|
|
735
837
|
|
|
@@ -99,7 +99,6 @@ CDS wires modules through **contracts**, not direct dependencies. This section h
|
|
|
99
99
|
Below, `local-dagster-postgres-superset` wires Dagster to Postgres to Superset through contracts:
|
|
100
100
|
|
|
101
101
|
```mermaid
|
|
102
|
-
---
|
|
103
102
|
flowchart TD
|
|
104
103
|
Dagster[Dagster]
|
|
105
104
|
Postgres[(Postgres)]
|
|
@@ -138,7 +137,6 @@ consumption in Superset), see
|
|
|
138
137
|
[docs/profile-testing/failure-path-and-ci.md](docs/profile-testing/failure-path-and-ci.md).
|
|
139
138
|
|
|
140
139
|
```mermaid
|
|
141
|
-
---
|
|
142
140
|
flowchart TD
|
|
143
141
|
subgraph compile["Compile-time (cds)"]
|
|
144
142
|
direction TB
|
|
@@ -597,8 +595,79 @@ Differences from 'dev' to 'prod':
|
|
|
597
595
|
~ spec.modules[postgres].config.storage.size: "2Gi" -> "20Gi"
|
|
598
596
|
```
|
|
599
597
|
|
|
600
|
-
|
|
601
|
-
|
|
598
|
+
Persist an environment default for a project with `cds config set environment
|
|
599
|
+
prod`; commands without `--environment` then use that overlay. An explicit
|
|
600
|
+
`--environment` always takes precedence.
|
|
601
|
+
|
|
602
|
+
---
|
|
603
|
+
|
|
604
|
+
### Profile Composition (`extends`)
|
|
605
|
+
|
|
606
|
+
A profile can also factor out shared configuration into one or more parent
|
|
607
|
+
profiles instead of duplicating it, using a top-level `extends` field:
|
|
608
|
+
|
|
609
|
+
```yaml
|
|
610
|
+
# profiles/analytics-prod/profile.yaml
|
|
611
|
+
apiVersion: cds/v1alpha1
|
|
612
|
+
kind: Profile
|
|
613
|
+
metadata:
|
|
614
|
+
name: analytics-prod
|
|
615
|
+
environment: production
|
|
616
|
+
extends:
|
|
617
|
+
- analytics-base # a bare name resolves to profiles/analytics-base/profile.yaml
|
|
618
|
+
spec:
|
|
619
|
+
modules:
|
|
620
|
+
- id: postgres
|
|
621
|
+
config:
|
|
622
|
+
storage:
|
|
623
|
+
size: 20Gi
|
|
624
|
+
```
|
|
625
|
+
|
|
626
|
+
`extends` accepts a non-empty list of parent references, each either a bare
|
|
627
|
+
profile name (resolved under the profiles root) or a path relative to the
|
|
628
|
+
child profile's directory (e.g. `../shared/profile.yaml`). Parents are
|
|
629
|
+
resolved and merged left-to-right — later parents win over earlier ones —
|
|
630
|
+
and then the child profile's own document is merged on top of all parents.
|
|
631
|
+
Composition uses the exact same deep-merge/module-merge-by-id engine as
|
|
632
|
+
environment overlays: mappings merge recursively, `spec.modules` entries
|
|
633
|
+
merge by stable `id`, and any other array is replaced wholesale rather than
|
|
634
|
+
concatenated. Parent profiles may themselves use `extends` (chains are
|
|
635
|
+
resolved transitively).
|
|
636
|
+
|
|
637
|
+
`extends` is not a CLI flag — it's read directly from `profile.yaml`, so
|
|
638
|
+
every profile-consuming command resolves it automatically, with no new
|
|
639
|
+
syntax to learn:
|
|
640
|
+
|
|
641
|
+
```bash
|
|
642
|
+
cds validate analytics-prod
|
|
643
|
+
cds plan analytics-prod
|
|
644
|
+
cds up analytics-prod
|
|
645
|
+
```
|
|
646
|
+
|
|
647
|
+
A profile can extend more than one parent, which is merged in the order
|
|
648
|
+
listed (later entries win over earlier ones):
|
|
649
|
+
|
|
650
|
+
```yaml
|
|
651
|
+
extends:
|
|
652
|
+
- networking-base
|
|
653
|
+
- observability-base
|
|
654
|
+
```
|
|
655
|
+
|
|
656
|
+
`extends` and `--environment` compose together: parents are merged first,
|
|
657
|
+
then the child, then the selected environment overlay is applied on top of
|
|
658
|
+
that fully-composed result — so a shared base profile and environment
|
|
659
|
+
promotion can both be used without duplicating configuration in either
|
|
660
|
+
dimension.
|
|
661
|
+
|
|
662
|
+
A malformed `extends` chain fails validation/planning before anything else
|
|
663
|
+
runs, with a dedicated diagnostic code:
|
|
664
|
+
|
|
665
|
+
|Code|Meaning|
|
|
666
|
+
|---|---|
|
|
667
|
+
|E110|`extends` is missing, not a list, empty, or contains a non-string/empty entry|
|
|
668
|
+
|E111|A parent reference resolves outside the profiles root|
|
|
669
|
+
|E112|A referenced parent profile does not exist|
|
|
670
|
+
|E113|A cycle was detected in the `extends` chain|
|
|
602
671
|
|
|
603
672
|
---
|
|
604
673
|
|
|
@@ -607,6 +676,7 @@ before and never look for an `environments/` directory.
|
|
|
607
676
|
|Command|Description|
|
|
608
677
|
|---|---|
|
|
609
678
|
|cds get \<profile\> [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>] [--into \<dir\>]|Fetch a profile plus its dependent module/runtime assets from GitHub into a local CDS layout|
|
|
679
|
+
|cds list profiles\|modules\|images [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>]|List available profiles, module sources, or module images and check for newer versions; add `--remote`/`--local` to inspect another repository before fetching from it|
|
|
610
680
|
|cds init [profile]|Generate a project `.env` template from profile secret definitions|
|
|
611
681
|
|cds validate [profile]|Validate modules and contracts|
|
|
612
682
|
|cds preflight [profile]|Check runtime tools, required environment values, and host ports without starting services|
|
|
@@ -618,6 +688,7 @@ before and never look for an `environments/` directory.
|
|
|
618
688
|
|cds security [profile]|Run rule-based security validation on a profile|
|
|
619
689
|
|cds diff [profile] --from \<env\> --to \<env\>|Show effective configuration differences between two environment overlays, secrets never included|
|
|
620
690
|
|cds use [profile] [--clear]|Save (show/clear) a default profile so it doesn't have to be passed to other commands|
|
|
691
|
+
|cds config get\|set\|unset\|list|Manage persisted project defaults in `.cds/config.json`|
|
|
621
692
|
|cds completion \<bash\|zsh\|powershell\>|Print shell setup instructions for tab-completion|
|
|
622
693
|
|
|
623
694
|
`init`, `validate`, `preflight`, `plan`, `render`, `up`, `test`, and `security`
|
|
@@ -638,6 +709,35 @@ destination root, `--dry-run` to inspect the copy plan first, and `--force` to
|
|
|
638
709
|
replace conflicting local files. Successful fetches record tracking metadata
|
|
639
710
|
in `.cds/get-manifest.json` for future update workflows.
|
|
640
711
|
|
|
712
|
+
`cds list profiles`, `cds list modules`, and `cds list images` accept the same
|
|
713
|
+
`--remote`/`--ref`/`--local` source-repository selection as `cds get`, so you
|
|
714
|
+
can discover what's available in another repository (a fork, or an existing
|
|
715
|
+
local checkout) before running `cds get` against it. Without these flags,
|
|
716
|
+
`cds list` inspects the local project as before.
|
|
717
|
+
|
|
718
|
+
### Project defaults
|
|
719
|
+
|
|
720
|
+
`cds config` manages the gitignored `.cds/config.json` file (or the path in
|
|
721
|
+
`CDS_CONFIG_PATH`). Supported settings are `profile`, `environment`, and
|
|
722
|
+
`security.strict`:
|
|
723
|
+
|
|
724
|
+
```bash
|
|
725
|
+
cds config set profile my-profile
|
|
726
|
+
cds config set environment prod
|
|
727
|
+
cds config set security.strict true
|
|
728
|
+
cds config list
|
|
729
|
+
```
|
|
730
|
+
|
|
731
|
+
`profile` is stored as its resolved path, and `environment` is validated
|
|
732
|
+
against that profile's `environments/` directory. `security.strict true`
|
|
733
|
+
applies the existing production security rules even if the profile declares a
|
|
734
|
+
local environment. `cds use` remains a shortcut for setting, showing, or
|
|
735
|
+
clearing `profile`.
|
|
736
|
+
|
|
737
|
+
CLI flags take precedence over these defaults. In particular,
|
|
738
|
+
`--environment` overrides `config environment`; `CDS_PROFILE_PATH` continues
|
|
739
|
+
to override the saved profile.
|
|
740
|
+
|
|
641
741
|
`[profile]` accepts:
|
|
642
742
|
|
|
643
743
|
| Form | Example |
|
|
@@ -646,7 +746,7 @@ in `.cds/get-manifest.json` for future update workflows.
|
|
|
646
746
|
| Path to a `profile.yaml` file | `profiles/local-dagster-postgres-superset/profile.yaml` |
|
|
647
747
|
| Path to a profiles root directory | `profiles/` |
|
|
648
748
|
|
|
649
|
-
When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds use <profile
|
|
749
|
+
When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds config set profile` (or `cds use <profile>`), then the single profile under `profiles/` if there is exactly one. An explicitly-set env var takes precedence over the persisted project default, matching common CLI convention (env vars are per-invocation and reflect the current session more reliably than a saved, gitignored default that's easy to forget about).
|
|
650
750
|
|
|
651
751
|
To view the full list of options for any command, use the `--help` flag:
|
|
652
752
|
|
|
@@ -694,6 +794,8 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
|
|
|
694
794
|
| `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
|
|
695
795
|
| `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
|
|
696
796
|
| `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
|
|
797
|
+
| `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
|
|
798
|
+
| `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
|
|
697
799
|
|
|
698
800
|
All diagnostics print with their error code and YAML path (e.g. `spec.modules[1].config`), so search the profile file for that path to find the exact line to fix.
|
|
699
801
|
|
|
@@ -18,6 +18,7 @@ from urllib.error import HTTPError, URLError
|
|
|
18
18
|
from urllib.request import Request, urlopen
|
|
19
19
|
|
|
20
20
|
from .loader import load_yaml_file, resolve_module_dir
|
|
21
|
+
from .overlay import _derive_profiles_root, _resolve_extends_ref, resolve_extends
|
|
21
22
|
from .planner import MaxNestingDepthExceeded, apply_defaults, substitute_string
|
|
22
23
|
|
|
23
24
|
# The upstream repository `cds get` downloads from when no `--remote` is
|
|
@@ -71,7 +72,7 @@ def fetch_profile(
|
|
|
71
72
|
destination_root: Path | None = None,
|
|
72
73
|
force: bool = False,
|
|
73
74
|
dry_run: bool = False,
|
|
74
|
-
) -> tuple[list[CopyAction], Path]:
|
|
75
|
+
) -> tuple[list[CopyAction], Path, list[str]]:
|
|
75
76
|
target_root = (destination_root or Path.cwd()).expanduser().resolve()
|
|
76
77
|
|
|
77
78
|
with _prepare_source_repository(remote, ref, local) as source_repo:
|
|
@@ -79,10 +80,15 @@ def fetch_profile(
|
|
|
79
80
|
asset_roots = _collect_asset_roots(source_repo, profile_path)
|
|
80
81
|
|
|
81
82
|
actions = _build_copy_plan(source_repo, asset_roots, target_root)
|
|
83
|
+
# Conflict detection must happen while `source_repo` (which may be a
|
|
84
|
+
# temporary directory holding a downloaded tarball) is still alive,
|
|
85
|
+
# since `_find_conflicts()` reads each action's source file to
|
|
86
|
+
# compare content. Compute it up front so both the dry-run and
|
|
87
|
+
# real-run paths can report/enforce conflicts consistently (#452).
|
|
88
|
+
conflicts = _find_conflicts(actions)
|
|
82
89
|
if dry_run:
|
|
83
|
-
return actions, target_root / _TRACKING_FILE
|
|
90
|
+
return actions, target_root / _TRACKING_FILE, conflicts
|
|
84
91
|
|
|
85
|
-
conflicts = _find_conflicts(actions)
|
|
86
92
|
if conflicts and not force:
|
|
87
93
|
rendered = ", ".join(conflicts[:5])
|
|
88
94
|
extra = "" if len(conflicts) <= 5 else f" (+{len(conflicts) - 5} more)"
|
|
@@ -104,18 +110,37 @@ def fetch_profile(
|
|
|
104
110
|
actions=actions,
|
|
105
111
|
asset_roots=asset_roots,
|
|
106
112
|
)
|
|
107
|
-
return actions, target_root / _TRACKING_FILE
|
|
113
|
+
return actions, target_root / _TRACKING_FILE, conflicts
|
|
108
114
|
|
|
109
115
|
|
|
110
|
-
def format_get_plan(
|
|
116
|
+
def format_get_plan(
|
|
117
|
+
actions: list[CopyAction],
|
|
118
|
+
*,
|
|
119
|
+
destination_root: Path,
|
|
120
|
+
conflicts: list[str] | None = None,
|
|
121
|
+
) -> str:
|
|
122
|
+
conflicts = conflicts or []
|
|
123
|
+
conflict_set = set(conflicts)
|
|
124
|
+
clean_actions = [a for a in actions if a.repo_relative_path not in conflict_set]
|
|
125
|
+
|
|
111
126
|
if not actions:
|
|
112
127
|
return f"No file changes required under {destination_root}."
|
|
113
128
|
|
|
114
129
|
lines = [f"Planned {len(actions)} file(s) under {destination_root}:"]
|
|
115
|
-
|
|
116
|
-
lines.append(f"
|
|
117
|
-
|
|
118
|
-
|
|
130
|
+
if clean_actions:
|
|
131
|
+
lines.append(f" Would copy ({len(clean_actions)}):")
|
|
132
|
+
for action in clean_actions[:20]:
|
|
133
|
+
lines.append(f" - {action.repo_relative_path}")
|
|
134
|
+
if len(clean_actions) > 20:
|
|
135
|
+
lines.append(f" - ... {len(clean_actions) - 20} more")
|
|
136
|
+
|
|
137
|
+
if conflicts:
|
|
138
|
+
lines.append(f" Would conflict without --force ({len(conflicts)}):")
|
|
139
|
+
for path in conflicts[:20]:
|
|
140
|
+
lines.append(f" - {path}")
|
|
141
|
+
if len(conflicts) > 20:
|
|
142
|
+
lines.append(f" - ... {len(conflicts) - 20} more")
|
|
143
|
+
|
|
119
144
|
return "\n".join(lines)
|
|
120
145
|
|
|
121
146
|
|
|
@@ -233,15 +258,67 @@ def _resolve_source_profile_path(source_repo: Path, profile: str) -> Path:
|
|
|
233
258
|
)
|
|
234
259
|
|
|
235
260
|
|
|
261
|
+
def _collect_extends_profile_dirs(
|
|
262
|
+
source_repo: Path, profile_path: Path, _visited: frozenset[Path] = frozenset()
|
|
263
|
+
) -> set[Path]:
|
|
264
|
+
"""
|
|
265
|
+
Returns every parent profile directory reachable via `extends`, so
|
|
266
|
+
`cds get` copies parent profile.yaml files (and, via
|
|
267
|
+
_collect_asset_roots's module walk, their modules) too instead of
|
|
268
|
+
silently omitting anything only declared in a parent profile. Reuses
|
|
269
|
+
cli.overlay's own extends-ref resolution rather than re-implementing it.
|
|
270
|
+
|
|
271
|
+
`_visited` tracks resolved profile paths already seen along the current
|
|
272
|
+
extends chain so a cyclic `extends` graph raises a clean GetError
|
|
273
|
+
instead of recursing until Python's recursion limit is hit; this
|
|
274
|
+
mirrors cli.overlay._compose_extends's own cycle-detection stack.
|
|
275
|
+
"""
|
|
276
|
+
resolved_profile_path = profile_path.resolve()
|
|
277
|
+
if resolved_profile_path in _visited:
|
|
278
|
+
raise GetError(f"Cycle detected in extends chain at {profile_path}")
|
|
279
|
+
_visited = _visited | {resolved_profile_path}
|
|
280
|
+
|
|
281
|
+
doc, _diagnostics = load_yaml_file(profile_path)
|
|
282
|
+
if doc is None:
|
|
283
|
+
raise GetError(f"Could not load source profile {profile_path}")
|
|
284
|
+
|
|
285
|
+
extends = doc.get("extends")
|
|
286
|
+
if not extends:
|
|
287
|
+
return set()
|
|
288
|
+
|
|
289
|
+
profile_dir = profile_path.parent.resolve()
|
|
290
|
+
profiles_root = _derive_profiles_root(profile_dir)
|
|
291
|
+
if profiles_root is None:
|
|
292
|
+
raise GetError(f'Could not derive a profiles root for extends in {profile_path}')
|
|
293
|
+
|
|
294
|
+
dirs: set[Path] = set()
|
|
295
|
+
for ref in extends:
|
|
296
|
+
parent_path = _resolve_extends_ref(ref, profile_dir, profiles_root)
|
|
297
|
+
if not parent_path.is_file():
|
|
298
|
+
raise GetError(f'Could not resolve extends parent "{ref}" for {profile_path}')
|
|
299
|
+
_require_within_repo(parent_path, source_repo, f'extends parent "{ref}"')
|
|
300
|
+
dirs.add(parent_path.parent)
|
|
301
|
+
dirs.update(_collect_extends_profile_dirs(source_repo, parent_path, _visited))
|
|
302
|
+
|
|
303
|
+
return dirs
|
|
304
|
+
|
|
305
|
+
|
|
236
306
|
def _collect_asset_roots(source_repo: Path, profile_path: Path) -> list[Path]:
|
|
237
307
|
profile_dir = profile_path.parent
|
|
238
308
|
asset_roots: set[Path] = {
|
|
239
309
|
profile_dir if profile_path.name == "profile.yaml" else profile_path
|
|
240
310
|
}
|
|
241
|
-
|
|
242
|
-
|
|
311
|
+
|
|
312
|
+
# Resolve through `extends` so modules/config contributed only by a
|
|
313
|
+
# parent profile (not redeclared in the child) are still collected;
|
|
314
|
+
# cli.overlay is the single source of truth for extends semantics (see
|
|
315
|
+
# cli.planner/cli.validator, which route through it the same way).
|
|
316
|
+
profile_doc, _provenance, diagnostics = resolve_extends(str(profile_path))
|
|
317
|
+
if profile_doc is None or any(d.level == "error" for d in diagnostics):
|
|
243
318
|
raise GetError(f"Could not load source profile {profile_path}")
|
|
244
319
|
|
|
320
|
+
asset_roots.update(_collect_extends_profile_dirs(source_repo, profile_path))
|
|
321
|
+
|
|
245
322
|
spec = profile_doc.get("spec")
|
|
246
323
|
modules = spec.get("modules", []) if isinstance(spec, dict) else []
|
|
247
324
|
if not isinstance(modules, list):
|
|
@@ -595,7 +672,19 @@ def _add_copy_action(
|
|
|
595
672
|
destination_root: Path,
|
|
596
673
|
actions_by_destination: dict[Path, CopyAction],
|
|
597
674
|
) -> None:
|
|
598
|
-
|
|
675
|
+
resolved_source = source_file.resolve()
|
|
676
|
+
try:
|
|
677
|
+
repo_relative = resolved_source.relative_to(source_repo.resolve())
|
|
678
|
+
except ValueError as exc:
|
|
679
|
+
# A symlink (or directory junction) planted under an asset root can
|
|
680
|
+
# resolve to a real path outside the source repository. Without this
|
|
681
|
+
# guard, Path.relative_to() raises a bare ValueError that escapes as
|
|
682
|
+
# an unhandled traceback instead of the stable GetError callers rely
|
|
683
|
+
# on (#454).
|
|
684
|
+
raise GetError(
|
|
685
|
+
f"Asset {source_file} resolves outside the source repository "
|
|
686
|
+
f"({source_repo}): {resolved_source}"
|
|
687
|
+
) from exc
|
|
599
688
|
# Deliberately do NOT call .resolve() on the combined destination path:
|
|
600
689
|
# destination_root is already an absolute, resolved path (see
|
|
601
690
|
# fetch_profile()), and resolving the full path here would follow a
|
|
@@ -607,12 +696,12 @@ def _add_copy_action(
|
|
|
607
696
|
existing = actions_by_destination.get(destination)
|
|
608
697
|
if existing is None:
|
|
609
698
|
actions_by_destination[destination] = CopyAction(
|
|
610
|
-
source=
|
|
699
|
+
source=resolved_source,
|
|
611
700
|
destination=destination,
|
|
612
701
|
repo_relative_path=repo_relative.as_posix(),
|
|
613
702
|
)
|
|
614
703
|
return
|
|
615
|
-
if existing.source.resolve() !=
|
|
704
|
+
if existing.source.resolve() != resolved_source:
|
|
616
705
|
raise GetError(
|
|
617
706
|
f"Multiple source files would map to the same destination: {destination}"
|
|
618
707
|
)
|
|
@@ -649,10 +738,17 @@ def _write_actions(actions: list[CopyAction]) -> None:
|
|
|
649
738
|
(e.g. a symlink planted at profiles/foo/profile.yaml pointing outside
|
|
650
739
|
the destination tree)."""
|
|
651
740
|
for action in actions:
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
action.destination.
|
|
655
|
-
|
|
741
|
+
try:
|
|
742
|
+
action.destination.parent.mkdir(parents=True, exist_ok=True)
|
|
743
|
+
if action.destination.is_symlink():
|
|
744
|
+
action.destination.unlink()
|
|
745
|
+
shutil.copy2(action.source, action.destination)
|
|
746
|
+
except OSError as exc:
|
|
747
|
+
# Surface filesystem failures (permission denied, read-only
|
|
748
|
+
# destination, disk full, ...) as a stable GetError instead of a
|
|
749
|
+
# raw OSError, so callers get a predictable error path and
|
|
750
|
+
# message regardless of the underlying platform/errno (#454).
|
|
751
|
+
raise GetError(f"Could not write {action.destination}: {exc}") from exc
|
|
656
752
|
|
|
657
753
|
|
|
658
754
|
def _write_tracking_manifest(
|