composable-data-stack 0.6.1__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/PKG-INFO +109 -7
  2. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/README.md +107 -5
  3. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/getter.py +114 -18
  4. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/main.py +364 -116
  5. composable_data_stack-0.8.0/cli/overlay.py +427 -0
  6. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/planner.py +81 -7
  7. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/preflight.py +10 -4
  8. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/renderer.py +39 -1
  9. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/profile.schema.json +9 -0
  10. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/security.py +13 -8
  11. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/state.py +5 -2
  12. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/validator.py +62 -14
  13. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/PKG-INFO +109 -7
  14. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/SOURCES.txt +3 -0
  15. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/requires.txt +1 -1
  16. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/pyproject.toml +2 -2
  17. composable_data_stack-0.8.0/tests/test_duckdb_hardening.py +123 -0
  18. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_environment_cli.py +39 -0
  19. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_getter.py +185 -8
  20. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_image_verification.py +25 -0
  21. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_main.py +280 -0
  22. composable_data_stack-0.8.0/tests/test_markdown_mermaid_blocks.py +56 -0
  23. composable_data_stack-0.8.0/tests/test_overlay.py +732 -0
  24. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_planner.py +235 -0
  25. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_publish_images_workflow.py +59 -0
  26. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_renderer.py +167 -0
  27. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_security.py +27 -0
  28. composable_data_stack-0.8.0/tests/test_standalone_module_profile.py +102 -0
  29. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_state.py +4 -0
  30. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_validator.py +102 -0
  31. composable_data_stack-0.6.1/cli/overlay.py +0 -239
  32. composable_data_stack-0.6.1/tests/test_overlay.py +0 -280
  33. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/LICENSE +0 -0
  34. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/__init__.py +0 -0
  35. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/constants.py +0 -0
  36. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/diagnostics.py +0 -0
  37. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/graph.py +0 -0
  38. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/image_updates.py +0 -0
  39. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/image_verification.py +0 -0
  40. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/loader.py +0 -0
  41. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resolver.py +0 -0
  42. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/__init__.py +0 -0
  43. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/contract.schema.json +0 -0
  44. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/module.schema.json +0 -0
  45. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/rule-schema.json +0 -0
  46. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/resources/rule-set.json +0 -0
  47. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/secrets.py +0 -0
  48. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/security_common.py +0 -0
  49. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/up_runner.py +0 -0
  50. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/cli/utils.py +0 -0
  51. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/dependency_links.txt +0 -0
  52. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/entry_points.txt +0 -0
  53. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/composable_data_stack.egg-info/top_level.txt +0 -0
  54. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/setup.cfg +0 -0
  55. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_cds_workflow.py +0 -0
  56. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_completion.py +0 -0
  57. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_compose_runtime_smoke.py +0 -0
  58. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_config_generation.py +0 -0
  59. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_definitions_uri.py +0 -0
  60. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_entrypoint_backend_guard.py +0 -0
  61. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_dagster_hardening.py +0 -0
  62. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_environment_classification.py +0 -0
  63. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_graph.py +0 -0
  64. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_image_security_scan_workflow.py +0 -0
  65. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_image_updates.py +0 -0
  66. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_module_isolation.py +0 -0
  67. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_modules_no_committed_secrets.py +0 -0
  68. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_mvp_proof_workflow.py +0 -0
  69. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_preflight.py +0 -0
  70. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_python_version_alignment.py +0 -0
  71. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_release_version_check.py +0 -0
  72. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_render_example_profile.py +0 -0
  73. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_resolve_profile_path.py +0 -0
  74. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_resolver.py +0 -0
  75. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_secrets.py +0 -0
  76. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_smoke_example_profile.py +0 -0
  77. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_superset_hardening.py +0 -0
  78. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_trivyignore.py +0 -0
  79. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_up_runner.py +0 -0
  80. {composable_data_stack-0.6.1 → composable_data_stack-0.8.0}/tests/test_upstream_image_hardening.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: composable-data-stack
3
- Version: 0.6.1
3
+ Version: 0.8.0
4
4
  Summary: CLI for validating and composing composable data stack profiles
5
5
  Author: Ronald Hensbergen
6
6
  License-Expression: Apache-2.0
@@ -31,7 +31,7 @@ Provides-Extra: dev
31
31
  Requires-Dist: coverage>=7.6; extra == "dev"
32
32
  Requires-Dist: bandit>=1.7; extra == "dev"
33
33
  Requires-Dist: pip-audit>=2.7; extra == "dev"
34
- Requires-Dist: ruff==0.16.4; extra == "dev"
34
+ Requires-Dist: ruff==0.16.6; extra == "dev"
35
35
  Dynamic: license-file
36
36
 
37
37
  # 🚀 Composable Data Stack (CDS)
@@ -135,7 +135,6 @@ CDS wires modules through **contracts**, not direct dependencies. This section h
135
135
  Below, `local-dagster-postgres-superset` wires Dagster to Postgres to Superset through contracts:
136
136
 
137
137
  ```mermaid
138
- ---
139
138
  flowchart TD
140
139
  Dagster[Dagster]
141
140
  Postgres[(Postgres)]
@@ -174,7 +173,6 @@ consumption in Superset), see
174
173
  [docs/profile-testing/failure-path-and-ci.md](docs/profile-testing/failure-path-and-ci.md).
175
174
 
176
175
  ```mermaid
177
- ---
178
176
  flowchart TD
179
177
  subgraph compile["Compile-time (cds)"]
180
178
  direction TB
@@ -633,8 +631,79 @@ Differences from 'dev' to 'prod':
633
631
  ~ spec.modules[postgres].config.storage.size: "2Gi" -> "20Gi"
634
632
  ```
635
633
 
636
- Commands without `--environment` are unaffected they behave exactly as
637
- before and never look for an `environments/` directory.
634
+ Persist an environment default for a project with `cds config set environment
635
+ prod`; commands without `--environment` then use that overlay. An explicit
636
+ `--environment` always takes precedence.
637
+
638
+ ---
639
+
640
+ ### Profile Composition (`extends`)
641
+
642
+ A profile can also factor out shared configuration into one or more parent
643
+ profiles instead of duplicating it, using a top-level `extends` field:
644
+
645
+ ```yaml
646
+ # profiles/analytics-prod/profile.yaml
647
+ apiVersion: cds/v1alpha1
648
+ kind: Profile
649
+ metadata:
650
+ name: analytics-prod
651
+ environment: production
652
+ extends:
653
+ - analytics-base # a bare name resolves to profiles/analytics-base/profile.yaml
654
+ spec:
655
+ modules:
656
+ - id: postgres
657
+ config:
658
+ storage:
659
+ size: 20Gi
660
+ ```
661
+
662
+ `extends` accepts a non-empty list of parent references, each either a bare
663
+ profile name (resolved under the profiles root) or a path relative to the
664
+ child profile's directory (e.g. `../shared/profile.yaml`). Parents are
665
+ resolved and merged left-to-right — later parents win over earlier ones —
666
+ and then the child profile's own document is merged on top of all parents.
667
+ Composition uses the exact same deep-merge/module-merge-by-id engine as
668
+ environment overlays: mappings merge recursively, `spec.modules` entries
669
+ merge by stable `id`, and any other array is replaced wholesale rather than
670
+ concatenated. Parent profiles may themselves use `extends` (chains are
671
+ resolved transitively).
672
+
673
+ `extends` is not a CLI flag — it's read directly from `profile.yaml`, so
674
+ every profile-consuming command resolves it automatically, with no new
675
+ syntax to learn:
676
+
677
+ ```bash
678
+ cds validate analytics-prod
679
+ cds plan analytics-prod
680
+ cds up analytics-prod
681
+ ```
682
+
683
+ A profile can extend more than one parent, which is merged in the order
684
+ listed (later entries win over earlier ones):
685
+
686
+ ```yaml
687
+ extends:
688
+ - networking-base
689
+ - observability-base
690
+ ```
691
+
692
+ `extends` and `--environment` compose together: parents are merged first,
693
+ then the child, then the selected environment overlay is applied on top of
694
+ that fully-composed result — so a shared base profile and environment
695
+ promotion can both be used without duplicating configuration in either
696
+ dimension.
697
+
698
+ A malformed `extends` chain fails validation/planning before anything else
699
+ runs, with a dedicated diagnostic code:
700
+
701
+ |Code|Meaning|
702
+ |---|---|
703
+ |E110|`extends` is missing, not a list, empty, or contains a non-string/empty entry|
704
+ |E111|A parent reference resolves outside the profiles root|
705
+ |E112|A referenced parent profile does not exist|
706
+ |E113|A cycle was detected in the `extends` chain|
638
707
 
639
708
  ---
640
709
 
@@ -643,6 +712,7 @@ before and never look for an `environments/` directory.
643
712
  |Command|Description|
644
713
  |---|---|
645
714
  |cds get \<profile\> [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>] [--into \<dir\>]|Fetch a profile plus its dependent module/runtime assets from GitHub into a local CDS layout|
715
+ |cds list profiles\|modules\|images [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>]|List available profiles, module sources, or module images and check for newer versions; add `--remote`/`--local` to inspect another repository before fetching from it|
646
716
  |cds init [profile]|Generate a project `.env` template from profile secret definitions|
647
717
  |cds validate [profile]|Validate modules and contracts|
648
718
  |cds preflight [profile]|Check runtime tools, required environment values, and host ports without starting services|
@@ -654,6 +724,7 @@ before and never look for an `environments/` directory.
654
724
  |cds security [profile]|Run rule-based security validation on a profile|
655
725
  |cds diff [profile] --from \<env\> --to \<env\>|Show effective configuration differences between two environment overlays, secrets never included|
656
726
  |cds use [profile] [--clear]|Save (show/clear) a default profile so it doesn't have to be passed to other commands|
727
+ |cds config get\|set\|unset\|list|Manage persisted project defaults in `.cds/config.json`|
657
728
  |cds completion \<bash\|zsh\|powershell\>|Print shell setup instructions for tab-completion|
658
729
 
659
730
  `init`, `validate`, `preflight`, `plan`, `render`, `up`, `test`, and `security`
@@ -674,6 +745,35 @@ destination root, `--dry-run` to inspect the copy plan first, and `--force` to
674
745
  replace conflicting local files. Successful fetches record tracking metadata
675
746
  in `.cds/get-manifest.json` for future update workflows.
676
747
 
748
+ `cds list profiles`, `cds list modules`, and `cds list images` accept the same
749
+ `--remote`/`--ref`/`--local` source-repository selection as `cds get`, so you
750
+ can discover what's available in another repository (a fork, or an existing
751
+ local checkout) before running `cds get` against it. Without these flags,
752
+ `cds list` inspects the local project as before.
753
+
754
+ ### Project defaults
755
+
756
+ `cds config` manages the gitignored `.cds/config.json` file (or the path in
757
+ `CDS_CONFIG_PATH`). Supported settings are `profile`, `environment`, and
758
+ `security.strict`:
759
+
760
+ ```bash
761
+ cds config set profile my-profile
762
+ cds config set environment prod
763
+ cds config set security.strict true
764
+ cds config list
765
+ ```
766
+
767
+ `profile` is stored as its resolved path, and `environment` is validated
768
+ against that profile's `environments/` directory. `security.strict true`
769
+ applies the existing production security rules even if the profile declares a
770
+ local environment. `cds use` remains a shortcut for setting, showing, or
771
+ clearing `profile`.
772
+
773
+ CLI flags take precedence over these defaults. In particular,
774
+ `--environment` overrides `config environment`; `CDS_PROFILE_PATH` continues
775
+ to override the saved profile.
776
+
677
777
  `[profile]` accepts:
678
778
 
679
779
  | Form | Example |
@@ -682,7 +782,7 @@ in `.cds/get-manifest.json` for future update workflows.
682
782
  | Path to a `profile.yaml` file | `profiles/local-dagster-postgres-superset/profile.yaml` |
683
783
  | Path to a profiles root directory | `profiles/` |
684
784
 
685
- When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds use <profile>`, then the single profile under `profiles/` if there is exactly one. An explicitly-set env var takes precedence over the persisted `cds use` default, matching common CLI convention (env vars are per-invocation and reflect the current session more reliably than a saved, gitignored default that's easy to forget about).
785
+ When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds config set profile` (or `cds use <profile>`), then the single profile under `profiles/` if there is exactly one. An explicitly-set env var takes precedence over the persisted project default, matching common CLI convention (env vars are per-invocation and reflect the current session more reliably than a saved, gitignored default that's easy to forget about).
686
786
 
687
787
  To view the full list of options for any command, use the `--help` flag:
688
788
 
@@ -730,6 +830,8 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
730
830
  | `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
731
831
  | `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
732
832
  | `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
833
+ | `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
834
+ | `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
733
835
 
734
836
  All diagnostics print with their error code and YAML path (e.g. `spec.modules[1].config`), so search the profile file for that path to find the exact line to fix.
735
837
 
@@ -99,7 +99,6 @@ CDS wires modules through **contracts**, not direct dependencies. This section h
99
99
  Below, `local-dagster-postgres-superset` wires Dagster to Postgres to Superset through contracts:
100
100
 
101
101
  ```mermaid
102
- ---
103
102
  flowchart TD
104
103
  Dagster[Dagster]
105
104
  Postgres[(Postgres)]
@@ -138,7 +137,6 @@ consumption in Superset), see
138
137
  [docs/profile-testing/failure-path-and-ci.md](docs/profile-testing/failure-path-and-ci.md).
139
138
 
140
139
  ```mermaid
141
- ---
142
140
  flowchart TD
143
141
  subgraph compile["Compile-time (cds)"]
144
142
  direction TB
@@ -597,8 +595,79 @@ Differences from 'dev' to 'prod':
597
595
  ~ spec.modules[postgres].config.storage.size: "2Gi" -> "20Gi"
598
596
  ```
599
597
 
600
- Commands without `--environment` are unaffected they behave exactly as
601
- before and never look for an `environments/` directory.
598
+ Persist an environment default for a project with `cds config set environment
599
+ prod`; commands without `--environment` then use that overlay. An explicit
600
+ `--environment` always takes precedence.
601
+
602
+ ---
603
+
604
+ ### Profile Composition (`extends`)
605
+
606
+ A profile can also factor out shared configuration into one or more parent
607
+ profiles instead of duplicating it, using a top-level `extends` field:
608
+
609
+ ```yaml
610
+ # profiles/analytics-prod/profile.yaml
611
+ apiVersion: cds/v1alpha1
612
+ kind: Profile
613
+ metadata:
614
+ name: analytics-prod
615
+ environment: production
616
+ extends:
617
+ - analytics-base # a bare name resolves to profiles/analytics-base/profile.yaml
618
+ spec:
619
+ modules:
620
+ - id: postgres
621
+ config:
622
+ storage:
623
+ size: 20Gi
624
+ ```
625
+
626
+ `extends` accepts a non-empty list of parent references, each either a bare
627
+ profile name (resolved under the profiles root) or a path relative to the
628
+ child profile's directory (e.g. `../shared/profile.yaml`). Parents are
629
+ resolved and merged left-to-right — later parents win over earlier ones —
630
+ and then the child profile's own document is merged on top of all parents.
631
+ Composition uses the exact same deep-merge/module-merge-by-id engine as
632
+ environment overlays: mappings merge recursively, `spec.modules` entries
633
+ merge by stable `id`, and any other array is replaced wholesale rather than
634
+ concatenated. Parent profiles may themselves use `extends` (chains are
635
+ resolved transitively).
636
+
637
+ `extends` is not a CLI flag — it's read directly from `profile.yaml`, so
638
+ every profile-consuming command resolves it automatically, with no new
639
+ syntax to learn:
640
+
641
+ ```bash
642
+ cds validate analytics-prod
643
+ cds plan analytics-prod
644
+ cds up analytics-prod
645
+ ```
646
+
647
+ A profile can extend more than one parent, which is merged in the order
648
+ listed (later entries win over earlier ones):
649
+
650
+ ```yaml
651
+ extends:
652
+ - networking-base
653
+ - observability-base
654
+ ```
655
+
656
+ `extends` and `--environment` compose together: parents are merged first,
657
+ then the child, then the selected environment overlay is applied on top of
658
+ that fully-composed result — so a shared base profile and environment
659
+ promotion can both be used without duplicating configuration in either
660
+ dimension.
661
+
662
+ A malformed `extends` chain fails validation/planning before anything else
663
+ runs, with a dedicated diagnostic code:
664
+
665
+ |Code|Meaning|
666
+ |---|---|
667
+ |E110|`extends` is missing, not a list, empty, or contains a non-string/empty entry|
668
+ |E111|A parent reference resolves outside the profiles root|
669
+ |E112|A referenced parent profile does not exist|
670
+ |E113|A cycle was detected in the `extends` chain|
602
671
 
603
672
  ---
604
673
 
@@ -607,6 +676,7 @@ before and never look for an `environments/` directory.
607
676
  |Command|Description|
608
677
  |---|---|
609
678
  |cds get \<profile\> [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>] [--into \<dir\>]|Fetch a profile plus its dependent module/runtime assets from GitHub into a local CDS layout|
679
+ |cds list profiles\|modules\|images [--remote \<owner/repo\>] [--ref \<ref\>] [--local \<dir\>]|List available profiles, module sources, or module images and check for newer versions; add `--remote`/`--local` to inspect another repository before fetching from it|
610
680
  |cds init [profile]|Generate a project `.env` template from profile secret definitions|
611
681
  |cds validate [profile]|Validate modules and contracts|
612
682
  |cds preflight [profile]|Check runtime tools, required environment values, and host ports without starting services|
@@ -618,6 +688,7 @@ before and never look for an `environments/` directory.
618
688
  |cds security [profile]|Run rule-based security validation on a profile|
619
689
  |cds diff [profile] --from \<env\> --to \<env\>|Show effective configuration differences between two environment overlays, secrets never included|
620
690
  |cds use [profile] [--clear]|Save (show/clear) a default profile so it doesn't have to be passed to other commands|
691
+ |cds config get\|set\|unset\|list|Manage persisted project defaults in `.cds/config.json`|
621
692
  |cds completion \<bash\|zsh\|powershell\>|Print shell setup instructions for tab-completion|
622
693
 
623
694
  `init`, `validate`, `preflight`, `plan`, `render`, `up`, `test`, and `security`
@@ -638,6 +709,35 @@ destination root, `--dry-run` to inspect the copy plan first, and `--force` to
638
709
  replace conflicting local files. Successful fetches record tracking metadata
639
710
  in `.cds/get-manifest.json` for future update workflows.
640
711
 
712
+ `cds list profiles`, `cds list modules`, and `cds list images` accept the same
713
+ `--remote`/`--ref`/`--local` source-repository selection as `cds get`, so you
714
+ can discover what's available in another repository (a fork, or an existing
715
+ local checkout) before running `cds get` against it. Without these flags,
716
+ `cds list` inspects the local project as before.
717
+
718
+ ### Project defaults
719
+
720
+ `cds config` manages the gitignored `.cds/config.json` file (or the path in
721
+ `CDS_CONFIG_PATH`). Supported settings are `profile`, `environment`, and
722
+ `security.strict`:
723
+
724
+ ```bash
725
+ cds config set profile my-profile
726
+ cds config set environment prod
727
+ cds config set security.strict true
728
+ cds config list
729
+ ```
730
+
731
+ `profile` is stored as its resolved path, and `environment` is validated
732
+ against that profile's `environments/` directory. `security.strict true`
733
+ applies the existing production security rules even if the profile declares a
734
+ local environment. `cds use` remains a shortcut for setting, showing, or
735
+ clearing `profile`.
736
+
737
+ CLI flags take precedence over these defaults. In particular,
738
+ `--environment` overrides `config environment`; `CDS_PROFILE_PATH` continues
739
+ to override the saved profile.
740
+
641
741
  `[profile]` accepts:
642
742
 
643
743
  | Form | Example |
@@ -646,7 +746,7 @@ in `.cds/get-manifest.json` for future update workflows.
646
746
  | Path to a `profile.yaml` file | `profiles/local-dagster-postgres-superset/profile.yaml` |
647
747
  | Path to a profiles root directory | `profiles/` |
648
748
 
649
- When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds use <profile>`, then the single profile under `profiles/` if there is exactly one. An explicitly-set env var takes precedence over the persisted `cds use` default, matching common CLI convention (env vars are per-invocation and reflect the current session more reliably than a saved, gitignored default that's easy to forget about).
749
+ When `[profile]` is omitted, resolution falls back in order to: `CDS_PROFILE_PATH` if set (accepts the same three forms), then the default profile saved via `cds config set profile` (or `cds use <profile>`), then the single profile under `profiles/` if there is exactly one. An explicitly-set env var takes precedence over the persisted project default, matching common CLI convention (env vars are per-invocation and reflect the current session more reliably than a saved, gitignored default that's easy to forget about).
650
750
 
651
751
  To view the full list of options for any command, use the `--help` flag:
652
752
 
@@ -694,6 +794,8 @@ Common errors from `cds validate`, `cds plan`, and `cds render`, and how to fix
694
794
  | `[E041] ... Contract ref "x.y" points to unknown module "x"` | A `consumes` binding's `contractRef` refers to a module ID that isn't defined in the profile. | Check `spec.modules` for the correct module `id`, and confirm the contract ref follows `<module-id>.<contract-name>`. |
695
795
  | `[E041] ... but it does not provide "<contract-name>"` | The referenced module exists, but its `spec.provides` list doesn't expose that contract name. | Check the producing module's `module.yaml` for the contracts it actually provides, and fix the consumer's `contractRef` to match. |
696
796
  | `[E042] ... Contract kind mismatch` | The consumer expects one contract kind (e.g. `sql-database`) but the producer exposes a different kind. | Point the binding at a module that provides the expected contract kind, or update the consumer's expected kind if the mismatch is intentional. |
797
+ | `[E103] ... config.image.tag is required ... when config.image.source is "registry"` | A module instance sets `config.image.source: registry` without also setting `config.image.tag`. | Set `config.image.tag` to a version, or switch back to `config.image.source: build`. |
798
+ | `[W097] ... config.image.tag is "latest" with config.image.source "registry"` | `config.image.tag: latest` under `source: registry` still validates, but drifts silently between deploys instead of pinning a reproducible version. | Pin an explicit tag from `publish-images.yml`'s output (or `tests/fixtures/signed-images.json`) instead of `latest`. |
697
799
 
698
800
  All diagnostics print with their error code and YAML path (e.g. `spec.modules[1].config`), so search the profile file for that path to find the exact line to fix.
699
801
 
@@ -18,6 +18,7 @@ from urllib.error import HTTPError, URLError
18
18
  from urllib.request import Request, urlopen
19
19
 
20
20
  from .loader import load_yaml_file, resolve_module_dir
21
+ from .overlay import _derive_profiles_root, _resolve_extends_ref, resolve_extends
21
22
  from .planner import MaxNestingDepthExceeded, apply_defaults, substitute_string
22
23
 
23
24
  # The upstream repository `cds get` downloads from when no `--remote` is
@@ -71,7 +72,7 @@ def fetch_profile(
71
72
  destination_root: Path | None = None,
72
73
  force: bool = False,
73
74
  dry_run: bool = False,
74
- ) -> tuple[list[CopyAction], Path]:
75
+ ) -> tuple[list[CopyAction], Path, list[str]]:
75
76
  target_root = (destination_root or Path.cwd()).expanduser().resolve()
76
77
 
77
78
  with _prepare_source_repository(remote, ref, local) as source_repo:
@@ -79,10 +80,15 @@ def fetch_profile(
79
80
  asset_roots = _collect_asset_roots(source_repo, profile_path)
80
81
 
81
82
  actions = _build_copy_plan(source_repo, asset_roots, target_root)
83
+ # Conflict detection must happen while `source_repo` (which may be a
84
+ # temporary directory holding a downloaded tarball) is still alive,
85
+ # since `_find_conflicts()` reads each action's source file to
86
+ # compare content. Compute it up front so both the dry-run and
87
+ # real-run paths can report/enforce conflicts consistently (#452).
88
+ conflicts = _find_conflicts(actions)
82
89
  if dry_run:
83
- return actions, target_root / _TRACKING_FILE
90
+ return actions, target_root / _TRACKING_FILE, conflicts
84
91
 
85
- conflicts = _find_conflicts(actions)
86
92
  if conflicts and not force:
87
93
  rendered = ", ".join(conflicts[:5])
88
94
  extra = "" if len(conflicts) <= 5 else f" (+{len(conflicts) - 5} more)"
@@ -104,18 +110,37 @@ def fetch_profile(
104
110
  actions=actions,
105
111
  asset_roots=asset_roots,
106
112
  )
107
- return actions, target_root / _TRACKING_FILE
113
+ return actions, target_root / _TRACKING_FILE, conflicts
108
114
 
109
115
 
110
- def format_get_plan(actions: list[CopyAction], *, destination_root: Path) -> str:
116
+ def format_get_plan(
117
+ actions: list[CopyAction],
118
+ *,
119
+ destination_root: Path,
120
+ conflicts: list[str] | None = None,
121
+ ) -> str:
122
+ conflicts = conflicts or []
123
+ conflict_set = set(conflicts)
124
+ clean_actions = [a for a in actions if a.repo_relative_path not in conflict_set]
125
+
111
126
  if not actions:
112
127
  return f"No file changes required under {destination_root}."
113
128
 
114
129
  lines = [f"Planned {len(actions)} file(s) under {destination_root}:"]
115
- for action in actions[:20]:
116
- lines.append(f" - {action.repo_relative_path}")
117
- if len(actions) > 20:
118
- lines.append(f" - ... {len(actions) - 20} more")
130
+ if clean_actions:
131
+ lines.append(f" Would copy ({len(clean_actions)}):")
132
+ for action in clean_actions[:20]:
133
+ lines.append(f" - {action.repo_relative_path}")
134
+ if len(clean_actions) > 20:
135
+ lines.append(f" - ... {len(clean_actions) - 20} more")
136
+
137
+ if conflicts:
138
+ lines.append(f" Would conflict without --force ({len(conflicts)}):")
139
+ for path in conflicts[:20]:
140
+ lines.append(f" - {path}")
141
+ if len(conflicts) > 20:
142
+ lines.append(f" - ... {len(conflicts) - 20} more")
143
+
119
144
  return "\n".join(lines)
120
145
 
121
146
 
@@ -233,15 +258,67 @@ def _resolve_source_profile_path(source_repo: Path, profile: str) -> Path:
233
258
  )
234
259
 
235
260
 
261
+ def _collect_extends_profile_dirs(
262
+ source_repo: Path, profile_path: Path, _visited: frozenset[Path] = frozenset()
263
+ ) -> set[Path]:
264
+ """
265
+ Returns every parent profile directory reachable via `extends`, so
266
+ `cds get` copies parent profile.yaml files (and, via
267
+ _collect_asset_roots's module walk, their modules) too instead of
268
+ silently omitting anything only declared in a parent profile. Reuses
269
+ cli.overlay's own extends-ref resolution rather than re-implementing it.
270
+
271
+ `_visited` tracks resolved profile paths already seen along the current
272
+ extends chain so a cyclic `extends` graph raises a clean GetError
273
+ instead of recursing until Python's recursion limit is hit; this
274
+ mirrors cli.overlay._compose_extends's own cycle-detection stack.
275
+ """
276
+ resolved_profile_path = profile_path.resolve()
277
+ if resolved_profile_path in _visited:
278
+ raise GetError(f"Cycle detected in extends chain at {profile_path}")
279
+ _visited = _visited | {resolved_profile_path}
280
+
281
+ doc, _diagnostics = load_yaml_file(profile_path)
282
+ if doc is None:
283
+ raise GetError(f"Could not load source profile {profile_path}")
284
+
285
+ extends = doc.get("extends")
286
+ if not extends:
287
+ return set()
288
+
289
+ profile_dir = profile_path.parent.resolve()
290
+ profiles_root = _derive_profiles_root(profile_dir)
291
+ if profiles_root is None:
292
+ raise GetError(f'Could not derive a profiles root for extends in {profile_path}')
293
+
294
+ dirs: set[Path] = set()
295
+ for ref in extends:
296
+ parent_path = _resolve_extends_ref(ref, profile_dir, profiles_root)
297
+ if not parent_path.is_file():
298
+ raise GetError(f'Could not resolve extends parent "{ref}" for {profile_path}')
299
+ _require_within_repo(parent_path, source_repo, f'extends parent "{ref}"')
300
+ dirs.add(parent_path.parent)
301
+ dirs.update(_collect_extends_profile_dirs(source_repo, parent_path, _visited))
302
+
303
+ return dirs
304
+
305
+
236
306
  def _collect_asset_roots(source_repo: Path, profile_path: Path) -> list[Path]:
237
307
  profile_dir = profile_path.parent
238
308
  asset_roots: set[Path] = {
239
309
  profile_dir if profile_path.name == "profile.yaml" else profile_path
240
310
  }
241
- profile_doc, profile_diags = load_yaml_file(profile_path)
242
- if profile_diags or profile_doc is None:
311
+
312
+ # Resolve through `extends` so modules/config contributed only by a
313
+ # parent profile (not redeclared in the child) are still collected;
314
+ # cli.overlay is the single source of truth for extends semantics (see
315
+ # cli.planner/cli.validator, which route through it the same way).
316
+ profile_doc, _provenance, diagnostics = resolve_extends(str(profile_path))
317
+ if profile_doc is None or any(d.level == "error" for d in diagnostics):
243
318
  raise GetError(f"Could not load source profile {profile_path}")
244
319
 
320
+ asset_roots.update(_collect_extends_profile_dirs(source_repo, profile_path))
321
+
245
322
  spec = profile_doc.get("spec")
246
323
  modules = spec.get("modules", []) if isinstance(spec, dict) else []
247
324
  if not isinstance(modules, list):
@@ -595,7 +672,19 @@ def _add_copy_action(
595
672
  destination_root: Path,
596
673
  actions_by_destination: dict[Path, CopyAction],
597
674
  ) -> None:
598
- repo_relative = source_file.resolve().relative_to(source_repo.resolve())
675
+ resolved_source = source_file.resolve()
676
+ try:
677
+ repo_relative = resolved_source.relative_to(source_repo.resolve())
678
+ except ValueError as exc:
679
+ # A symlink (or directory junction) planted under an asset root can
680
+ # resolve to a real path outside the source repository. Without this
681
+ # guard, Path.relative_to() raises a bare ValueError that escapes as
682
+ # an unhandled traceback instead of the stable GetError callers rely
683
+ # on (#454).
684
+ raise GetError(
685
+ f"Asset {source_file} resolves outside the source repository "
686
+ f"({source_repo}): {resolved_source}"
687
+ ) from exc
599
688
  # Deliberately do NOT call .resolve() on the combined destination path:
600
689
  # destination_root is already an absolute, resolved path (see
601
690
  # fetch_profile()), and resolving the full path here would follow a
@@ -607,12 +696,12 @@ def _add_copy_action(
607
696
  existing = actions_by_destination.get(destination)
608
697
  if existing is None:
609
698
  actions_by_destination[destination] = CopyAction(
610
- source=source_file.resolve(),
699
+ source=resolved_source,
611
700
  destination=destination,
612
701
  repo_relative_path=repo_relative.as_posix(),
613
702
  )
614
703
  return
615
- if existing.source.resolve() != source_file.resolve():
704
+ if existing.source.resolve() != resolved_source:
616
705
  raise GetError(
617
706
  f"Multiple source files would map to the same destination: {destination}"
618
707
  )
@@ -649,10 +738,17 @@ def _write_actions(actions: list[CopyAction]) -> None:
649
738
  (e.g. a symlink planted at profiles/foo/profile.yaml pointing outside
650
739
  the destination tree)."""
651
740
  for action in actions:
652
- action.destination.parent.mkdir(parents=True, exist_ok=True)
653
- if action.destination.is_symlink():
654
- action.destination.unlink()
655
- shutil.copy2(action.source, action.destination)
741
+ try:
742
+ action.destination.parent.mkdir(parents=True, exist_ok=True)
743
+ if action.destination.is_symlink():
744
+ action.destination.unlink()
745
+ shutil.copy2(action.source, action.destination)
746
+ except OSError as exc:
747
+ # Surface filesystem failures (permission denied, read-only
748
+ # destination, disk full, ...) as a stable GetError instead of a
749
+ # raw OSError, so callers get a predictable error path and
750
+ # message regardless of the underlying platform/errno (#454).
751
+ raise GetError(f"Could not write {action.destination}: {exc}") from exc
656
752
 
657
753
 
658
754
  def _write_tracking_manifest(