spark-kindling-cli 0.14.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. spark_kindling_cli-0.14.0/PKG-INFO +226 -0
  2. spark_kindling_cli-0.14.0/README.md +194 -0
  3. spark_kindling_cli-0.14.0/kindling_cli/__init__.py +8 -0
  4. spark_kindling_cli-0.14.0/kindling_cli/_runner.py +334 -0
  5. spark_kindling_cli-0.14.0/kindling_cli/_tracing.py +104 -0
  6. spark_kindling_cli-0.14.0/kindling_cli/agent_files.py +306 -0
  7. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/SKILL.md +83 -0
  8. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/project_instructions.md +15 -0
  9. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/apps.md +311 -0
  10. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/config.md +291 -0
  11. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/databricks.md +345 -0
  12. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/entities.md +310 -0
  13. spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/pipes.md +420 -0
  14. spark_kindling_cli-0.14.0/kindling_cli/bundle.py +1344 -0
  15. spark_kindling_cli-0.14.0/kindling_cli/cli.py +9927 -0
  16. spark_kindling_cli-0.14.0/kindling_cli/scaffold.py +346 -0
  17. spark_kindling_cli-0.14.0/kindling_cli/templates/.devcontainer/devcontainer.json.j2 +59 -0
  18. spark_kindling_cli-0.14.0/kindling_cli/templates/.env.example.j2 +40 -0
  19. spark_kindling_cli-0.14.0/kindling_cli/templates/.github/workflows/ci.yml.j2 +46 -0
  20. spark_kindling_cli-0.14.0/kindling_cli/templates/.gitignore.j2 +10 -0
  21. spark_kindling_cli-0.14.0/kindling_cli/templates/app.batch.py.j2 +13 -0
  22. spark_kindling_cli-0.14.0/kindling_cli/templates/app.file-ingestion.py.j2 +14 -0
  23. spark_kindling_cli-0.14.0/kindling_cli/templates/app.py.j2 +13 -0
  24. spark_kindling_cli-0.14.0/kindling_cli/templates/app.streaming.py.j2 +13 -0
  25. spark_kindling_cli-0.14.0/kindling_cli/templates/app.yaml.j2 +2 -0
  26. spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/README.md +23 -0
  27. spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/databricks.yml.j2 +18 -0
  28. spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/resources/__pipeline__.pipeline.yml.j2 +21 -0
  29. spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/src/kindling_lakeflow.py +9 -0
  30. spark_kindling_cli-0.14.0/kindling_cli/templates/lake-reqs.txt.j2 +5 -0
  31. spark_kindling_cli-0.14.0/kindling_cli/templates/package/PACKAGE_QUICKSTART.md.j2 +39 -0
  32. spark_kindling_cli-0.14.0/kindling_cli/templates/package/QUICKSTART.md.j2 +94 -0
  33. spark_kindling_cli-0.14.0/kindling_cli/templates/pyproject.root.toml.j2 +26 -0
  34. spark_kindling_cli-0.14.0/kindling_cli/templates/pyproject.toml.j2 +81 -0
  35. spark_kindling_cli-0.14.0/kindling_cli/templates/scripts/setup-local-dev.sh.j2 +34 -0
  36. spark_kindling_cli-0.14.0/kindling_cli/templates/settings.local.yaml.j2 +43 -0
  37. spark_kindling_cli-0.14.0/kindling_cli/templates/settings.yaml.j2 +14 -0
  38. spark_kindling_cli-0.14.0/kindling_cli/templates/src/entities/records.medallion.py.j2 +56 -0
  39. spark_kindling_cli-0.14.0/kindling_cli/templates/src/entities/records.minimal.py.j2 +31 -0
  40. spark_kindling_cli-0.14.0/kindling_cli/templates/src/pipes/bronze_to_silver.py.j2 +24 -0
  41. spark_kindling_cli-0.14.0/kindling_cli/templates/src/pipes/process.py.j2 +17 -0
  42. spark_kindling_cli-0.14.0/kindling_cli/templates/src/transforms/quality.medallion.py.j2 +26 -0
  43. spark_kindling_cli-0.14.0/kindling_cli/templates/src/transforms/quality.minimal.py.j2 +12 -0
  44. spark_kindling_cli-0.14.0/kindling_cli/templates/tests/component/test_registration.py.j2 +96 -0
  45. spark_kindling_cli-0.14.0/kindling_cli/templates/tests/conftest.py.j2 +325 -0
  46. spark_kindling_cli-0.14.0/kindling_cli/templates/tests/integration/test_pipeline.py.j2 +8 -0
  47. spark_kindling_cli-0.14.0/kindling_cli/templates/tests/integration/test_pipeline_azure.py.j2 +78 -0
  48. spark_kindling_cli-0.14.0/kindling_cli/templates/tests/integration/test_pipeline_local.py.j2 +45 -0
  49. spark_kindling_cli-0.14.0/kindling_cli/templates/tests/unit/test_transforms.py.j2 +60 -0
  50. spark_kindling_cli-0.14.0/kindling_cli/test_runner.py +451 -0
  51. spark_kindling_cli-0.14.0/pyproject.toml +54 -0
  52. spark_kindling_cli-0.14.0/pyproject.toml.orig +44 -0
@@ -0,0 +1,226 @@
1
+ Metadata-Version: 2.4
2
+ Name: spark-kindling-cli
3
+ Version: 0.14.0
4
+ Summary: Command-line tooling for Kindling
5
+ Author: SEP Engineering
6
+ Author-email: SEP Engineering <engineering@sep.com>
7
+ License-Expression: MIT
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Operating System :: OS Independent
11
+ Classifier: Programming Language :: Python :: 3
12
+ Classifier: Programming Language :: Python :: 3 :: Only
13
+ Classifier: Topic :: Database
14
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
15
+ Requires-Dist: click>=8.1.0
16
+ Requires-Dist: jinja2>=3.1.0
17
+ Requires-Dist: pyyaml>=6.0
18
+ Requires-Dist: spark-kindling-sdk>=0.9.10
19
+ Requires-Dist: requests>=2.31.0 ; extra == 'deploy'
20
+ Requires-Dist: azure-identity>=1.15.0 ; extra == 'deploy'
21
+ Requires-Dist: azure-storage-blob>=12.14.0 ; extra == 'deploy'
22
+ Requires-Dist: azure-identity>=1.15.0 ; extra == 'test-cleanup'
23
+ Requires-Dist: azure-storage-file-datalake>=12.14.0 ; extra == 'test-cleanup'
24
+ Requires-Python: >=3.10
25
+ Project-URL: Homepage, https://github.com/sep/spark-kindling-framework
26
+ Project-URL: Documentation, https://github.com/sep/spark-kindling-framework/tree/main/docs
27
+ Project-URL: Changelog, https://github.com/sep/spark-kindling-framework/blob/main/CHANGELOG.md
28
+ Project-URL: Issues, https://github.com/sep/spark-kindling-framework/issues
29
+ Provides-Extra: deploy
30
+ Provides-Extra: test-cleanup
31
+ Description-Content-Type: text/markdown
32
+
33
+ # spark-kindling-cli
34
+
35
+ Command-line tooling for the Spark Kindling Framework. Distributed as
36
+ `spark-kindling-cli`; installs the `kindling` console script.
37
+
38
+ ## Install
39
+
40
+ ```bash
41
+ pip install spark-kindling-cli
42
+ ```
43
+
44
+ The CLI depends on `spark-kindling-sdk` (used for remote platform lifecycle
45
+ operations), so this installs the SDK too. The CLI only requires a minimum SDK
46
+ version, so to pin one Kindling release, pin both:
47
+ `pip install spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`.
48
+
49
+ Without PyPI access (or for a release before 0.14.0, which exists only on
50
+ GitHub), install both from a
51
+ [GitHub Release](https://github.com/sep/spark-kindling-framework/releases)'s
52
+ wheel URLs:
53
+
54
+ ```bash
55
+ V=0.14.0 # any release, without the leading v
56
+ BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
57
+ pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" \
58
+ "$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
59
+ ```
60
+
61
+ Inside the Kindling devcontainer, `kindling` is a shim that runs the project's
62
+ own `./.venv/bin/kindling` (pinned in `pyproject.toml`) when present, and
63
+ otherwise installs `spark-kindling-cli` from PyPI (falling back to the latest
64
+ GitHub release's wheels).
65
+
66
+ ## Commands
67
+
68
+ - `kindling repo init` — scaffold a multi-package repo root (uv workspace root `pyproject.toml`, devcontainer, CI) with shared dev tooling
69
+ - `kindling package init` — scaffold a package (a uv workspace member) under `packages/<name>` in an existing repo
70
+ - `kindling app init` — scaffold an app under `apps/<name>` that runs a domain package
71
+ - `kindling config init` — generate a starter `settings.yaml`
72
+ - `kindling config set <key> <value>` — set a config value using dot-notation
73
+ - `kindling env check` — validate the local Python/config environment
74
+ - `kindling env bootstrap` — pin Kindling if nothing declares it yet and `uv sync --all-packages` (the devcontainer's `postCreateCommand`)
75
+ - `kindling env update` / `kindling env add` — move or add Kindling pins to a release (PyPI version pins; release wheel URLs for packages not on PyPI; `--source` forces either)
76
+ - `kindling env ensure` — download Hadoop/ABFSS JARs into `/tmp/hadoop-jars/`
77
+ - `kindling workspace check` — validate the configured platform workspace
78
+ - `kindling workspace init` — scaffold bootstrap + starter notebook files for a platform
79
+ - `kindling workspace deploy` — upload the runtime wheel, bootstrap script, config, and notebooks to artifact storage
80
+ - `kindling app package` — build a `.kda` archive from a local app directory
81
+ - `kindling app deploy` — deploy an app directory or `.kda` package through the SDK
82
+ - `kindling app run` — run all local app pipes on `standalone`, or submit to the durable Kindling runner on a managed platform
83
+ - `kindling app status` — fetch the current remote app run status
84
+ - `kindling app logs` — fetch or stream remote app run logs
85
+ - `kindling app cancel` — cancel an active remote app run
86
+ - `kindling app cleanup` — remove a deployed app from remote storage
87
+ - `kindling runner ensure` — install or verify the durable Kindling runner on a platform
88
+ - `kindling runner status` — check whether the runner is installed and healthy
89
+ - `kindling runner repair` — reinstall the runner (delete and recreate)
90
+ - `kindling runner delete` — remove the runner from a platform
91
+ - `kindling bundle build` — precompile a project into a Databricks bundle (`databricks.yml`, pipeline resources, generic Lakeflow source, manifest) for `databricks bundle validate/deploy/run`
92
+
93
+ Run any command with `--help` for full options.
94
+
95
+ ## Databricks bundles
96
+
97
+ `kindling bundle build` assembles a disposable Databricks bundle from a
98
+ project's `config/` overlays and `data-apps/` (or `apps/`) directories. It
99
+ starts no Spark session and imports no app code. Deployment inputs come from
100
+ options or `KINDLING_BUNDLE_*` environment variables (options win; runtime
101
+ settings files are never consulted):
102
+
103
+ ```bash
104
+ kindling bundle build \
105
+ --name sales --target dev --app orders \
106
+ --workspace-host https://adb-123.azuredatabricks.net \
107
+ --catalog dev_sales --schema orders \
108
+ --wheel dist/spark_kindling-0.12.49-py3-none-any.whl \
109
+ --wheel dist/spark_kindling_ext_sdp-0.3.4-py3-none-any.whl \
110
+ --wheel dist/spark_kindling_ext_databricks-0.2.0-py3-none-any.whl \
111
+ --wheel dist/orders_kindling_app-1.4.0-py3-none-any.whl
112
+
113
+ cd dist/bundles/databricks
114
+ databricks bundle validate -t dev
115
+ databricks bundle deploy -t dev
116
+ databricks bundle run -t dev orders
117
+ ```
118
+
119
+ The bundle is rendered from a **template**: ordinary Databricks bundle YAML
120
+ with Jinja placeholders. The built-in template produces `databricks.yml`, one
121
+ `resources/<key>.pipeline.yml` per pipeline and the generic Lakeflow source,
122
+ driven by `--app-options-json` (per-app catalog/schema/continuous/pipes, or a
123
+ `pipelines` map to split an app). When a project needs its own resource keys,
124
+ display names, tags, per-pipeline permissions, clusters or anything else DAB
125
+ supports, copy the template and own it:
126
+
127
+ ```bash
128
+ kindling bundle template init # -> bundle-template/
129
+ kindling bundle build --template-dir bundle-template ...
130
+ ```
131
+
132
+ Inside a template the generator supplies only what it uniquely knows. Settings
133
+ are found by convention, never listed: the shared `config/` overlays
134
+ (`settings.yaml`, `settings.databricks.yaml`, `workspace_<id>.yaml`,
135
+ `settings.<env>.yaml`) and then each app's own `settings*.yaml` are merged at
136
+ build time in the runtime's order and exposed as `apps[<name>].settings`.
137
+ `kindling.configuration(<app>, pipes=[...], extra={...})` returns a pipeline's
138
+ complete `configuration` map: the app selection, the inline
139
+ `kindling.lakeflow.settings_json`, the pipe subset, any extra flat overrides,
140
+ and a `kindling.lakeflow.config_keys` naming every emitted key a restricted
141
+ runtime must point-look-up. Rendered resources are parsed back and a pipeline
142
+ whose configuration did not come from the helper, or whose `config_keys` is
143
+ incomplete, fails the build. The deployed pipeline reads no settings files and
144
+ the resource file is the complete, reviewable description of what it runs
145
+ with. Files ending in `.j2` are rendered (`__pipeline__` in a name renders
146
+ once per pipeline); other files are copied verbatim. `manifest.json` records
147
+ the generator version, inputs, the SHA-256 of every settings file, wheel and
148
+ template file; no timestamps, so identical inputs give identical output.
149
+ `--app` may be omitted to build every app under `data-apps/` or `apps/`.
150
+ See `docs/proposals/databricks_bundle_deployment.md`.
151
+
152
+ Wheels passed with `--wheel` are uploaded by `databricks bundle deploy` under
153
+ the bundle's own workspace root and become the pipeline environment's
154
+ dependencies in the order given. Serverless installs them one at a time and
155
+ a wheel's missing dependency would otherwise be fetched from PyPI (or fail
156
+ where PyPI is unreachable) instead of using the uploaded wheel, so list
157
+ dependencies first: framework core, then `spark-kindling-ext-sdp`, then
158
+ `spark-kindling-ext-databricks`, then the app wheel.
159
+
160
+ The default `--workspace-root` is `/Workspace/Shared/kindling/<name>/<target>`:
161
+ a stable path that survives redeploys by different principals, which is
162
+ what shared dev/prod targets need. `databricks bundle validate` warns that
163
+ `/Workspace/Shared` is writable by all workspace users; for a personal
164
+ development target pass `--workspace-root /Workspace/Users/<you>/...`, and
165
+ for shared targets either accept the warning or grant the intended group
166
+ through `--permissions-json`.
167
+
168
+ ## Scaffolding
169
+
170
+ The scaffolding commands now target true multi-package repos:
171
+
172
+ ```bash
173
+ git clone <your-empty-repo-url> data-platform
174
+ cd data-platform
175
+ kindling repo init data-platform
176
+
177
+ kindling package init sales-ops --repo-root .
178
+ ```
179
+
180
+ This produces a repo root with shared `.devcontainer/`, CI,
181
+ `scripts/setup-local-dev.sh` and `.gitignore` plus independently buildable
182
+ packages under `packages/`. The root `pyproject.toml` is a non-package uv
183
+ workspace root (`[tool.uv] package = false`,
184
+ `[tool.uv.workspace] members = ["packages/*"]`), so the repo shares one
185
+ `.venv/` and `uv.lock` with every package installed editable. Run
186
+ `kindling env bootstrap` at the root (the devcontainer does this on create) to
187
+ pin Kindling there and sync. Every package must pin the same Kindling release
188
+ as the root; `kindling env update` moves them together.
189
+ If `.devcontainer/` already exists, `kindling repo init` warns and leaves it
190
+ unchanged. Pass `--overwrite-devcontainer` to replace the generated
191
+ devcontainer config. An existing root `pyproject.toml` is also kept; add
192
+ `[tool.uv.workspace] members = ["packages/*"]` to it.
193
+
194
+ To add a second package later, return to the repo root and scaffold another
195
+ package:
196
+
197
+ ```bash
198
+ kindling package init customer-360 --repo-root .
199
+
200
+ cd packages/customer_360
201
+ uv run poe test
202
+ ```
203
+
204
+ `uv run` syncs what the package needs before running. Don't run a bare `uv sync` inside a package directory: in the workspace it is an exact sync of that one package and removes the others from the shared `.venv/`. Resync the whole repo with `uv sync --all-packages` (or `kindling env bootstrap`) at the root.
205
+
206
+ The generated CI workflow runs in the devcontainer image and tests each
207
+ scaffolded package independently by iterating `packages/*` and executing:
208
+
209
+ ```bash
210
+ uv run poe test
211
+ uv run poe build
212
+ ```
213
+
214
+ The CI job fails if no package `pyproject.toml` files are found under
215
+ `packages/`.
216
+
217
+ Apps are scaffolded separately from packages:
218
+
219
+ ```bash
220
+ kindling app init sales-ops --package sales-ops --repo-root .
221
+ ```
222
+
223
+ ## Related
224
+
225
+ - Runtime framework: `pip install 'spark-kindling[<platform>]'` where `<platform>` is one of `synapse`, `databricks`, `fabric`, `standalone`.
226
+ - Design-time SDK: `pip install spark-kindling-sdk`.
@@ -0,0 +1,194 @@
1
+ # spark-kindling-cli
2
+
3
+ Command-line tooling for the Spark Kindling Framework. Distributed as
4
+ `spark-kindling-cli`; installs the `kindling` console script.
5
+
6
+ ## Install
7
+
8
+ ```bash
9
+ pip install spark-kindling-cli
10
+ ```
11
+
12
+ The CLI depends on `spark-kindling-sdk` (used for remote platform lifecycle
13
+ operations), so this installs the SDK too. The CLI only requires a minimum SDK
14
+ version, so to pin one Kindling release, pin both:
15
+ `pip install spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`.
16
+
17
+ Without PyPI access (or for a release before 0.14.0, which exists only on
18
+ GitHub), install both from a
19
+ [GitHub Release](https://github.com/sep/spark-kindling-framework/releases)'s
20
+ wheel URLs:
21
+
22
+ ```bash
23
+ V=0.14.0 # any release, without the leading v
24
+ BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
25
+ pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" \
26
+ "$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
27
+ ```
28
+
29
+ Inside the Kindling devcontainer, `kindling` is a shim that runs the project's
30
+ own `./.venv/bin/kindling` (pinned in `pyproject.toml`) when present, and
31
+ otherwise installs `spark-kindling-cli` from PyPI (falling back to the latest
32
+ GitHub release's wheels).
33
+
34
+ ## Commands
35
+
36
+ - `kindling repo init` — scaffold a multi-package repo root (uv workspace root `pyproject.toml`, devcontainer, CI) with shared dev tooling
37
+ - `kindling package init` — scaffold a package (a uv workspace member) under `packages/<name>` in an existing repo
38
+ - `kindling app init` — scaffold an app under `apps/<name>` that runs a domain package
39
+ - `kindling config init` — generate a starter `settings.yaml`
40
+ - `kindling config set <key> <value>` — set a config value using dot-notation
41
+ - `kindling env check` — validate the local Python/config environment
42
+ - `kindling env bootstrap` — pin Kindling if nothing declares it yet and `uv sync --all-packages` (the devcontainer's `postCreateCommand`)
43
+ - `kindling env update` / `kindling env add` — move or add Kindling pins to a release (PyPI version pins; release wheel URLs for packages not on PyPI; `--source` forces either)
44
+ - `kindling env ensure` — download Hadoop/ABFSS JARs into `/tmp/hadoop-jars/`
45
+ - `kindling workspace check` — validate the configured platform workspace
46
+ - `kindling workspace init` — scaffold bootstrap + starter notebook files for a platform
47
+ - `kindling workspace deploy` — upload the runtime wheel, bootstrap script, config, and notebooks to artifact storage
48
+ - `kindling app package` — build a `.kda` archive from a local app directory
49
+ - `kindling app deploy` — deploy an app directory or `.kda` package through the SDK
50
+ - `kindling app run` — run all local app pipes on `standalone`, or submit to the durable Kindling runner on a managed platform
51
+ - `kindling app status` — fetch the current remote app run status
52
+ - `kindling app logs` — fetch or stream remote app run logs
53
+ - `kindling app cancel` — cancel an active remote app run
54
+ - `kindling app cleanup` — remove a deployed app from remote storage
55
+ - `kindling runner ensure` — install or verify the durable Kindling runner on a platform
56
+ - `kindling runner status` — check whether the runner is installed and healthy
57
+ - `kindling runner repair` — reinstall the runner (delete and recreate)
58
+ - `kindling runner delete` — remove the runner from a platform
59
+ - `kindling bundle build` — precompile a project into a Databricks bundle (`databricks.yml`, pipeline resources, generic Lakeflow source, manifest) for `databricks bundle validate/deploy/run`
60
+
61
+ Run any command with `--help` for full options.
62
+
63
+ ## Databricks bundles
64
+
65
+ `kindling bundle build` assembles a disposable Databricks bundle from a
66
+ project's `config/` overlays and `data-apps/` (or `apps/`) directories. It
67
+ starts no Spark session and imports no app code. Deployment inputs come from
68
+ options or `KINDLING_BUNDLE_*` environment variables (options win; runtime
69
+ settings files are never consulted):
70
+
71
+ ```bash
72
+ kindling bundle build \
73
+ --name sales --target dev --app orders \
74
+ --workspace-host https://adb-123.azuredatabricks.net \
75
+ --catalog dev_sales --schema orders \
76
+ --wheel dist/spark_kindling-0.12.49-py3-none-any.whl \
77
+ --wheel dist/spark_kindling_ext_sdp-0.3.4-py3-none-any.whl \
78
+ --wheel dist/spark_kindling_ext_databricks-0.2.0-py3-none-any.whl \
79
+ --wheel dist/orders_kindling_app-1.4.0-py3-none-any.whl
80
+
81
+ cd dist/bundles/databricks
82
+ databricks bundle validate -t dev
83
+ databricks bundle deploy -t dev
84
+ databricks bundle run -t dev orders
85
+ ```
86
+
87
+ The bundle is rendered from a **template**: ordinary Databricks bundle YAML
88
+ with Jinja placeholders. The built-in template produces `databricks.yml`, one
89
+ `resources/<key>.pipeline.yml` per pipeline and the generic Lakeflow source,
90
+ driven by `--app-options-json` (per-app catalog/schema/continuous/pipes, or a
91
+ `pipelines` map to split an app). When a project needs its own resource keys,
92
+ display names, tags, per-pipeline permissions, clusters or anything else DAB
93
+ supports, copy the template and own it:
94
+
95
+ ```bash
96
+ kindling bundle template init # -> bundle-template/
97
+ kindling bundle build --template-dir bundle-template ...
98
+ ```
99
+
100
+ Inside a template the generator supplies only what it uniquely knows. Settings
101
+ are found by convention, never listed: the shared `config/` overlays
102
+ (`settings.yaml`, `settings.databricks.yaml`, `workspace_<id>.yaml`,
103
+ `settings.<env>.yaml`) and then each app's own `settings*.yaml` are merged at
104
+ build time in the runtime's order and exposed as `apps[<name>].settings`.
105
+ `kindling.configuration(<app>, pipes=[...], extra={...})` returns a pipeline's
106
+ complete `configuration` map: the app selection, the inline
107
+ `kindling.lakeflow.settings_json`, the pipe subset, any extra flat overrides,
108
+ and a `kindling.lakeflow.config_keys` naming every emitted key a restricted
109
+ runtime must point-look-up. Rendered resources are parsed back and a pipeline
110
+ whose configuration did not come from the helper, or whose `config_keys` is
111
+ incomplete, fails the build. The deployed pipeline reads no settings files and
112
+ the resource file is the complete, reviewable description of what it runs
113
+ with. Files ending in `.j2` are rendered (`__pipeline__` in a name renders
114
+ once per pipeline); other files are copied verbatim. `manifest.json` records
115
+ the generator version, inputs, the SHA-256 of every settings file, wheel and
116
+ template file; no timestamps, so identical inputs give identical output.
117
+ `--app` may be omitted to build every app under `data-apps/` or `apps/`.
118
+ See `docs/proposals/databricks_bundle_deployment.md`.
119
+
120
+ Wheels passed with `--wheel` are uploaded by `databricks bundle deploy` under
121
+ the bundle's own workspace root and become the pipeline environment's
122
+ dependencies in the order given. Serverless installs them one at a time and
123
+ a wheel's missing dependency would otherwise be fetched from PyPI (or fail
124
+ where PyPI is unreachable) instead of using the uploaded wheel, so list
125
+ dependencies first: framework core, then `spark-kindling-ext-sdp`, then
126
+ `spark-kindling-ext-databricks`, then the app wheel.
127
+
128
+ The default `--workspace-root` is `/Workspace/Shared/kindling/<name>/<target>`:
129
+ a stable path that survives redeploys by different principals, which is
130
+ what shared dev/prod targets need. `databricks bundle validate` warns that
131
+ `/Workspace/Shared` is writable by all workspace users; for a personal
132
+ development target pass `--workspace-root /Workspace/Users/<you>/...`, and
133
+ for shared targets either accept the warning or grant the intended group
134
+ through `--permissions-json`.
135
+
136
+ ## Scaffolding
137
+
138
+ The scaffolding commands now target true multi-package repos:
139
+
140
+ ```bash
141
+ git clone <your-empty-repo-url> data-platform
142
+ cd data-platform
143
+ kindling repo init data-platform
144
+
145
+ kindling package init sales-ops --repo-root .
146
+ ```
147
+
148
+ This produces a repo root with shared `.devcontainer/`, CI,
149
+ `scripts/setup-local-dev.sh` and `.gitignore` plus independently buildable
150
+ packages under `packages/`. The root `pyproject.toml` is a non-package uv
151
+ workspace root (`[tool.uv] package = false`,
152
+ `[tool.uv.workspace] members = ["packages/*"]`), so the repo shares one
153
+ `.venv/` and `uv.lock` with every package installed editable. Run
154
+ `kindling env bootstrap` at the root (the devcontainer does this on create) to
155
+ pin Kindling there and sync. Every package must pin the same Kindling release
156
+ as the root; `kindling env update` moves them together.
157
+ If `.devcontainer/` already exists, `kindling repo init` warns and leaves it
158
+ unchanged. Pass `--overwrite-devcontainer` to replace the generated
159
+ devcontainer config. An existing root `pyproject.toml` is also kept; add
160
+ `[tool.uv.workspace] members = ["packages/*"]` to it.
161
+
162
+ To add a second package later, return to the repo root and scaffold another
163
+ package:
164
+
165
+ ```bash
166
+ kindling package init customer-360 --repo-root .
167
+
168
+ cd packages/customer_360
169
+ uv run poe test
170
+ ```
171
+
172
+ `uv run` syncs what the package needs before running. Don't run a bare `uv sync` inside a package directory: in the workspace it is an exact sync of that one package and removes the others from the shared `.venv/`. Resync the whole repo with `uv sync --all-packages` (or `kindling env bootstrap`) at the root.
173
+
174
+ The generated CI workflow runs in the devcontainer image and tests each
175
+ scaffolded package independently by iterating `packages/*` and executing:
176
+
177
+ ```bash
178
+ uv run poe test
179
+ uv run poe build
180
+ ```
181
+
182
+ The CI job fails if no package `pyproject.toml` files are found under
183
+ `packages/`.
184
+
185
+ Apps are scaffolded separately from packages:
186
+
187
+ ```bash
188
+ kindling app init sales-ops --package sales-ops --repo-root .
189
+ ```
190
+
191
+ ## Related
192
+
193
+ - Runtime framework: `pip install 'spark-kindling[<platform>]'` where `<platform>` is one of `synapse`, `databricks`, `fabric`, `standalone`.
194
+ - Design-time SDK: `pip install spark-kindling-sdk`.
@@ -0,0 +1,8 @@
1
+ """Command-line tooling for Kindling.
2
+
3
+ INVARIANT: this module must stay import-light (no click, no third-party
4
+ imports). ``python -m kindling_cli.test_runner`` is the frontend Poe tasks
5
+ use to run tests from any interpreter — importing the package must never
6
+ require CLI dependencies. Pinned by
7
+ ``tests/unit/test_test_runner.py::test_runner_module_imports_without_click``.
8
+ """