spark-kindling-cli 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spark_kindling_cli-0.14.0/PKG-INFO +226 -0
- spark_kindling_cli-0.14.0/README.md +194 -0
- spark_kindling_cli-0.14.0/kindling_cli/__init__.py +8 -0
- spark_kindling_cli-0.14.0/kindling_cli/_runner.py +334 -0
- spark_kindling_cli-0.14.0/kindling_cli/_tracing.py +104 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_files.py +306 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/SKILL.md +83 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/project_instructions.md +15 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/apps.md +311 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/config.md +291 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/databricks.md +345 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/entities.md +310 -0
- spark_kindling_cli-0.14.0/kindling_cli/agent_skill/references/pipes.md +420 -0
- spark_kindling_cli-0.14.0/kindling_cli/bundle.py +1344 -0
- spark_kindling_cli-0.14.0/kindling_cli/cli.py +9927 -0
- spark_kindling_cli-0.14.0/kindling_cli/scaffold.py +346 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/.devcontainer/devcontainer.json.j2 +59 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/.env.example.j2 +40 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/.github/workflows/ci.yml.j2 +46 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/.gitignore.j2 +10 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/app.batch.py.j2 +13 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/app.file-ingestion.py.j2 +14 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/app.py.j2 +13 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/app.streaming.py.j2 +13 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/app.yaml.j2 +2 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/README.md +23 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/databricks.yml.j2 +18 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/resources/__pipeline__.pipeline.yml.j2 +21 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/bundle/databricks/src/kindling_lakeflow.py +9 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/lake-reqs.txt.j2 +5 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/package/PACKAGE_QUICKSTART.md.j2 +39 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/package/QUICKSTART.md.j2 +94 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/pyproject.root.toml.j2 +26 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/pyproject.toml.j2 +81 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/scripts/setup-local-dev.sh.j2 +34 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/settings.local.yaml.j2 +43 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/settings.yaml.j2 +14 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/src/entities/records.medallion.py.j2 +56 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/src/entities/records.minimal.py.j2 +31 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/src/pipes/bronze_to_silver.py.j2 +24 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/src/pipes/process.py.j2 +17 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/src/transforms/quality.medallion.py.j2 +26 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/src/transforms/quality.minimal.py.j2 +12 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/tests/component/test_registration.py.j2 +96 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/tests/conftest.py.j2 +325 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/tests/integration/test_pipeline.py.j2 +8 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/tests/integration/test_pipeline_azure.py.j2 +78 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/tests/integration/test_pipeline_local.py.j2 +45 -0
- spark_kindling_cli-0.14.0/kindling_cli/templates/tests/unit/test_transforms.py.j2 +60 -0
- spark_kindling_cli-0.14.0/kindling_cli/test_runner.py +451 -0
- spark_kindling_cli-0.14.0/pyproject.toml +54 -0
- spark_kindling_cli-0.14.0/pyproject.toml.orig +44 -0
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spark-kindling-cli
|
|
3
|
+
Version: 0.14.0
|
|
4
|
+
Summary: Command-line tooling for Kindling
|
|
5
|
+
Author: SEP Engineering
|
|
6
|
+
Author-email: SEP Engineering <engineering@sep.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Database
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Dist: click>=8.1.0
|
|
16
|
+
Requires-Dist: jinja2>=3.1.0
|
|
17
|
+
Requires-Dist: pyyaml>=6.0
|
|
18
|
+
Requires-Dist: spark-kindling-sdk>=0.9.10
|
|
19
|
+
Requires-Dist: requests>=2.31.0 ; extra == 'deploy'
|
|
20
|
+
Requires-Dist: azure-identity>=1.15.0 ; extra == 'deploy'
|
|
21
|
+
Requires-Dist: azure-storage-blob>=12.14.0 ; extra == 'deploy'
|
|
22
|
+
Requires-Dist: azure-identity>=1.15.0 ; extra == 'test-cleanup'
|
|
23
|
+
Requires-Dist: azure-storage-file-datalake>=12.14.0 ; extra == 'test-cleanup'
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Project-URL: Homepage, https://github.com/sep/spark-kindling-framework
|
|
26
|
+
Project-URL: Documentation, https://github.com/sep/spark-kindling-framework/tree/main/docs
|
|
27
|
+
Project-URL: Changelog, https://github.com/sep/spark-kindling-framework/blob/main/CHANGELOG.md
|
|
28
|
+
Project-URL: Issues, https://github.com/sep/spark-kindling-framework/issues
|
|
29
|
+
Provides-Extra: deploy
|
|
30
|
+
Provides-Extra: test-cleanup
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
|
|
33
|
+
# spark-kindling-cli
|
|
34
|
+
|
|
35
|
+
Command-line tooling for the Spark Kindling Framework. Distributed as
|
|
36
|
+
`spark-kindling-cli`; installs the `kindling` console script.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install spark-kindling-cli
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
The CLI depends on `spark-kindling-sdk` (used for remote platform lifecycle
|
|
45
|
+
operations), so this installs the SDK too. The CLI only requires a minimum SDK
|
|
46
|
+
version, so to pin one Kindling release, pin both:
|
|
47
|
+
`pip install spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`.
|
|
48
|
+
|
|
49
|
+
Without PyPI access (or for a release before 0.14.0, which exists only on
|
|
50
|
+
GitHub), install both from a
|
|
51
|
+
[GitHub Release](https://github.com/sep/spark-kindling-framework/releases)'s
|
|
52
|
+
wheel URLs:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
V=0.14.0 # any release, without the leading v
|
|
56
|
+
BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
|
|
57
|
+
pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" \
|
|
58
|
+
"$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Inside the Kindling devcontainer, `kindling` is a shim that runs the project's
|
|
62
|
+
own `./.venv/bin/kindling` (pinned in `pyproject.toml`) when present, and
|
|
63
|
+
otherwise installs `spark-kindling-cli` from PyPI (falling back to the latest
|
|
64
|
+
GitHub release's wheels).
|
|
65
|
+
|
|
66
|
+
## Commands
|
|
67
|
+
|
|
68
|
+
- `kindling repo init` — scaffold a multi-package repo root (uv workspace root `pyproject.toml`, devcontainer, CI) with shared dev tooling
|
|
69
|
+
- `kindling package init` — scaffold a package (a uv workspace member) under `packages/<name>` in an existing repo
|
|
70
|
+
- `kindling app init` — scaffold an app under `apps/<name>` that runs a domain package
|
|
71
|
+
- `kindling config init` — generate a starter `settings.yaml`
|
|
72
|
+
- `kindling config set <key> <value>` — set a config value using dot-notation
|
|
73
|
+
- `kindling env check` — validate the local Python/config environment
|
|
74
|
+
- `kindling env bootstrap` — pin Kindling if nothing declares it yet and `uv sync --all-packages` (the devcontainer's `postCreateCommand`)
|
|
75
|
+
- `kindling env update` / `kindling env add` — move or add Kindling pins to a release (PyPI version pins; release wheel URLs for packages not on PyPI; `--source` forces either)
|
|
76
|
+
- `kindling env ensure` — download Hadoop/ABFSS JARs into `/tmp/hadoop-jars/`
|
|
77
|
+
- `kindling workspace check` — validate the configured platform workspace
|
|
78
|
+
- `kindling workspace init` — scaffold bootstrap + starter notebook files for a platform
|
|
79
|
+
- `kindling workspace deploy` — upload the runtime wheel, bootstrap script, config, and notebooks to artifact storage
|
|
80
|
+
- `kindling app package` — build a `.kda` archive from a local app directory
|
|
81
|
+
- `kindling app deploy` — deploy an app directory or `.kda` package through the SDK
|
|
82
|
+
- `kindling app run` — run all local app pipes on `standalone`, or submit to the durable Kindling runner on a managed platform
|
|
83
|
+
- `kindling app status` — fetch the current remote app run status
|
|
84
|
+
- `kindling app logs` — fetch or stream remote app run logs
|
|
85
|
+
- `kindling app cancel` — cancel an active remote app run
|
|
86
|
+
- `kindling app cleanup` — remove a deployed app from remote storage
|
|
87
|
+
- `kindling runner ensure` — install or verify the durable Kindling runner on a platform
|
|
88
|
+
- `kindling runner status` — check whether the runner is installed and healthy
|
|
89
|
+
- `kindling runner repair` — reinstall the runner (delete and recreate)
|
|
90
|
+
- `kindling runner delete` — remove the runner from a platform
|
|
91
|
+
- `kindling bundle build` — precompile a project into a Databricks bundle (`databricks.yml`, pipeline resources, generic Lakeflow source, manifest) for `databricks bundle validate/deploy/run`
|
|
92
|
+
|
|
93
|
+
Run any command with `--help` for full options.
|
|
94
|
+
|
|
95
|
+
## Databricks bundles
|
|
96
|
+
|
|
97
|
+
`kindling bundle build` assembles a disposable Databricks bundle from a
|
|
98
|
+
project's `config/` overlays and `data-apps/` (or `apps/`) directories. It
|
|
99
|
+
starts no Spark session and imports no app code. Deployment inputs come from
|
|
100
|
+
options or `KINDLING_BUNDLE_*` environment variables (options win; runtime
|
|
101
|
+
settings files are never consulted):
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
kindling bundle build \
|
|
105
|
+
--name sales --target dev --app orders \
|
|
106
|
+
--workspace-host https://adb-123.azuredatabricks.net \
|
|
107
|
+
--catalog dev_sales --schema orders \
|
|
108
|
+
--wheel dist/spark_kindling-0.12.49-py3-none-any.whl \
|
|
109
|
+
--wheel dist/spark_kindling_ext_sdp-0.3.4-py3-none-any.whl \
|
|
110
|
+
--wheel dist/spark_kindling_ext_databricks-0.2.0-py3-none-any.whl \
|
|
111
|
+
--wheel dist/orders_kindling_app-1.4.0-py3-none-any.whl
|
|
112
|
+
|
|
113
|
+
cd dist/bundles/databricks
|
|
114
|
+
databricks bundle validate -t dev
|
|
115
|
+
databricks bundle deploy -t dev
|
|
116
|
+
databricks bundle run -t dev orders
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The bundle is rendered from a **template**: ordinary Databricks bundle YAML
|
|
120
|
+
with Jinja placeholders. The built-in template produces `databricks.yml`, one
|
|
121
|
+
`resources/<key>.pipeline.yml` per pipeline and the generic Lakeflow source,
|
|
122
|
+
driven by `--app-options-json` (per-app catalog/schema/continuous/pipes, or a
|
|
123
|
+
`pipelines` map to split an app). When a project needs its own resource keys,
|
|
124
|
+
display names, tags, per-pipeline permissions, clusters or anything else DAB
|
|
125
|
+
supports, copy the template and own it:
|
|
126
|
+
|
|
127
|
+
```bash
|
|
128
|
+
kindling bundle template init # -> bundle-template/
|
|
129
|
+
kindling bundle build --template-dir bundle-template ...
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Inside a template the generator supplies only what it uniquely knows. Settings
|
|
133
|
+
are found by convention, never listed: the shared `config/` overlays
|
|
134
|
+
(`settings.yaml`, `settings.databricks.yaml`, `workspace_<id>.yaml`,
|
|
135
|
+
`settings.<env>.yaml`) and then each app's own `settings*.yaml` are merged at
|
|
136
|
+
build time in the runtime's order and exposed as `apps[<name>].settings`.
|
|
137
|
+
`kindling.configuration(<app>, pipes=[...], extra={...})` returns a pipeline's
|
|
138
|
+
complete `configuration` map: the app selection, the inline
|
|
139
|
+
`kindling.lakeflow.settings_json`, the pipe subset, any extra flat overrides,
|
|
140
|
+
and a `kindling.lakeflow.config_keys` naming every emitted key a restricted
|
|
141
|
+
runtime must point-look-up. Rendered resources are parsed back and a pipeline
|
|
142
|
+
whose configuration did not come from the helper, or whose `config_keys` is
|
|
143
|
+
incomplete, fails the build. The deployed pipeline reads no settings files and
|
|
144
|
+
the resource file is the complete, reviewable description of what it runs
|
|
145
|
+
with. Files ending in `.j2` are rendered (`__pipeline__` in a name renders
|
|
146
|
+
once per pipeline); other files are copied verbatim. `manifest.json` records
|
|
147
|
+
the generator version, inputs, the SHA-256 of every settings file, wheel and
|
|
148
|
+
template file; no timestamps, so identical inputs give identical output.
|
|
149
|
+
`--app` may be omitted to build every app under `data-apps/` or `apps/`.
|
|
150
|
+
See `docs/proposals/databricks_bundle_deployment.md`.
|
|
151
|
+
|
|
152
|
+
Wheels passed with `--wheel` are uploaded by `databricks bundle deploy` under
|
|
153
|
+
the bundle's own workspace root and become the pipeline environment's
|
|
154
|
+
dependencies in the order given. Serverless installs them one at a time and
|
|
155
|
+
a wheel's missing dependency would otherwise be fetched from PyPI (or fail
|
|
156
|
+
where PyPI is unreachable) instead of using the uploaded wheel, so list
|
|
157
|
+
dependencies first: framework core, then `spark-kindling-ext-sdp`, then
|
|
158
|
+
`spark-kindling-ext-databricks`, then the app wheel.
|
|
159
|
+
|
|
160
|
+
The default `--workspace-root` is `/Workspace/Shared/kindling/<name>/<target>`:
|
|
161
|
+
a stable path that survives redeploys by different principals, which is
|
|
162
|
+
what shared dev/prod targets need. `databricks bundle validate` warns that
|
|
163
|
+
`/Workspace/Shared` is writable by all workspace users; for a personal
|
|
164
|
+
development target pass `--workspace-root /Workspace/Users/<you>/...`, and
|
|
165
|
+
for shared targets either accept the warning or grant the intended group
|
|
166
|
+
through `--permissions-json`.
|
|
167
|
+
|
|
168
|
+
## Scaffolding
|
|
169
|
+
|
|
170
|
+
The scaffolding commands now target true multi-package repos:
|
|
171
|
+
|
|
172
|
+
```bash
|
|
173
|
+
git clone <your-empty-repo-url> data-platform
|
|
174
|
+
cd data-platform
|
|
175
|
+
kindling repo init data-platform
|
|
176
|
+
|
|
177
|
+
kindling package init sales-ops --repo-root .
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
This produces a repo root with shared `.devcontainer/`, CI,
|
|
181
|
+
`scripts/setup-local-dev.sh` and `.gitignore` plus independently buildable
|
|
182
|
+
packages under `packages/`. The root `pyproject.toml` is a non-package uv
|
|
183
|
+
workspace root (`[tool.uv] package = false`,
|
|
184
|
+
`[tool.uv.workspace] members = ["packages/*"]`), so the repo shares one
|
|
185
|
+
`.venv/` and `uv.lock` with every package installed editable. Run
|
|
186
|
+
`kindling env bootstrap` at the root (the devcontainer does this on create) to
|
|
187
|
+
pin Kindling there and sync. Every package must pin the same Kindling release
|
|
188
|
+
as the root; `kindling env update` moves them together.
|
|
189
|
+
If `.devcontainer/` already exists, `kindling repo init` warns and leaves it
|
|
190
|
+
unchanged. Pass `--overwrite-devcontainer` to replace the generated
|
|
191
|
+
devcontainer config. An existing root `pyproject.toml` is also kept; add
|
|
192
|
+
`[tool.uv.workspace] members = ["packages/*"]` to it.
|
|
193
|
+
|
|
194
|
+
To add a second package later, return to the repo root and scaffold another
|
|
195
|
+
package:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
kindling package init customer-360 --repo-root .
|
|
199
|
+
|
|
200
|
+
cd packages/customer_360
|
|
201
|
+
uv run poe test
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
`uv run` syncs what the package needs before running. Don't run a bare `uv sync` inside a package directory: in the workspace it is an exact sync of that one package and removes the others from the shared `.venv/`. Resync the whole repo with `uv sync --all-packages` (or `kindling env bootstrap`) at the root.
|
|
205
|
+
|
|
206
|
+
The generated CI workflow runs in the devcontainer image and tests each
|
|
207
|
+
scaffolded package independently by iterating `packages/*` and executing:
|
|
208
|
+
|
|
209
|
+
```bash
|
|
210
|
+
uv run poe test
|
|
211
|
+
uv run poe build
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
The CI job fails if no package `pyproject.toml` files are found under
|
|
215
|
+
`packages/`.
|
|
216
|
+
|
|
217
|
+
Apps are scaffolded separately from packages:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
kindling app init sales-ops --package sales-ops --repo-root .
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
## Related
|
|
224
|
+
|
|
225
|
+
- Runtime framework: `pip install 'spark-kindling[<platform>]'` where `<platform>` is one of `synapse`, `databricks`, `fabric`, `standalone`.
|
|
226
|
+
- Design-time SDK: `pip install spark-kindling-sdk`.
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
# spark-kindling-cli
|
|
2
|
+
|
|
3
|
+
Command-line tooling for the Spark Kindling Framework. Distributed as
|
|
4
|
+
`spark-kindling-cli`; installs the `kindling` console script.
|
|
5
|
+
|
|
6
|
+
## Install
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install spark-kindling-cli
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
The CLI depends on `spark-kindling-sdk` (used for remote platform lifecycle
|
|
13
|
+
operations), so this installs the SDK too. The CLI only requires a minimum SDK
|
|
14
|
+
version, so to pin one Kindling release, pin both:
|
|
15
|
+
`pip install spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`.
|
|
16
|
+
|
|
17
|
+
Without PyPI access (or for a release before 0.14.0, which exists only on
|
|
18
|
+
GitHub), install both from a
|
|
19
|
+
[GitHub Release](https://github.com/sep/spark-kindling-framework/releases)'s
|
|
20
|
+
wheel URLs:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
V=0.14.0 # any release, without the leading v
|
|
24
|
+
BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
|
|
25
|
+
pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" \
|
|
26
|
+
"$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Inside the Kindling devcontainer, `kindling` is a shim that runs the project's
|
|
30
|
+
own `./.venv/bin/kindling` (pinned in `pyproject.toml`) when present, and
|
|
31
|
+
otherwise installs `spark-kindling-cli` from PyPI (falling back to the latest
|
|
32
|
+
GitHub release's wheels).
|
|
33
|
+
|
|
34
|
+
## Commands
|
|
35
|
+
|
|
36
|
+
- `kindling repo init` — scaffold a multi-package repo root (uv workspace root `pyproject.toml`, devcontainer, CI) with shared dev tooling
|
|
37
|
+
- `kindling package init` — scaffold a package (a uv workspace member) under `packages/<name>` in an existing repo
|
|
38
|
+
- `kindling app init` — scaffold an app under `apps/<name>` that runs a domain package
|
|
39
|
+
- `kindling config init` — generate a starter `settings.yaml`
|
|
40
|
+
- `kindling config set <key> <value>` — set a config value using dot-notation
|
|
41
|
+
- `kindling env check` — validate the local Python/config environment
|
|
42
|
+
- `kindling env bootstrap` — pin Kindling if nothing declares it yet and `uv sync --all-packages` (the devcontainer's `postCreateCommand`)
|
|
43
|
+
- `kindling env update` / `kindling env add` — move or add Kindling pins to a release (PyPI version pins; release wheel URLs for packages not on PyPI; `--source` forces either)
|
|
44
|
+
- `kindling env ensure` — download Hadoop/ABFSS JARs into `/tmp/hadoop-jars/`
|
|
45
|
+
- `kindling workspace check` — validate the configured platform workspace
|
|
46
|
+
- `kindling workspace init` — scaffold bootstrap + starter notebook files for a platform
|
|
47
|
+
- `kindling workspace deploy` — upload the runtime wheel, bootstrap script, config, and notebooks to artifact storage
|
|
48
|
+
- `kindling app package` — build a `.kda` archive from a local app directory
|
|
49
|
+
- `kindling app deploy` — deploy an app directory or `.kda` package through the SDK
|
|
50
|
+
- `kindling app run` — run all local app pipes on `standalone`, or submit to the durable Kindling runner on a managed platform
|
|
51
|
+
- `kindling app status` — fetch the current remote app run status
|
|
52
|
+
- `kindling app logs` — fetch or stream remote app run logs
|
|
53
|
+
- `kindling app cancel` — cancel an active remote app run
|
|
54
|
+
- `kindling app cleanup` — remove a deployed app from remote storage
|
|
55
|
+
- `kindling runner ensure` — install or verify the durable Kindling runner on a platform
|
|
56
|
+
- `kindling runner status` — check whether the runner is installed and healthy
|
|
57
|
+
- `kindling runner repair` — reinstall the runner (delete and recreate)
|
|
58
|
+
- `kindling runner delete` — remove the runner from a platform
|
|
59
|
+
- `kindling bundle build` — precompile a project into a Databricks bundle (`databricks.yml`, pipeline resources, generic Lakeflow source, manifest) for `databricks bundle validate/deploy/run`
|
|
60
|
+
|
|
61
|
+
Run any command with `--help` for full options.
|
|
62
|
+
|
|
63
|
+
## Databricks bundles
|
|
64
|
+
|
|
65
|
+
`kindling bundle build` assembles a disposable Databricks bundle from a
|
|
66
|
+
project's `config/` overlays and `data-apps/` (or `apps/`) directories. It
|
|
67
|
+
starts no Spark session and imports no app code. Deployment inputs come from
|
|
68
|
+
options or `KINDLING_BUNDLE_*` environment variables (options win; runtime
|
|
69
|
+
settings files are never consulted):
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
kindling bundle build \
|
|
73
|
+
--name sales --target dev --app orders \
|
|
74
|
+
--workspace-host https://adb-123.azuredatabricks.net \
|
|
75
|
+
--catalog dev_sales --schema orders \
|
|
76
|
+
--wheel dist/spark_kindling-0.12.49-py3-none-any.whl \
|
|
77
|
+
--wheel dist/spark_kindling_ext_sdp-0.3.4-py3-none-any.whl \
|
|
78
|
+
--wheel dist/spark_kindling_ext_databricks-0.2.0-py3-none-any.whl \
|
|
79
|
+
--wheel dist/orders_kindling_app-1.4.0-py3-none-any.whl
|
|
80
|
+
|
|
81
|
+
cd dist/bundles/databricks
|
|
82
|
+
databricks bundle validate -t dev
|
|
83
|
+
databricks bundle deploy -t dev
|
|
84
|
+
databricks bundle run -t dev orders
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
The bundle is rendered from a **template**: ordinary Databricks bundle YAML
|
|
88
|
+
with Jinja placeholders. The built-in template produces `databricks.yml`, one
|
|
89
|
+
`resources/<key>.pipeline.yml` per pipeline and the generic Lakeflow source,
|
|
90
|
+
driven by `--app-options-json` (per-app catalog/schema/continuous/pipes, or a
|
|
91
|
+
`pipelines` map to split an app). When a project needs its own resource keys,
|
|
92
|
+
display names, tags, per-pipeline permissions, clusters or anything else DAB
|
|
93
|
+
supports, copy the template and own it:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
kindling bundle template init # -> bundle-template/
|
|
97
|
+
kindling bundle build --template-dir bundle-template ...
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Inside a template the generator supplies only what it uniquely knows. Settings
|
|
101
|
+
are found by convention, never listed: the shared `config/` overlays
|
|
102
|
+
(`settings.yaml`, `settings.databricks.yaml`, `workspace_<id>.yaml`,
|
|
103
|
+
`settings.<env>.yaml`) and then each app's own `settings*.yaml` are merged at
|
|
104
|
+
build time in the runtime's order and exposed as `apps[<name>].settings`.
|
|
105
|
+
`kindling.configuration(<app>, pipes=[...], extra={...})` returns a pipeline's
|
|
106
|
+
complete `configuration` map: the app selection, the inline
|
|
107
|
+
`kindling.lakeflow.settings_json`, the pipe subset, any extra flat overrides,
|
|
108
|
+
and a `kindling.lakeflow.config_keys` naming every emitted key a restricted
|
|
109
|
+
runtime must point-look-up. Rendered resources are parsed back and a pipeline
|
|
110
|
+
whose configuration did not come from the helper, or whose `config_keys` is
|
|
111
|
+
incomplete, fails the build. The deployed pipeline reads no settings files and
|
|
112
|
+
the resource file is the complete, reviewable description of what it runs
|
|
113
|
+
with. Files ending in `.j2` are rendered (`__pipeline__` in a name renders
|
|
114
|
+
once per pipeline); other files are copied verbatim. `manifest.json` records
|
|
115
|
+
the generator version, inputs, the SHA-256 of every settings file, wheel and
|
|
116
|
+
template file; no timestamps, so identical inputs give identical output.
|
|
117
|
+
`--app` may be omitted to build every app under `data-apps/` or `apps/`.
|
|
118
|
+
See `docs/proposals/databricks_bundle_deployment.md`.
|
|
119
|
+
|
|
120
|
+
Wheels passed with `--wheel` are uploaded by `databricks bundle deploy` under
|
|
121
|
+
the bundle's own workspace root and become the pipeline environment's
|
|
122
|
+
dependencies in the order given. Serverless installs them one at a time and
|
|
123
|
+
a wheel's missing dependency would otherwise be fetched from PyPI (or fail
|
|
124
|
+
where PyPI is unreachable) instead of using the uploaded wheel, so list
|
|
125
|
+
dependencies first: framework core, then `spark-kindling-ext-sdp`, then
|
|
126
|
+
`spark-kindling-ext-databricks`, then the app wheel.
|
|
127
|
+
|
|
128
|
+
The default `--workspace-root` is `/Workspace/Shared/kindling/<name>/<target>`:
|
|
129
|
+
a stable path that survives redeploys by different principals, which is
|
|
130
|
+
what shared dev/prod targets need. `databricks bundle validate` warns that
|
|
131
|
+
`/Workspace/Shared` is writable by all workspace users; for a personal
|
|
132
|
+
development target pass `--workspace-root /Workspace/Users/<you>/...`, and
|
|
133
|
+
for shared targets either accept the warning or grant the intended group
|
|
134
|
+
through `--permissions-json`.
|
|
135
|
+
|
|
136
|
+
## Scaffolding
|
|
137
|
+
|
|
138
|
+
The scaffolding commands now target true multi-package repos:
|
|
139
|
+
|
|
140
|
+
```bash
|
|
141
|
+
git clone <your-empty-repo-url> data-platform
|
|
142
|
+
cd data-platform
|
|
143
|
+
kindling repo init data-platform
|
|
144
|
+
|
|
145
|
+
kindling package init sales-ops --repo-root .
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
This produces a repo root with shared `.devcontainer/`, CI,
|
|
149
|
+
`scripts/setup-local-dev.sh` and `.gitignore` plus independently buildable
|
|
150
|
+
packages under `packages/`. The root `pyproject.toml` is a non-package uv
|
|
151
|
+
workspace root (`[tool.uv] package = false`,
|
|
152
|
+
`[tool.uv.workspace] members = ["packages/*"]`), so the repo shares one
|
|
153
|
+
`.venv/` and `uv.lock` with every package installed editable. Run
|
|
154
|
+
`kindling env bootstrap` at the root (the devcontainer does this on create) to
|
|
155
|
+
pin Kindling there and sync. Every package must pin the same Kindling release
|
|
156
|
+
as the root; `kindling env update` moves them together.
|
|
157
|
+
If `.devcontainer/` already exists, `kindling repo init` warns and leaves it
|
|
158
|
+
unchanged. Pass `--overwrite-devcontainer` to replace the generated
|
|
159
|
+
devcontainer config. An existing root `pyproject.toml` is also kept; add
|
|
160
|
+
`[tool.uv.workspace] members = ["packages/*"]` to it.
|
|
161
|
+
|
|
162
|
+
To add a second package later, return to the repo root and scaffold another
|
|
163
|
+
package:
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
kindling package init customer-360 --repo-root .
|
|
167
|
+
|
|
168
|
+
cd packages/customer_360
|
|
169
|
+
uv run poe test
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
`uv run` syncs what the package needs before running. Don't run a bare `uv sync` inside a package directory: in the workspace it is an exact sync of that one package and removes the others from the shared `.venv/`. Resync the whole repo with `uv sync --all-packages` (or `kindling env bootstrap`) at the root.
|
|
173
|
+
|
|
174
|
+
The generated CI workflow runs in the devcontainer image and tests each
|
|
175
|
+
scaffolded package independently by iterating `packages/*` and executing:
|
|
176
|
+
|
|
177
|
+
```bash
|
|
178
|
+
uv run poe test
|
|
179
|
+
uv run poe build
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
The CI job fails if no package `pyproject.toml` files are found under
|
|
183
|
+
`packages/`.
|
|
184
|
+
|
|
185
|
+
Apps are scaffolded separately from packages:
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
kindling app init sales-ops --package sales-ops --repo-root .
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## Related
|
|
192
|
+
|
|
193
|
+
- Runtime framework: `pip install 'spark-kindling[<platform>]'` where `<platform>` is one of `synapse`, `databricks`, `fabric`, `standalone`.
|
|
194
|
+
- Design-time SDK: `pip install spark-kindling-sdk`.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Command-line tooling for Kindling.
|
|
2
|
+
|
|
3
|
+
INVARIANT: this module must stay import-light (no click, no third-party
|
|
4
|
+
imports). ``python -m kindling_cli.test_runner`` is the frontend Poe tasks
|
|
5
|
+
use to run tests from any interpreter — importing the package must never
|
|
6
|
+
require CLI dependencies. Pinned by
|
|
7
|
+
``tests/unit/test_test_runner.py::test_runner_module_imports_without_click``.
|
|
8
|
+
"""
|