spark-kindling-ext-sdp 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spark_kindling_ext_sdp-0.14.0/PKG-INFO +213 -0
- spark_kindling_ext_sdp-0.14.0/README.md +191 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/__init__.py +87 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/bootstrap.py +105 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/capabilities.py +100 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/declaration_engine.py +949 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/declaration_plan.py +209 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/dry_run.py +161 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/engine_extension.py +39 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/guard_provider.py +91 -0
- spark_kindling_ext_sdp-0.14.0/kindling_ext_sdp/oss_engine.py +256 -0
- spark_kindling_ext_sdp-0.14.0/pyproject.toml +41 -0
- spark_kindling_ext_sdp-0.14.0/pyproject.toml.orig +42 -0
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spark-kindling-ext-sdp
|
|
3
|
+
Version: 0.14.0
|
|
4
|
+
Summary: Spark Declarative Pipelines (SDP) declaration engine for Kindling
|
|
5
|
+
Author: SEP Engineering
|
|
6
|
+
Author-email: SEP Engineering <engineering@sep.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Topic :: Database
|
|
14
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
15
|
+
Requires-Dist: spark-kindling>=0.14.0,<0.15
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Project-URL: Homepage, https://github.com/sep/spark-kindling-framework
|
|
18
|
+
Project-URL: Documentation, https://github.com/sep/spark-kindling-framework/tree/main/docs
|
|
19
|
+
Project-URL: Changelog, https://github.com/sep/spark-kindling-framework/blob/main/CHANGELOG.md
|
|
20
|
+
Project-URL: Issues, https://github.com/sep/spark-kindling-framework/issues
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# kindling_ext_sdp
|
|
24
|
+
|
|
25
|
+
Spark Declarative Pipelines (SDP) declaration engine for Kindling.
|
|
26
|
+
|
|
27
|
+
This package turns existing `DataEntities`/`DataPipes` registrations into a
|
|
28
|
+
validated, pure-metadata `DeclarationPlan` that a concrete engine can emit as
|
|
29
|
+
`pyspark.pipelines` (OSS Spark 4.1+) or Databricks Lakeflow declarations.
|
|
30
|
+
|
|
31
|
+
Install it from PyPI with `pip install spark-kindling-ext-sdp`; in a uv project,
|
|
32
|
+
`kindling env add spark-kindling-ext-sdp` pins the version that matches the
|
|
33
|
+
project's Kindling release.
|
|
34
|
+
|
|
35
|
+
Phase 1 contains:
|
|
36
|
+
|
|
37
|
+
- The abstract `DeclarationEngine` interface and the shared plan builder.
|
|
38
|
+
- Internal/external input classification derived from the registries.
|
|
39
|
+
- Capability gating (`OSS_SDP` vs `DATABRICKS_SDP` feature sets) with
|
|
40
|
+
fail-fast, all-errors-at-once validation.
|
|
41
|
+
- Dataset-type selection (entity tag, then engine config, then default
|
|
42
|
+
`materialized_view`).
|
|
43
|
+
|
|
44
|
+
Phase 2 adds:
|
|
45
|
+
|
|
46
|
+
- `OssSdpEngine` — emits the plan through the OSS `pyspark.pipelines`
|
|
47
|
+
decorator API (materialized views only). Provider-owned streaming sources
|
|
48
|
+
are recognized in the shared plan but capability-gated off for OSS SDP
|
|
49
|
+
until Spark pipeline parity is accepted separately. The
|
|
50
|
+
`dp` module is an injected dependency resolved lazily at declaration
|
|
51
|
+
time, so importing this package still never imports `pyspark.pipelines`
|
|
52
|
+
and Kindling keeps supporting Spark runtimes older than 4.1.
|
|
53
|
+
- `SdpWriteGuardProvider` — the write-inert provider personality installed
|
|
54
|
+
by SDP-mode bootstrap ("SDP owns persistence"): reads delegate, every
|
|
55
|
+
write path raises `SdpModeWriteError`.
|
|
56
|
+
- The bootstrap surface: `kindling.initialize(engine="sdp")` activates the
|
|
57
|
+
guard and skips the watermark aspect (SDP owns incrementality);
|
|
58
|
+
`kindling.declare_pipeline()` — the mandatory LAST step of the entry
|
|
59
|
+
point — builds, validates, and declares the plan.
|
|
60
|
+
- The local validation harness (`write_pipeline_spec` + `dry_run`):
|
|
61
|
+
ephemeral `spark-pipeline.yml` + `spark-pipelines dry-run`, a
|
|
62
|
+
unit-test-tier gate for the declared graph with no cluster in the loop
|
|
63
|
+
(requires the Spark 4.1+ CLI: `pip install 'pyspark[pipelines]>=4.1'`).
|
|
64
|
+
|
|
65
|
+
Phase 3 adds:
|
|
66
|
+
|
|
67
|
+
- Full entity-metadata emission on the OSS engine: `comment`,
|
|
68
|
+
`table_properties` (entity tags `sdp.table_properties.<key>` over an
|
|
69
|
+
engine-config `table_properties` block, tag winning per key),
|
|
70
|
+
`partition_cols`, `cluster_by`, and `schema` — the complete Spark 4.1
|
|
71
|
+
`dp.materialized_view` keyword surface. Deliberate divergence from the
|
|
72
|
+
runner engine, per the dual-engine parity criterion: SDP does NOT force
|
|
73
|
+
`delta.enableChangeDataFeed` (that feeds the runner's watermark
|
|
74
|
+
machinery); declare it as a tag if external consumers need CDF.
|
|
75
|
+
- An `engine_factory` seam on `declare_pipeline()` so adapter packages
|
|
76
|
+
reuse the whole bootstrap path.
|
|
77
|
+
- The `kindling_ext_databricks` adapter package (separate README):
|
|
78
|
+
Lakeflow expectations, selected via `engine="databricks_sdp"`.
|
|
79
|
+
|
|
80
|
+
Provider-owned streaming sources:
|
|
81
|
+
|
|
82
|
+
- The shared planner can classify an external input as
|
|
83
|
+
`EXTERNAL_STREAMING_SOURCE` when its provider implements
|
|
84
|
+
`DeclarableStreamingSource` and `StreamableEntityProvider`.
|
|
85
|
+
- The provider returns an inert, secret-safe `StreamingSourceSpec`; plans and
|
|
86
|
+
diagnostics include tag/option names and validation constraints, never
|
|
87
|
+
connection strings, SAS keys, passwords, JAAS values, or resolved secrets.
|
|
88
|
+
- A streaming source must be a driving input selected by
|
|
89
|
+
`driving_entity_ids`; only one declarable streaming source is supported in
|
|
90
|
+
a pipe, and temporal/AUTO CDC compositions reject the ambiguous shape.
|
|
91
|
+
- The output dataset type is inferred as `streaming_table`. Explicit
|
|
92
|
+
materialized-view requests fail with `streaming_dataset_type_conflict`
|
|
93
|
+
instead of being silently overridden.
|
|
94
|
+
- `engine="sdp"` reports `streaming_source_lowering_not_supported` for this
|
|
95
|
+
shape today. `engine="databricks_sdp"` lowers it through Lakeflow; see the
|
|
96
|
+
Databricks extension README.
|
|
97
|
+
- An ordinary external **Delta** input has no provider-owned stream, so it is
|
|
98
|
+
a batch read unless a pipe opts in with the adapter-tier engine-config key
|
|
99
|
+
`streaming_inputs: [<entity id>]`. The opt-in reaches the same emission
|
|
100
|
+
(streaming table + one append flow, non-driving inputs staying batch), is
|
|
101
|
+
never inferred, and must name every driving input. See the Databricks
|
|
102
|
+
extension README for the full rejection list.
|
|
103
|
+
|
|
104
|
+
Entry point shape (fixed bootstrap surface — never generated code):
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
import kindling
|
|
108
|
+
|
|
109
|
+
kindling.initialize(engine="sdp")
|
|
110
|
+
|
|
111
|
+
from my_app import register_all
|
|
112
|
+
register_all()
|
|
113
|
+
|
|
114
|
+
kindling.declare_pipeline()
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
See:
|
|
118
|
+
|
|
119
|
+
- `docs/proposals/declarative_pipelines_engine.md` — the full proposal.
|
|
120
|
+
- `docs/proposals/obsolete/sdp_engine_phase1_notes.md` — Phase-1 architecture
|
|
121
|
+
decisions (bootstrap ordering, provider write-inertness, deferrals).
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
## Output dataset naming
|
|
125
|
+
|
|
126
|
+
Configure names within the pipeline's target catalog/schema independently
|
|
127
|
+
of logical entity IDs and external table resolution:
|
|
128
|
+
|
|
129
|
+
```yaml
|
|
130
|
+
kindling:
|
|
131
|
+
sdp:
|
|
132
|
+
dataset_naming: leaf
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
| Mode | Logical entity ID | Emitted dataset name |
|
|
136
|
+
|---|---|---|
|
|
137
|
+
| `normalized` (default) | `silver.device_telemetry` | `silver_device_telemetry` |
|
|
138
|
+
| `leaf` | `silver.device_telemetry` | `device_telemetry` |
|
|
139
|
+
|
|
140
|
+
Both modes replace hyphens with underscores. Config values ignore surrounding
|
|
141
|
+
whitespace and case; an omitted or null value uses `normalized`. Unknown modes
|
|
142
|
+
are reported alongside other declaration validation errors.
|
|
143
|
+
The setting is resolved by `declare_pipeline()` after configuration overlays
|
|
144
|
+
and applies to both `sdp` and `databricks_sdp`. Direct engine constructors
|
|
145
|
+
accept `dataset_naming="leaf"`; `DatasetNameMapper` implements the shared
|
|
146
|
+
pipeline-local naming strategy.
|
|
147
|
+
|
|
148
|
+
Names must be unique (case-insensitively) among selected outputs in one
|
|
149
|
+
declaration plan. Use `kindling.declare_pipeline(pipe_ids=[...])` to select
|
|
150
|
+
the pipes belonging to each resource. Separate bronze and silver resources
|
|
151
|
+
can each emit `device_telemetry`; selecting both same-leaf outputs in one
|
|
152
|
+
resource fails before emission with the conflicting entity and pipe IDs.
|
|
153
|
+
|
|
154
|
+
Internal batch/stream reads use these single-part names. External Delta batch
|
|
155
|
+
reads resolve the registered entity metadata through the runtime's
|
|
156
|
+
`EntityNameMapper` before calling `spark.table`. This honors
|
|
157
|
+
`provider.table_catalog`, `provider.table_schema`, `provider.table_name`, and
|
|
158
|
+
`provider.table_name_strategy`, independently of the pipeline's current catalog
|
|
159
|
+
and its output naming mode. Logical IDs and pipe function argument names stay
|
|
160
|
+
unchanged; an explicit `external_read_resolver` still receives the logical ID.
|
|
161
|
+
Provider-owned streaming sources retain their provider streaming read path.
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
### Reading leaf-named outputs from elsewhere
|
|
165
|
+
|
|
166
|
+
Enabling `leaf` does not change the consumer's `EntityNameMapper`. With
|
|
167
|
+
catalog `dev_silver` and schema `cwmdp` configured, that mapper still resolves
|
|
168
|
+
`silver.device_telemetry` to `dev_silver.cwmdp.silver_device_telemetry`,
|
|
169
|
+
while the leaf-mode pipeline writes `dev_silver.cwmdp.device_telemetry`.
|
|
170
|
+
Align the consumer's entity metadata with the mapper's `leaf` name strategy:
|
|
171
|
+
|
|
172
|
+
```yaml
|
|
173
|
+
dataentities-bytag:
|
|
174
|
+
ldp_output:
|
|
175
|
+
"true":
|
|
176
|
+
tags:
|
|
177
|
+
provider.table_name_strategy: leaf
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The strategy retains the entity's resolved catalog and schema, but uses only the
|
|
181
|
+
logical entity ID's final segment for its table name. Apply the `ldp_output`
|
|
182
|
+
tag to the matching entities; `dataentities-bytag` adds the mapper strategy to
|
|
183
|
+
the whole family. The pipeline continues to declare the single-part name
|
|
184
|
+
`device_telemetry`.
|
|
185
|
+
|
|
186
|
+
`provider.table_name` remains a complete override and takes precedence over
|
|
187
|
+
the name strategy, `provider.table_catalog`, and `provider.table_schema`.
|
|
188
|
+
|
|
189
|
+
Temporal chain base-event sources currently always use external table
|
|
190
|
+
resolution, even if a producing pipe is selected in the same pipeline.
|
|
191
|
+
Keep those sources in an upstream resource and align their external names as
|
|
192
|
+
above; producing a source in the same resource does not establish a local
|
|
193
|
+
temporal dependency.
|
|
194
|
+
|
|
195
|
+
On `engine="databricks_sdp"`, `kindling.lakeflow.temporal_mode: batch` lowers
|
|
196
|
+
`<events>__g0..gN` as materialized views with batch reads instead of streaming
|
|
197
|
+
tables with append flows; the reserved names below are unchanged in both
|
|
198
|
+
modes. See the
|
|
199
|
+
[Databricks extension documentation](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks/README.md#temporal-chain-execution-mode).
|
|
200
|
+
|
|
201
|
+
### Generated dataset names
|
|
202
|
+
|
|
203
|
+
Lakeflow reserves helper names as well as selected outputs. Validation rejects
|
|
204
|
+
collisions with AUTO CDC `<target>__scd_source`, temporal `<events>__g0..N`
|
|
205
|
+
(where N is `kindling.temporal.max_generations`), `<events>__ghi`,
|
|
206
|
+
`<events>__determinations`, and `<episodes>__episode_snapshot`.
|
|
207
|
+
Conditional temporal helper names are reserved even when the current rules
|
|
208
|
+
do not use them, so later rule changes cannot introduce name collisions.
|
|
209
|
+
|
|
210
|
+
Selecting a temporal chain-events pipe also reserves and emits its registered
|
|
211
|
+
chain-episodes sibling, even when the episodes pipe is not explicitly selected.
|
|
212
|
+
These reservations apply only to the resource containing that chain or CDC
|
|
213
|
+
target; other pipeline resources can use the same names.
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
# kindling_ext_sdp
|
|
2
|
+
|
|
3
|
+
Spark Declarative Pipelines (SDP) declaration engine for Kindling.
|
|
4
|
+
|
|
5
|
+
This package turns existing `DataEntities`/`DataPipes` registrations into a
|
|
6
|
+
validated, pure-metadata `DeclarationPlan` that a concrete engine can emit as
|
|
7
|
+
`pyspark.pipelines` (OSS Spark 4.1+) or Databricks Lakeflow declarations.
|
|
8
|
+
|
|
9
|
+
Install it from PyPI with `pip install spark-kindling-ext-sdp`; in a uv project,
|
|
10
|
+
`kindling env add spark-kindling-ext-sdp` pins the version that matches the
|
|
11
|
+
project's Kindling release.
|
|
12
|
+
|
|
13
|
+
Phase 1 contains:
|
|
14
|
+
|
|
15
|
+
- The abstract `DeclarationEngine` interface and the shared plan builder.
|
|
16
|
+
- Internal/external input classification derived from the registries.
|
|
17
|
+
- Capability gating (`OSS_SDP` vs `DATABRICKS_SDP` feature sets) with
|
|
18
|
+
fail-fast, all-errors-at-once validation.
|
|
19
|
+
- Dataset-type selection (entity tag, then engine config, then default
|
|
20
|
+
`materialized_view`).
|
|
21
|
+
|
|
22
|
+
Phase 2 adds:
|
|
23
|
+
|
|
24
|
+
- `OssSdpEngine` — emits the plan through the OSS `pyspark.pipelines`
|
|
25
|
+
decorator API (materialized views only). Provider-owned streaming sources
|
|
26
|
+
are recognized in the shared plan but capability-gated off for OSS SDP
|
|
27
|
+
until Spark pipeline parity is accepted separately. The
|
|
28
|
+
`dp` module is an injected dependency resolved lazily at declaration
|
|
29
|
+
time, so importing this package still never imports `pyspark.pipelines`
|
|
30
|
+
and Kindling keeps supporting Spark runtimes older than 4.1.
|
|
31
|
+
- `SdpWriteGuardProvider` — the write-inert provider personality installed
|
|
32
|
+
by SDP-mode bootstrap ("SDP owns persistence"): reads delegate, every
|
|
33
|
+
write path raises `SdpModeWriteError`.
|
|
34
|
+
- The bootstrap surface: `kindling.initialize(engine="sdp")` activates the
|
|
35
|
+
guard and skips the watermark aspect (SDP owns incrementality);
|
|
36
|
+
`kindling.declare_pipeline()` — the mandatory LAST step of the entry
|
|
37
|
+
point — builds, validates, and declares the plan.
|
|
38
|
+
- The local validation harness (`write_pipeline_spec` + `dry_run`):
|
|
39
|
+
ephemeral `spark-pipeline.yml` + `spark-pipelines dry-run`, a
|
|
40
|
+
unit-test-tier gate for the declared graph with no cluster in the loop
|
|
41
|
+
(requires the Spark 4.1+ CLI: `pip install 'pyspark[pipelines]>=4.1'`).
|
|
42
|
+
|
|
43
|
+
Phase 3 adds:
|
|
44
|
+
|
|
45
|
+
- Full entity-metadata emission on the OSS engine: `comment`,
|
|
46
|
+
`table_properties` (entity tags `sdp.table_properties.<key>` over an
|
|
47
|
+
engine-config `table_properties` block, tag winning per key),
|
|
48
|
+
`partition_cols`, `cluster_by`, and `schema` — the complete Spark 4.1
|
|
49
|
+
`dp.materialized_view` keyword surface. Deliberate divergence from the
|
|
50
|
+
runner engine, per the dual-engine parity criterion: SDP does NOT force
|
|
51
|
+
`delta.enableChangeDataFeed` (that feeds the runner's watermark
|
|
52
|
+
machinery); declare it as a tag if external consumers need CDF.
|
|
53
|
+
- An `engine_factory` seam on `declare_pipeline()` so adapter packages
|
|
54
|
+
reuse the whole bootstrap path.
|
|
55
|
+
- The `kindling_ext_databricks` adapter package (separate README):
|
|
56
|
+
Lakeflow expectations, selected via `engine="databricks_sdp"`.
|
|
57
|
+
|
|
58
|
+
Provider-owned streaming sources:
|
|
59
|
+
|
|
60
|
+
- The shared planner can classify an external input as
|
|
61
|
+
`EXTERNAL_STREAMING_SOURCE` when its provider implements
|
|
62
|
+
`DeclarableStreamingSource` and `StreamableEntityProvider`.
|
|
63
|
+
- The provider returns an inert, secret-safe `StreamingSourceSpec`; plans and
|
|
64
|
+
diagnostics include tag/option names and validation constraints, never
|
|
65
|
+
connection strings, SAS keys, passwords, JAAS values, or resolved secrets.
|
|
66
|
+
- A streaming source must be a driving input selected by
|
|
67
|
+
`driving_entity_ids`; only one declarable streaming source is supported in
|
|
68
|
+
a pipe, and temporal/AUTO CDC compositions reject the ambiguous shape.
|
|
69
|
+
- The output dataset type is inferred as `streaming_table`. Explicit
|
|
70
|
+
materialized-view requests fail with `streaming_dataset_type_conflict`
|
|
71
|
+
instead of being silently overridden.
|
|
72
|
+
- `engine="sdp"` reports `streaming_source_lowering_not_supported` for this
|
|
73
|
+
shape today. `engine="databricks_sdp"` lowers it through Lakeflow; see the
|
|
74
|
+
Databricks extension README.
|
|
75
|
+
- An ordinary external **Delta** input has no provider-owned stream, so it is
|
|
76
|
+
a batch read unless a pipe opts in with the adapter-tier engine-config key
|
|
77
|
+
`streaming_inputs: [<entity id>]`. The opt-in reaches the same emission
|
|
78
|
+
(streaming table + one append flow, non-driving inputs staying batch), is
|
|
79
|
+
never inferred, and must name every driving input. See the Databricks
|
|
80
|
+
extension README for the full rejection list.
|
|
81
|
+
|
|
82
|
+
Entry point shape (fixed bootstrap surface — never generated code):
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import kindling
|
|
86
|
+
|
|
87
|
+
kindling.initialize(engine="sdp")
|
|
88
|
+
|
|
89
|
+
from my_app import register_all
|
|
90
|
+
register_all()
|
|
91
|
+
|
|
92
|
+
kindling.declare_pipeline()
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
See:
|
|
96
|
+
|
|
97
|
+
- `docs/proposals/declarative_pipelines_engine.md` — the full proposal.
|
|
98
|
+
- `docs/proposals/obsolete/sdp_engine_phase1_notes.md` — Phase-1 architecture
|
|
99
|
+
decisions (bootstrap ordering, provider write-inertness, deferrals).
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
## Output dataset naming
|
|
103
|
+
|
|
104
|
+
Configure names within the pipeline's target catalog/schema independently
|
|
105
|
+
of logical entity IDs and external table resolution:
|
|
106
|
+
|
|
107
|
+
```yaml
|
|
108
|
+
kindling:
|
|
109
|
+
sdp:
|
|
110
|
+
dataset_naming: leaf
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
| Mode | Logical entity ID | Emitted dataset name |
|
|
114
|
+
|---|---|---|
|
|
115
|
+
| `normalized` (default) | `silver.device_telemetry` | `silver_device_telemetry` |
|
|
116
|
+
| `leaf` | `silver.device_telemetry` | `device_telemetry` |
|
|
117
|
+
|
|
118
|
+
Both modes replace hyphens with underscores. Config values ignore surrounding
|
|
119
|
+
whitespace and case; an omitted or null value uses `normalized`. Unknown modes
|
|
120
|
+
are reported alongside other declaration validation errors.
|
|
121
|
+
The setting is resolved by `declare_pipeline()` after configuration overlays
|
|
122
|
+
and applies to both `sdp` and `databricks_sdp`. Direct engine constructors
|
|
123
|
+
accept `dataset_naming="leaf"`; `DatasetNameMapper` implements the shared
|
|
124
|
+
pipeline-local naming strategy.
|
|
125
|
+
|
|
126
|
+
Names must be unique (case-insensitively) among selected outputs in one
|
|
127
|
+
declaration plan. Use `kindling.declare_pipeline(pipe_ids=[...])` to select
|
|
128
|
+
the pipes belonging to each resource. Separate bronze and silver resources
|
|
129
|
+
can each emit `device_telemetry`; selecting both same-leaf outputs in one
|
|
130
|
+
resource fails before emission with the conflicting entity and pipe IDs.
|
|
131
|
+
|
|
132
|
+
Internal batch/stream reads use these single-part names. External Delta batch
|
|
133
|
+
reads resolve the registered entity metadata through the runtime's
|
|
134
|
+
`EntityNameMapper` before calling `spark.table`. This honors
|
|
135
|
+
`provider.table_catalog`, `provider.table_schema`, `provider.table_name`, and
|
|
136
|
+
`provider.table_name_strategy`, independently of the pipeline's current catalog
|
|
137
|
+
and its output naming mode. Logical IDs and pipe function argument names stay
|
|
138
|
+
unchanged; an explicit `external_read_resolver` still receives the logical ID.
|
|
139
|
+
Provider-owned streaming sources retain their provider streaming read path.
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
### Reading leaf-named outputs from elsewhere
|
|
143
|
+
|
|
144
|
+
Enabling `leaf` does not change the consumer's `EntityNameMapper`. With
|
|
145
|
+
catalog `dev_silver` and schema `cwmdp` configured, that mapper still resolves
|
|
146
|
+
`silver.device_telemetry` to `dev_silver.cwmdp.silver_device_telemetry`,
|
|
147
|
+
while the leaf-mode pipeline writes `dev_silver.cwmdp.device_telemetry`.
|
|
148
|
+
Align the consumer's entity metadata with the mapper's `leaf` name strategy:
|
|
149
|
+
|
|
150
|
+
```yaml
|
|
151
|
+
dataentities-bytag:
|
|
152
|
+
ldp_output:
|
|
153
|
+
"true":
|
|
154
|
+
tags:
|
|
155
|
+
provider.table_name_strategy: leaf
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
The strategy retains the entity's resolved catalog and schema, but uses only the
|
|
159
|
+
logical entity ID's final segment for its table name. Apply the `ldp_output`
|
|
160
|
+
tag to the matching entities; `dataentities-bytag` adds the mapper strategy to
|
|
161
|
+
the whole family. The pipeline continues to declare the single-part name
|
|
162
|
+
`device_telemetry`.
|
|
163
|
+
|
|
164
|
+
`provider.table_name` remains a complete override and takes precedence over
|
|
165
|
+
the name strategy, `provider.table_catalog`, and `provider.table_schema`.
|
|
166
|
+
|
|
167
|
+
Temporal chain base-event sources currently always use external table
|
|
168
|
+
resolution, even if a producing pipe is selected in the same pipeline.
|
|
169
|
+
Keep those sources in an upstream resource and align their external names as
|
|
170
|
+
above; producing a source in the same resource does not establish a local
|
|
171
|
+
temporal dependency.
|
|
172
|
+
|
|
173
|
+
On `engine="databricks_sdp"`, `kindling.lakeflow.temporal_mode: batch` lowers
|
|
174
|
+
`<events>__g0..gN` as materialized views with batch reads instead of streaming
|
|
175
|
+
tables with append flows; the reserved names below are unchanged in both
|
|
176
|
+
modes. See the
|
|
177
|
+
[Databricks extension documentation](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks/README.md#temporal-chain-execution-mode).
|
|
178
|
+
|
|
179
|
+
### Generated dataset names
|
|
180
|
+
|
|
181
|
+
Lakeflow reserves helper names as well as selected outputs. Validation rejects
|
|
182
|
+
collisions with AUTO CDC `<target>__scd_source`, temporal `<events>__g0..N`
|
|
183
|
+
(where N is `kindling.temporal.max_generations`), `<events>__ghi`,
|
|
184
|
+
`<events>__determinations`, and `<episodes>__episode_snapshot`.
|
|
185
|
+
Conditional temporal helper names are reserved even when the current rules
|
|
186
|
+
do not use them, so later rule changes cannot introduce name collisions.
|
|
187
|
+
|
|
188
|
+
Selecting a temporal chain-events pipe also reserves and emits its registered
|
|
189
|
+
chain-episodes sibling, even when the episodes pipe is not explicitly selected.
|
|
190
|
+
These reservations apply only to the resource containing that chain or CDC
|
|
191
|
+
target; other pipeline resources can use the same names.
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
"""Spark Declarative Pipelines (SDP) declaration engine for Kindling.
|
|
2
|
+
|
|
3
|
+
Phase 1: abstract declaration-engine interface, internal/external input
|
|
4
|
+
classification, capability gating, and the pure-metadata DeclarationPlan.
|
|
5
|
+
|
|
6
|
+
Phase 2: the OSS emission engine (``OssSdpEngine``), the write-inert
|
|
7
|
+
provider guard, the SDP bootstrap surface (``kindling.initialize(
|
|
8
|
+
engine="sdp")`` / ``kindling.declare_pipeline()``), and the local
|
|
9
|
+
``spark-pipelines dry-run`` validation harness.
|
|
10
|
+
|
|
11
|
+
Importing this package never imports ``pyspark.pipelines``; the ``dp``
|
|
12
|
+
module is resolved lazily at declaration time (Spark 4.1+) or injected.
|
|
13
|
+
|
|
14
|
+
See ``docs/proposals/declarative_pipelines_engine.md`` and
|
|
15
|
+
``docs/proposals/sdp_engine_phase1_notes.md``.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from kindling_ext_sdp.capabilities import (
|
|
19
|
+
ADAPTER_TIER_CONFIG_KEYS,
|
|
20
|
+
DATABRICKS_SDP,
|
|
21
|
+
OSS_SDP,
|
|
22
|
+
CapabilitySet,
|
|
23
|
+
SdpFeature,
|
|
24
|
+
supports_auto_cdc,
|
|
25
|
+
supports_expectations,
|
|
26
|
+
supports_incremental_mv_refresh,
|
|
27
|
+
supports_streaming_source_lowering,
|
|
28
|
+
)
|
|
29
|
+
from kindling_ext_sdp.declaration_engine import (
|
|
30
|
+
DATASET_TYPE_CONFIG_KEY,
|
|
31
|
+
DATASET_TYPE_TAG,
|
|
32
|
+
DeclarationEngine,
|
|
33
|
+
)
|
|
34
|
+
from kindling_ext_sdp.declaration_plan import (
|
|
35
|
+
ClassifiedInput,
|
|
36
|
+
DatasetDeclaration,
|
|
37
|
+
DatasetNameMapper,
|
|
38
|
+
DatasetType,
|
|
39
|
+
DeclarationIssue,
|
|
40
|
+
DeclarationPlan,
|
|
41
|
+
DeclarationValidationError,
|
|
42
|
+
InputClassification,
|
|
43
|
+
)
|
|
44
|
+
from kindling_ext_sdp.dry_run import (
|
|
45
|
+
DryRunResult,
|
|
46
|
+
SparkPipelinesCliNotFoundError,
|
|
47
|
+
dry_run,
|
|
48
|
+
write_pipeline_spec,
|
|
49
|
+
)
|
|
50
|
+
from kindling_ext_sdp.engine_extension import SdpEngineExtension, engine_extension
|
|
51
|
+
from kindling_ext_sdp.guard_provider import SdpModeWriteError, SdpWriteGuardProvider
|
|
52
|
+
from kindling_ext_sdp.oss_engine import OssSdpEngine, SdpRuntimeUnavailableError
|
|
53
|
+
|
|
54
|
+
__version__ = "0.14.0"
|
|
55
|
+
|
|
56
|
+
__all__ = [
|
|
57
|
+
"ADAPTER_TIER_CONFIG_KEYS",
|
|
58
|
+
"CapabilitySet",
|
|
59
|
+
"ClassifiedInput",
|
|
60
|
+
"DATABRICKS_SDP",
|
|
61
|
+
"DATASET_TYPE_CONFIG_KEY",
|
|
62
|
+
"DATASET_TYPE_TAG",
|
|
63
|
+
"DatasetDeclaration",
|
|
64
|
+
"DatasetNameMapper",
|
|
65
|
+
"DatasetType",
|
|
66
|
+
"DeclarationEngine",
|
|
67
|
+
"DeclarationIssue",
|
|
68
|
+
"DeclarationPlan",
|
|
69
|
+
"DeclarationValidationError",
|
|
70
|
+
"DryRunResult",
|
|
71
|
+
"InputClassification",
|
|
72
|
+
"OSS_SDP",
|
|
73
|
+
"OssSdpEngine",
|
|
74
|
+
"SdpEngineExtension",
|
|
75
|
+
"SdpFeature",
|
|
76
|
+
"SdpModeWriteError",
|
|
77
|
+
"SdpRuntimeUnavailableError",
|
|
78
|
+
"SdpWriteGuardProvider",
|
|
79
|
+
"SparkPipelinesCliNotFoundError",
|
|
80
|
+
"dry_run",
|
|
81
|
+
"engine_extension",
|
|
82
|
+
"write_pipeline_spec",
|
|
83
|
+
"supports_auto_cdc",
|
|
84
|
+
"supports_expectations",
|
|
85
|
+
"supports_incremental_mv_refresh",
|
|
86
|
+
"supports_streaming_source_lowering",
|
|
87
|
+
]
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""SDP-mode bootstrap surface: guard installation and ``declare_pipeline``.
|
|
2
|
+
|
|
3
|
+
The entry-point shape from the proposal:
|
|
4
|
+
|
|
5
|
+
.. code-block:: python
|
|
6
|
+
|
|
7
|
+
import kindling
|
|
8
|
+
kindling.initialize(engine="sdp") # activates SDP mode (guard, no
|
|
9
|
+
# watermark aspect)
|
|
10
|
+
from my_app import register_all
|
|
11
|
+
register_all()
|
|
12
|
+
kindling.declare_pipeline() # delegates here
|
|
13
|
+
|
|
14
|
+
Ordering matters (Phase-1 decision, ``sdp_engine_phase1_notes.md``):
|
|
15
|
+
``declare_pipeline()`` must be the LAST step — after registrations and
|
|
16
|
+
after the post-registration config overlay — or engine keys like
|
|
17
|
+
``dataset_type`` bake in stale values. Entity tags are overlay-safe
|
|
18
|
+
automatically (``get_entity_definition()`` merges config tag overrides at
|
|
19
|
+
retrieval time); the per-pipe engine config is resolved here, from the
|
|
20
|
+
fully-overlaid config, at declaration time.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from typing import Any, Dict, List, Optional
|
|
24
|
+
|
|
25
|
+
from kindling_ext_sdp.declaration_plan import DeclarationPlan
|
|
26
|
+
from kindling_ext_sdp.guard_provider import SdpWriteGuardProvider
|
|
27
|
+
from kindling_ext_sdp.oss_engine import OssSdpEngine
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def activate_sdp_mode() -> None:
|
|
31
|
+
"""Install the write-inert provider guard for this process.
|
|
32
|
+
|
|
33
|
+
Called by ``kindling.initialize(engine="sdp")`` after core framework
|
|
34
|
+
initialization. Idempotent: the registry treats re-installing the same
|
|
35
|
+
decorator as a no-op (and refuses to stack a different one).
|
|
36
|
+
"""
|
|
37
|
+
from kindling.entity_provider_registry import EntityProviderRegistry
|
|
38
|
+
from kindling.injection import GlobalInjector
|
|
39
|
+
|
|
40
|
+
registry = GlobalInjector.get(EntityProviderRegistry)
|
|
41
|
+
registry.set_provider_decorator(SdpWriteGuardProvider)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def resolve_engine_config(config_service, pipe_ids: List[str]) -> Dict[str, Dict[str, Any]]:
|
|
45
|
+
"""Resolve ``datapipes.<pipeid>.engine`` blocks from the overlaid config."""
|
|
46
|
+
engine_config: Dict[str, Dict[str, Any]] = {}
|
|
47
|
+
for pipe_id in pipe_ids:
|
|
48
|
+
block = config_service.get(f"datapipes.{pipe_id}.engine", None)
|
|
49
|
+
if block:
|
|
50
|
+
engine_config[pipe_id] = dict(block)
|
|
51
|
+
return engine_config
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def declare_pipeline(
|
|
55
|
+
pipe_ids: Optional[List[str]] = None,
|
|
56
|
+
dp_module: Any = None,
|
|
57
|
+
engine_factory: Any = None,
|
|
58
|
+
) -> DeclarationPlan:
|
|
59
|
+
"""Build, validate, and declare the pipeline from the live registries.
|
|
60
|
+
|
|
61
|
+
``engine_factory(entity_registry, pipe_registry, engine_config=...,
|
|
62
|
+
dp_module=..., dataset_naming=...)`` constructs the concrete engine; defaults to
|
|
63
|
+
:class:`OssSdpEngine`. Adapter packages (``kindling_ext_databricks``)
|
|
64
|
+
pass their own engine class here and reuse everything else.
|
|
65
|
+
|
|
66
|
+
Returns the validated plan (useful for logging/inspection). Raises
|
|
67
|
+
``DeclarationValidationError`` with every accumulated issue when any
|
|
68
|
+
selected pipe cannot be declared safely.
|
|
69
|
+
"""
|
|
70
|
+
from kindling.data_entities import DataEntityRegistry
|
|
71
|
+
from kindling.data_pipes import DataPipesRegistry
|
|
72
|
+
from kindling.injection import GlobalInjector
|
|
73
|
+
from kindling.spark_config import ConfigService
|
|
74
|
+
|
|
75
|
+
entity_registry = GlobalInjector.get(DataEntityRegistry)
|
|
76
|
+
pipe_registry = GlobalInjector.get(DataPipesRegistry)
|
|
77
|
+
config_service = GlobalInjector.get(ConfigService)
|
|
78
|
+
pending: Dict[str, str] = {}
|
|
79
|
+
for registry in (entity_registry, pipe_registry):
|
|
80
|
+
reported = getattr(registry, "pending_derivations", lambda: {})()
|
|
81
|
+
if isinstance(reported, dict): # custom or mocked registries may not implement it
|
|
82
|
+
pending.update(reported)
|
|
83
|
+
if pending:
|
|
84
|
+
details = "; ".join(f"{target}: {reason}" for target, reason in sorted(pending.items()))
|
|
85
|
+
raise ValueError(
|
|
86
|
+
"Cannot declare the pipeline while declaration derivations are unresolved "
|
|
87
|
+
f"({details}). Register the source declarations or drop the derivations."
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
selected = list(pipe_ids) if pipe_ids is not None else list(pipe_registry.get_pipe_ids())
|
|
91
|
+
factory = engine_factory or OssSdpEngine
|
|
92
|
+
engine = factory(
|
|
93
|
+
entity_registry,
|
|
94
|
+
pipe_registry,
|
|
95
|
+
engine_config=resolve_engine_config(config_service, selected),
|
|
96
|
+
dp_module=dp_module,
|
|
97
|
+
dataset_naming=str(
|
|
98
|
+
config_service.get("kindling.sdp.dataset_naming", "normalized") or "normalized"
|
|
99
|
+
)
|
|
100
|
+
.strip()
|
|
101
|
+
.lower(),
|
|
102
|
+
)
|
|
103
|
+
plan = engine.build_plan(selected)
|
|
104
|
+
engine.declare_pipeline(plan)
|
|
105
|
+
return plan
|