spark-kindling 0.14.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. spark_kindling-0.14.0/LICENSE +21 -0
  2. spark_kindling-0.14.0/PKG-INFO +319 -0
  3. spark_kindling-0.14.0/README.md +265 -0
  4. spark_kindling-0.14.0/packages/kindling/__init__.py +174 -0
  5. spark_kindling-0.14.0/packages/kindling/app_files.py +60 -0
  6. spark_kindling-0.14.0/packages/kindling/apps.py +59 -0
  7. spark_kindling-0.14.0/packages/kindling/bootstrap.py +3108 -0
  8. spark_kindling-0.14.0/packages/kindling/cache_optimizer.py +94 -0
  9. spark_kindling-0.14.0/packages/kindling/common_transforms.py +28 -0
  10. spark_kindling-0.14.0/packages/kindling/config_loaders.py +380 -0
  11. spark_kindling-0.14.0/packages/kindling/config_patterns.py +322 -0
  12. spark_kindling-0.14.0/packages/kindling/data_apps.py +1427 -0
  13. spark_kindling-0.14.0/packages/kindling/data_entities.py +2061 -0
  14. spark_kindling-0.14.0/packages/kindling/data_pipes.py +1075 -0
  15. spark_kindling-0.14.0/packages/kindling/declaration_derivations.py +437 -0
  16. spark_kindling-0.14.0/packages/kindling/entity_provider.py +495 -0
  17. spark_kindling-0.14.0/packages/kindling/entity_provider_adx.py +537 -0
  18. spark_kindling-0.14.0/packages/kindling/entity_provider_csv.py +380 -0
  19. spark_kindling-0.14.0/packages/kindling/entity_provider_current_view.py +81 -0
  20. spark_kindling-0.14.0/packages/kindling/entity_provider_delta.py +2646 -0
  21. spark_kindling-0.14.0/packages/kindling/entity_provider_eventhub.py +979 -0
  22. spark_kindling-0.14.0/packages/kindling/entity_provider_eventhub_declaration.py +246 -0
  23. spark_kindling-0.14.0/packages/kindling/entity_provider_memory.py +873 -0
  24. spark_kindling-0.14.0/packages/kindling/entity_provider_parquet.py +243 -0
  25. spark_kindling-0.14.0/packages/kindling/entity_provider_registry.py +266 -0
  26. spark_kindling-0.14.0/packages/kindling/entity_provider_sql.py +221 -0
  27. spark_kindling-0.14.0/packages/kindling/entity_resolution.py +354 -0
  28. spark_kindling-0.14.0/packages/kindling/execution_orchestrator.py +148 -0
  29. spark_kindling-0.14.0/packages/kindling/execution_strategy.py +675 -0
  30. spark_kindling-0.14.0/packages/kindling/features.py +224 -0
  31. spark_kindling-0.14.0/packages/kindling/file_ingestion.py +778 -0
  32. spark_kindling-0.14.0/packages/kindling/generation_executor.py +1299 -0
  33. spark_kindling-0.14.0/packages/kindling/injection.py +121 -0
  34. spark_kindling-0.14.0/packages/kindling/job_deployment.py +545 -0
  35. spark_kindling-0.14.0/packages/kindling/migration.py +1051 -0
  36. spark_kindling-0.14.0/packages/kindling/notebook_framework.py +1974 -0
  37. spark_kindling-0.14.0/packages/kindling/notebook_source.py +125 -0
  38. spark_kindling-0.14.0/packages/kindling/package_registrations.py +60 -0
  39. spark_kindling-0.14.0/packages/kindling/pip_manager.py +334 -0
  40. spark_kindling-0.14.0/packages/kindling/pipe_graph.py +569 -0
  41. spark_kindling-0.14.0/packages/kindling/pipe_streaming.py +258 -0
  42. spark_kindling-0.14.0/packages/kindling/plain_telemetry.py +274 -0
  43. spark_kindling-0.14.0/packages/kindling/platform_databricks.py +1293 -0
  44. spark_kindling-0.14.0/packages/kindling/platform_fabric.py +1142 -0
  45. spark_kindling-0.14.0/packages/kindling/platform_provider.py +177 -0
  46. spark_kindling-0.14.0/packages/kindling/platform_standalone.py +807 -0
  47. spark_kindling-0.14.0/packages/kindling/platform_synapse.py +1145 -0
  48. spark_kindling-0.14.0/packages/kindling/sentinels.py +22 -0
  49. spark_kindling-0.14.0/packages/kindling/signaling.py +620 -0
  50. spark_kindling-0.14.0/packages/kindling/simple_read_persist_strategy.py +467 -0
  51. spark_kindling-0.14.0/packages/kindling/simple_stage_processor.py +128 -0
  52. spark_kindling-0.14.0/packages/kindling/spark_config.py +954 -0
  53. spark_kindling-0.14.0/packages/kindling/spark_jdbc.py +197 -0
  54. spark_kindling-0.14.0/packages/kindling/spark_log.py +156 -0
  55. spark_kindling-0.14.0/packages/kindling/spark_log_provider.py +23 -0
  56. spark_kindling-0.14.0/packages/kindling/spark_session.py +131 -0
  57. spark_kindling-0.14.0/packages/kindling/spark_trace.py +558 -0
  58. spark_kindling-0.14.0/packages/kindling/streaming_health_monitor.py +593 -0
  59. spark_kindling-0.14.0/packages/kindling/streaming_listener.py +546 -0
  60. spark_kindling-0.14.0/packages/kindling/streaming_orchestrator.py +428 -0
  61. spark_kindling-0.14.0/packages/kindling/streaming_query_manager.py +589 -0
  62. spark_kindling-0.14.0/packages/kindling/streaming_recovery_manager.py +737 -0
  63. spark_kindling-0.14.0/packages/kindling/surrogate_keys.py +887 -0
  64. spark_kindling-0.14.0/packages/kindling/test_framework.py +1438 -0
  65. spark_kindling-0.14.0/packages/kindling/trace_ops.py +265 -0
  66. spark_kindling-0.14.0/packages/kindling/watermarking.py +684 -0
  67. spark_kindling-0.14.0/pyproject.toml +525 -0
  68. spark_kindling-0.14.0/pyproject.toml.orig +264 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Software Engineering Professionals, Inc.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,319 @@
1
+ Metadata-Version: 2.4
2
+ Name: spark-kindling
3
+ Version: 0.14.0
4
+ Summary: A unified framework for data apps across Fabric, Synapse, and Databricks
5
+ Keywords: spark,pyspark,databricks,microsoft-fabric,azure-synapse,delta-lake,data-engineering
6
+ Author: SEP Engineering
7
+ Author-email: SEP Engineering <engineering@sep.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Topic :: Database
16
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
17
+ Requires-Dist: injector>=0.20.1
18
+ Requires-Dist: dynaconf>=3.1.0,!=3.3.4,<3.3.6
19
+ Requires-Dist: pyyaml>=6.0
20
+ Requires-Dist: packaging>=23.0
21
+ Requires-Dist: blinker>=1.6.0
22
+ Requires-Dist: azure-core>=1.30.0
23
+ Requires-Dist: azure-identity>=1.12.0
24
+ Requires-Dist: azure-storage-file-datalake>=12.0.0
25
+ Requires-Dist: azure-kusto-data>=4.0.0 ; extra == 'adx'
26
+ Requires-Dist: azure-kusto-ingest>=4.0.0 ; extra == 'adx'
27
+ Requires-Dist: azure-synapse-artifacts>=0.17.0 ; extra == 'all'
28
+ Requires-Dist: databricks-sdk>=0.12.0 ; extra == 'all'
29
+ Requires-Dist: pyspark>=3.4.0,<4.0.0 ; extra == 'all'
30
+ Requires-Dist: delta-spark>=2.4.0,<4.0.0 ; extra == 'all'
31
+ Requires-Dist: pandas>=2.0.0 ; extra == 'all'
32
+ Requires-Dist: pyarrow>=12.0.0 ; extra == 'all'
33
+ Requires-Dist: azure-kusto-data>=4.0.0 ; extra == 'all'
34
+ Requires-Dist: azure-kusto-ingest>=4.0.0 ; extra == 'all'
35
+ Requires-Dist: databricks-sdk>=0.12.0 ; extra == 'databricks'
36
+ Requires-Dist: azure-core>=1.30.0,<1.31.0 ; extra == 'fabric'
37
+ Requires-Dist: pyspark>=3.4.0,<4.0.0 ; extra == 'standalone'
38
+ Requires-Dist: delta-spark>=2.4.0,<4.0.0 ; extra == 'standalone'
39
+ Requires-Dist: pandas>=2.0.0 ; extra == 'standalone'
40
+ Requires-Dist: pyarrow>=12.0.0 ; extra == 'standalone'
41
+ Requires-Dist: azure-synapse-artifacts>=0.17.0 ; extra == 'synapse'
42
+ Requires-Python: >=3.10
43
+ Project-URL: Homepage, https://github.com/sep/spark-kindling-framework
44
+ Project-URL: Documentation, https://github.com/sep/spark-kindling-framework/tree/main/docs
45
+ Project-URL: Changelog, https://github.com/sep/spark-kindling-framework/blob/main/CHANGELOG.md
46
+ Project-URL: Issues, https://github.com/sep/spark-kindling-framework/issues
47
+ Provides-Extra: adx
48
+ Provides-Extra: all
49
+ Provides-Extra: databricks
50
+ Provides-Extra: fabric
51
+ Provides-Extra: standalone
52
+ Provides-Extra: synapse
53
+ Description-Content-Type: text/markdown
54
+
55
+ <!-- polecat smoke test: workflow verified ok -->
56
+ # Spark Kindling Framework
57
+
58
+ **Import name:** `kindling`
59
+ **Platforms:** Microsoft Fabric, Azure Synapse Analytics, Databricks
60
+
61
+ ## Overview
62
+
63
+ Spark Kindling Framework is a comprehensive solution for building robust data pipelines on Apache Spark, specifically designed for cross-platform solutions, via notebook or python-first development. It provides a declarative, dependency-injection-driven approach to defining and executing data transformations while maintaining strong governance and enabling robust observability.
64
+
65
+ ### Key Capabilities
66
+
67
+ - **Multi-Platform Support** - Unified API across Fabric, Synapse, and Databricks (including UC shared/standard access mode clusters) plus local/standalone Spark
68
+ - **Data Apps & Job Deployment** - Package and deploy apps as Spark jobs
69
+ - **Batch & Streaming Pipelines** - Incremental watermarked reads, streaming pipes with SCD1/SCD2 merge sinks
70
+ - **Pluggable Entity Providers** - Delta, Parquet, CSV, SQL views, in-memory, Event Hubs, and Azure Data Explorer out of the box
71
+ - **Hierarchical Configuration** - Platform, workspace, and environment-specific configs; execution options resolve config-first
72
+ - **Schema Reconciliation** - `kindling migrate` plans and applies drift between declared entities and live tables
73
+ - **Extensibility** - Plugin system for custom telemetry and integrations
74
+ - **Enterprise Observability** - Built-in logging, tracing, and Azure Monitor integration
75
+
76
+ ## Documentation
77
+
78
+ ### Core Features
79
+ - [Introduction](https://github.com/sep/spark-kindling-framework/blob/main/docs/intro.md) - Framework overview and architecture
80
+ - [Setup Guide](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/setup_guide.md) - Installation and configuration
81
+ - [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) - Scaffold, run, and test without cloud credentials
82
+ - [CLI Reference](https://github.com/sep/spark-kindling-framework/blob/main/docs/reference/cli_reference.md) - Every `kindling` command (scaffolding, apps, migrate, notebooks, workspace)
83
+ - [Data Entities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_entities.md) - Data entity management system
84
+ - [Entity Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/entity_configuration.md) - Tags-first declaration conventions
85
+ - [Data Pipes](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_pipes.md) - Transformation pipeline system
86
+ - [Derived Datasets](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/derived_datasets.md) - Replacement writes: derived datasets, slice replace, insert-only
87
+ - [Entity Providers](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/entity_providers.md) - Storage abstraction system
88
+ - [Migrating from runMultiple](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/migrating_from_runmultiple.md) - Move Synapse/Fabric notebook DAGs to Kindling pipes
89
+
90
+ ### Advanced Features
91
+ - [Job Deployment](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/job_deployment.md) - Deploy apps as Spark jobs
92
+ - [Hierarchical Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_workspace_config.md) - Multi-level YAML config system
93
+ - [Logging & Tracing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/logging_tracing.md) - Observability foundation (including JVM-free telemetry for UC shared clusters)
94
+ - [Watermarking](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/watermarking.md) - Change tracking and incremental processing
95
+ - [File Ingestion](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/file_ingestion.md) - Built-in file ingestion capabilities
96
+ - [Stage Processing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/stage_processing.md) - Pipeline stage orchestration
97
+ - [Temporal End-to-End](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/temporal_end_to_end.md) - Events, conditions, and episodes (spark-kindling-ext-temporal)
98
+ - [Lakeflow App Selection](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/lakeflow_app_selection.md) - Run Kindling data apps inside Databricks Lakeflow pipelines
99
+ - [Dynamic Registration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/dynamic_registration.md) - Register entities and pipes at runtime
100
+
101
+ ### Platform & Development
102
+ - [Platform API Architecture](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_api_architecture.md) - Multi-platform abstraction
103
+ - [Platform Storage Utils](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_storage_utils.md) - Storage operations
104
+ - [Utilities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/utilities.md) - Common utilities and helper functions
105
+ - [Build System](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/build_system.md) - How `poe build` produces the runtime, CLI, SDK and extension wheels
106
+ - [CI/CD Setup](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/ci_cd_setup.md) - Continuous integration and deployment
107
+ - [Testing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/testing.md) - Unit, integration, and system testing
108
+
109
+ ## Core Modules
110
+
111
+ The framework consists of several modular components:
112
+
113
+ - **Dependency Injection Engine** - Provides IoC container for loose coupling
114
+ - **Data Entities** - Entity registry and storage abstraction
115
+ - **Data Pipes** - Transformation pipeline definition and execution
116
+ - **Data Apps** - Package and deploy apps as .kda archives
117
+ - **Job Deployment** - Deploy apps as Spark jobs across platforms
118
+ - **Configuration System** - Hierarchical YAML configuration (platform/workspace/environment)
119
+ - **Platform Services** - Unified abstraction for Fabric/Synapse/Databricks
120
+ - **Watermarking** - Change tracking for incremental processing
121
+ - **Streaming Orchestration** - Streaming queries with health monitoring, recovery, and merge sinks
122
+ - **Migration** - Plan/apply reconciliation of declared entities against live Delta tables and views
123
+ - **File Ingestion** - File pattern discovery and loading
124
+ - **Stage Processing** - Orchestration of multi-stage pipelines
125
+ - **Common Transforms** - Reusable data transformation utilities
126
+ - **Logging & Tracing** - Comprehensive observability features
127
+
128
+ ## Extensions
129
+
130
+ - **[spark-kindling-ext-otel-azure](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_otel_azure)** - Azure Monitor OpenTelemetry integration
131
+ - **[spark-kindling-ext-sdp](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_sdp)** - Spark Declarative Pipelines (SDP) declaration engine
132
+ - **[spark-kindling-ext-databricks](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks)** - Databricks Lakeflow adapter for the SDP declaration engine
133
+ - **[spark-kindling-ext-databricks-autoloader](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks_autoloader)** - Auto Loader (`cloudFiles`) discovery for file ingestion
134
+ - **[spark-kindling-ext-temporal](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_temporal)** - Temporal event, condition, and episode primitives
135
+ - **[spark-kindling-ext-adx](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_adx)** - Azure Data Explorer entity provider (Kusto Spark connector; an API-based `adx-api` provider ships in core)
136
+ - **[spark-kindling-ext-cosmos](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_cosmos)** - Azure Cosmos DB entity provider (idempotent upsert writes)
137
+ - **[spark-kindling-ext-visualization](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_visualization)** - Matplotlib visualization helpers
138
+
139
+ In a Kindling project, add any extension with `kindling env add <name>`. The Databricks, SDP, Cosmos DB, Temporal and OpenTelemetry Azure extensions are also on PyPI (`pip install <name>`). The ADX, Auto Loader and visualization extensions are not; they are attached to each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) as wheels, which `kindling env add` pins by URL (or `pip install` the wheel's release URL).
140
+
141
+ ## Install
142
+
143
+ Kindling is on [PyPI](https://pypi.org/project/spark-kindling/). One distribution, platform-specific extras:
144
+
145
+ ```bash
146
+ pip install 'spark-kindling[synapse]' # Azure Synapse Analytics
147
+ pip install 'spark-kindling[databricks]' # Databricks
148
+ pip install 'spark-kindling[fabric]' # Microsoft Fabric
149
+ pip install 'spark-kindling[standalone]' # Local development / generic Spark
150
+ ```
151
+
152
+ The Python import name is `kindling` (unchanged):
153
+
154
+ ```python
155
+ from kindling.data_entities import DataEntities
156
+ ```
157
+
158
+ Design-time tooling ships separately:
159
+
160
+ ```bash
161
+ # `kindling` CLI for scaffolding and deploy; it pulls in spark-kindling-sdk (programmatic access to platform APIs)
162
+ pip install spark-kindling-cli
163
+ ```
164
+
165
+ The runtime, CLI and SDK are versioned together: pin all three to one release (`spark-kindling[...]==0.14.0 spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`; the CLI alone only requires a minimum SDK). Extensions keep their own versions; each GitHub release lists the extension versions that go with it, and `kindling env add` picks them. Projects scaffolded with `kindling repo init` / `kindling package init` pin these versions in `pyproject.toml` for you, and `kindling env update` moves them to a newer release.
166
+
167
+ ### Without PyPI access
168
+
169
+ Each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) also attaches the wheels, so environments that cannot reach PyPI (and releases before 0.14.0, which exist only on GitHub) can install by URL:
170
+
171
+ ```bash
172
+ V=0.14.0 # any release, without the leading v
173
+ BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
174
+ pip install "spark-kindling[databricks] @ $BASE/spark_kindling-$V-py3-none-any.whl"
175
+ pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" "$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
176
+ ```
177
+
178
+ See [docs/release_process.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/release_process.md) for how releases are published and [docs/developer_workflow.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/developer_workflow.md) for local development.
179
+
180
+ ## CLI Quick Start
181
+
182
+ Install the CLI (`pip install spark-kindling-cli`; you also need [uv](https://docs.astral.sh/uv/)) and scaffold a repo, package, and app explicitly:
183
+
184
+ ```bash
185
+ kindling repo init my-app --output-dir ./my_app
186
+ cd my_app
187
+ kindling package init my-app
188
+ kindling app init my-app --package my-app
189
+ kindling env bootstrap # pins Kindling at the root and syncs one repo-wide .venv/
190
+ uv run kindling app run my_app --env local
191
+ ```
192
+
193
+ Or open the generated repo in its devcontainer (`.devcontainer/devcontainer.json`), which runs `kindling env bootstrap` for you.
194
+ No Azure credentials needed — the scaffold uses in-memory entity providers by default.
195
+ See [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) for the full local workflow.
196
+
197
+ ## Notebook Quick Start
198
+
199
+ ```python
200
+ # Import Kindling framework
201
+ from kindling.data_entities import *
202
+ from kindling.data_pipes import *
203
+ from kindling.injection import get_kindling_service
204
+
205
+ # Define data entity (a plain call — registration happens immediately)
206
+ DataEntities.entity(
207
+ entityid="customers.raw",
208
+ name="Raw Customer Data",
209
+ partition_columns=["country"],
210
+ merge_columns=["customer_id"],
211
+ tags={"domain": "customer", "layer": "bronze"},
212
+ schema=customer_schema,
213
+ )
214
+
215
+ # Create data transformation pipe
216
+ @DataPipes.pipe(
217
+ pipeid="customers.transform",
218
+ name="Transform Customers",
219
+ tags={"layer": "silver"},
220
+ input_entity_ids=["customers.raw"],
221
+ output_entity_id="customers.silver",
222
+ output_type="table"
223
+ )
224
+ def transform_customers(customers_raw):
225
+ # Transformation logic here
226
+ return customers_raw.filter(...)
227
+
228
+ # Execute pipeline
229
+ executer = get_kindling_service(DataPipesExecution)
230
+ executer.run_datapipes(["customers.transform"])
231
+ ```
232
+
233
+ ## Key Features
234
+
235
+ ### Core Framework
236
+ - **Declarative definitions** for data entities and transformations (tags-first conventions)
237
+ - **Dependency injection** for loose coupling and testability
238
+ - **Delta Lake integration** for reliable storage and time travel
239
+ - **Built-in entity providers** - `delta`, `parquet`, `csv`, `memory`, `sql` (views), `current_view` (SCD2), `eventhub`, `adx-api`; ADX (connector) and Cosmos DB via extensions
240
+ - **Watermarking** for change tracking and incremental loads
241
+ - **Streaming pipes** with per-micro-batch SCD1/SCD2 merge sinks (`write.mode` tag)
242
+ - **Config-first execution options** - parallelism, error strategies (incl. `skip_dependents`), per-pipe retry via `kindling.execution.*`
243
+ - **Schema reconciliation** - `kindling migrate plan|apply` converges live tables to declared entities
244
+ - **Notebook round-tripping** - `kindling notebook list|pull|push` workspace notebooks as git-friendly `.py` source
245
+ - **Observability** through logging and tracing (JVM-free fallback on UC shared clusters)
246
+
247
+ ### Multi-Platform Support
248
+ - **Unified API** across Microsoft Fabric, Azure Synapse, and Databricks
249
+ - **Platform abstraction** - write once, deploy anywhere
250
+ - **Platform-specific optimizations** via configuration
251
+ - **Smart platform detection** - automatic platform identification
252
+
253
+ ### Data Apps & Deployment
254
+ - **Package apps** as .kda archives (Kindling Data Apps)
255
+ - **Run through the durable runner** - apps submit to one managed platform runner
256
+ - **Cross-platform deployment** - same app runs on all platforms
257
+ - **Artifact management** - wheels, configs, and dependencies
258
+
259
+ ### Configuration System
260
+ - **Hierarchical YAML configs** - settings, platform, workspace, environment layers
261
+ - **Auto-detection** - platform and workspace ID discovery
262
+ - **Flexible overrides** - bootstrap config for runtime changes
263
+ - **Multi-team support** - workspace-specific configurations
264
+
265
+ ### Extensibility
266
+ - **Extension system** - load custom packages via configuration
267
+ - **Azure Monitor integration** - via spark-kindling-ext-otel-azure extension
268
+ - **Declarative engines** - SDP declaration engine with a Databricks Lakeflow adapter (spark-kindling-ext-sdp / spark-kindling-ext-databricks)
269
+ - **Temporal primitives** - events, conditions, and episode lifecycle (spark-kindling-ext-temporal)
270
+ - **Custom providers** - implement your own storage backends
271
+ - **Signal/event system** - blinker-based pub/sub for custom workflows
272
+
273
+ ## Requirements
274
+ - Python 3.10+
275
+ - Apache Spark 3.4+
276
+ - Microsoft Fabric, Azure Synapse Analytics, Databricks — or local/standalone Spark for development
277
+
278
+ ## Dependencies
279
+ This framework builds upon several excellent open source projects:
280
+
281
+ - **Apache Spark** - Unified analytics engine for large-scale data processing (Apache 2.0)
282
+ - **Delta Lake** - Storage framework for reliable data lakes (Apache 2.0)
283
+ - **injector** - Python dependency injection framework (BSD)
284
+ - **blinker** - Python signal/event framework for pub/sub (MIT)
285
+ - **dynaconf** - Configuration management for Python (MIT)
286
+ - **pytest** - Testing framework (MIT)
287
+
288
+ ### Platform SDKs (auto-detected, only one required):
289
+ - **notebookutils** / **mssparkutils** - Microsoft Fabric & Synapse (MIT)
290
+ - **dbutils** - Databricks utilities (Databricks)
291
+
292
+ ### Extensions (optional):
293
+ - **azure-monitor-opentelemetry** - Azure Monitor integration (MIT)
294
+ - **opentelemetry-api/sdk** - OpenTelemetry tracing (Apache 2.0)
295
+
296
+
297
+ ## Contributing
298
+ We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/sep/spark-kindling-framework/blob/main/CONTRIBUTING.md) for guidelines.
299
+
300
+ ## Support
301
+ This is open source software provided without warranty or guaranteed support.
302
+
303
+ **Commercial Support:** Professional support, training, and consulting services are available from Software Engineering Professionals, Inc. Contact us at [contact information](mailto:engineering@sep.com).
304
+
305
+ ## License
306
+
307
+ This project is licensed under the MIT License - see the [LICENSE](https://github.com/sep/spark-kindling-framework/blob/main/LICENSE) file for details.
308
+
309
+ ## Developed By
310
+ **Software Engineering Professionals, Inc.**
311
+ 16080 Westfield Blvd.
312
+ Carmel, IN 46033
313
+ [www.sep.com](https://www.sep.com)
314
+
315
+ ## Acknowledgments
316
+ This framework was developed to solve real-world data processing challenges encountered across multiple enterprise engagements. We're grateful to our clients who have helped shape the requirements and validate the approach.
317
+
318
+ ---
319
+ **Note:** This framework is maintained by SEP and used across multiple projects. If you're using this framework and encounter issues or have suggestions, please open an issue or submit a pull request.
@@ -0,0 +1,265 @@
1
+ <!-- polecat smoke test: workflow verified ok -->
2
+ # Spark Kindling Framework
3
+
4
+ **Import name:** `kindling`
5
+ **Platforms:** Microsoft Fabric, Azure Synapse Analytics, Databricks
6
+
7
+ ## Overview
8
+
9
+ Spark Kindling Framework is a comprehensive solution for building robust data pipelines on Apache Spark, specifically designed for cross-platform solutions, via notebook or python-first development. It provides a declarative, dependency-injection-driven approach to defining and executing data transformations while maintaining strong governance and enabling robust observability.
10
+
11
+ ### Key Capabilities
12
+
13
+ - **Multi-Platform Support** - Unified API across Fabric, Synapse, and Databricks (including UC shared/standard access mode clusters) plus local/standalone Spark
14
+ - **Data Apps & Job Deployment** - Package and deploy apps as Spark jobs
15
+ - **Batch & Streaming Pipelines** - Incremental watermarked reads, streaming pipes with SCD1/SCD2 merge sinks
16
+ - **Pluggable Entity Providers** - Delta, Parquet, CSV, SQL views, in-memory, Event Hubs, and Azure Data Explorer out of the box
17
+ - **Hierarchical Configuration** - Platform, workspace, and environment-specific configs; execution options resolve config-first
18
+ - **Schema Reconciliation** - `kindling migrate` plans and applies drift between declared entities and live tables
19
+ - **Extensibility** - Plugin system for custom telemetry and integrations
20
+ - **Enterprise Observability** - Built-in logging, tracing, and Azure Monitor integration
21
+
22
+ ## Documentation
23
+
24
+ ### Core Features
25
+ - [Introduction](https://github.com/sep/spark-kindling-framework/blob/main/docs/intro.md) - Framework overview and architecture
26
+ - [Setup Guide](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/setup_guide.md) - Installation and configuration
27
+ - [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) - Scaffold, run, and test without cloud credentials
28
+ - [CLI Reference](https://github.com/sep/spark-kindling-framework/blob/main/docs/reference/cli_reference.md) - Every `kindling` command (scaffolding, apps, migrate, notebooks, workspace)
29
+ - [Data Entities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_entities.md) - Data entity management system
30
+ - [Entity Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/entity_configuration.md) - Tags-first declaration conventions
31
+ - [Data Pipes](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_pipes.md) - Transformation pipeline system
32
+ - [Derived Datasets](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/derived_datasets.md) - Replacement writes: derived datasets, slice replace, insert-only
33
+ - [Entity Providers](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/entity_providers.md) - Storage abstraction system
34
+ - [Migrating from runMultiple](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/migrating_from_runmultiple.md) - Move Synapse/Fabric notebook DAGs to Kindling pipes
35
+
36
+ ### Advanced Features
37
+ - [Job Deployment](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/job_deployment.md) - Deploy apps as Spark jobs
38
+ - [Hierarchical Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_workspace_config.md) - Multi-level YAML config system
39
+ - [Logging & Tracing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/logging_tracing.md) - Observability foundation (including JVM-free telemetry for UC shared clusters)
40
+ - [Watermarking](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/watermarking.md) - Change tracking and incremental processing
41
+ - [File Ingestion](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/file_ingestion.md) - Built-in file ingestion capabilities
42
+ - [Stage Processing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/stage_processing.md) - Pipeline stage orchestration
43
+ - [Temporal End-to-End](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/temporal_end_to_end.md) - Events, conditions, and episodes (spark-kindling-ext-temporal)
44
+ - [Lakeflow App Selection](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/lakeflow_app_selection.md) - Run Kindling data apps inside Databricks Lakeflow pipelines
45
+ - [Dynamic Registration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/dynamic_registration.md) - Register entities and pipes at runtime
46
+
47
+ ### Platform & Development
48
+ - [Platform API Architecture](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_api_architecture.md) - Multi-platform abstraction
49
+ - [Platform Storage Utils](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_storage_utils.md) - Storage operations
50
+ - [Utilities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/utilities.md) - Common utilities and helper functions
51
+ - [Build System](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/build_system.md) - How `poe build` produces the runtime, CLI, SDK and extension wheels
52
+ - [CI/CD Setup](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/ci_cd_setup.md) - Continuous integration and deployment
53
+ - [Testing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/testing.md) - Unit, integration, and system testing
54
+
55
+ ## Core Modules
56
+
57
+ The framework consists of several modular components:
58
+
59
+ - **Dependency Injection Engine** - Provides IoC container for loose coupling
60
+ - **Data Entities** - Entity registry and storage abstraction
61
+ - **Data Pipes** - Transformation pipeline definition and execution
62
+ - **Data Apps** - Package and deploy apps as .kda archives
63
+ - **Job Deployment** - Deploy apps as Spark jobs across platforms
64
+ - **Configuration System** - Hierarchical YAML configuration (platform/workspace/environment)
65
+ - **Platform Services** - Unified abstraction for Fabric/Synapse/Databricks
66
+ - **Watermarking** - Change tracking for incremental processing
67
+ - **Streaming Orchestration** - Streaming queries with health monitoring, recovery, and merge sinks
68
+ - **Migration** - Plan/apply reconciliation of declared entities against live Delta tables and views
69
+ - **File Ingestion** - File pattern discovery and loading
70
+ - **Stage Processing** - Orchestration of multi-stage pipelines
71
+ - **Common Transforms** - Reusable data transformation utilities
72
+ - **Logging & Tracing** - Comprehensive observability features
73
+
74
+ ## Extensions
75
+
76
+ - **[spark-kindling-ext-otel-azure](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_otel_azure)** - Azure Monitor OpenTelemetry integration
77
+ - **[spark-kindling-ext-sdp](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_sdp)** - Spark Declarative Pipelines (SDP) declaration engine
78
+ - **[spark-kindling-ext-databricks](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks)** - Databricks Lakeflow adapter for the SDP declaration engine
79
+ - **[spark-kindling-ext-databricks-autoloader](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks_autoloader)** - Auto Loader (`cloudFiles`) discovery for file ingestion
80
+ - **[spark-kindling-ext-temporal](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_temporal)** - Temporal event, condition, and episode primitives
81
+ - **[spark-kindling-ext-adx](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_adx)** - Azure Data Explorer entity provider (Kusto Spark connector; an API-based `adx-api` provider ships in core)
82
+ - **[spark-kindling-ext-cosmos](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_cosmos)** - Azure Cosmos DB entity provider (idempotent upsert writes)
83
+ - **[spark-kindling-ext-visualization](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_visualization)** - Matplotlib visualization helpers
84
+
85
+ In a Kindling project, add any extension with `kindling env add <name>`. The Databricks, SDP, Cosmos DB, Temporal and OpenTelemetry Azure extensions are also on PyPI (`pip install <name>`). The ADX, Auto Loader and visualization extensions are not; they are attached to each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) as wheels, which `kindling env add` pins by URL (or `pip install` the wheel's release URL).
86
+
87
+ ## Install
88
+
89
+ Kindling is on [PyPI](https://pypi.org/project/spark-kindling/). One distribution, platform-specific extras:
90
+
91
+ ```bash
92
+ pip install 'spark-kindling[synapse]' # Azure Synapse Analytics
93
+ pip install 'spark-kindling[databricks]' # Databricks
94
+ pip install 'spark-kindling[fabric]' # Microsoft Fabric
95
+ pip install 'spark-kindling[standalone]' # Local development / generic Spark
96
+ ```
97
+
98
+ The Python import name is `kindling` (unchanged):
99
+
100
+ ```python
101
+ from kindling.data_entities import DataEntities
102
+ ```
103
+
104
+ Design-time tooling ships separately:
105
+
106
+ ```bash
107
+ # `kindling` CLI for scaffolding and deploy; it pulls in spark-kindling-sdk (programmatic access to platform APIs)
108
+ pip install spark-kindling-cli
109
+ ```
110
+
111
+ The runtime, CLI and SDK are versioned together: pin all three to one release (`spark-kindling[...]==0.14.0 spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`; the CLI alone only requires a minimum SDK). Extensions keep their own versions; each GitHub release lists the extension versions that go with it, and `kindling env add` picks them. Projects scaffolded with `kindling repo init` / `kindling package init` pin these versions in `pyproject.toml` for you, and `kindling env update` moves them to a newer release.
112
+
113
+ ### Without PyPI access
114
+
115
+ Each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) also attaches the wheels, so environments that cannot reach PyPI (and releases before 0.14.0, which exist only on GitHub) can install by URL:
116
+
117
+ ```bash
118
+ V=0.14.0 # any release, without the leading v
119
+ BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
120
+ pip install "spark-kindling[databricks] @ $BASE/spark_kindling-$V-py3-none-any.whl"
121
+ pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" "$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
122
+ ```
123
+
124
+ See [docs/release_process.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/release_process.md) for how releases are published and [docs/developer_workflow.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/developer_workflow.md) for local development.
125
+
126
+ ## CLI Quick Start
127
+
128
+ Install the CLI (`pip install spark-kindling-cli`; you also need [uv](https://docs.astral.sh/uv/)) and scaffold a repo, package, and app explicitly:
129
+
130
+ ```bash
131
+ kindling repo init my-app --output-dir ./my_app
132
+ cd my_app
133
+ kindling package init my-app
134
+ kindling app init my-app --package my-app
135
+ kindling env bootstrap # pins Kindling at the root and syncs one repo-wide .venv/
136
+ uv run kindling app run my_app --env local
137
+ ```
138
+
139
+ Or open the generated repo in its devcontainer (`.devcontainer/devcontainer.json`), which runs `kindling env bootstrap` for you.
140
+ No Azure credentials needed — the scaffold uses in-memory entity providers by default.
141
+ See [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) for the full local workflow.
142
+
143
+ ## Notebook Quick Start
144
+
145
+ ```python
146
+ # Import Kindling framework
147
+ from kindling.data_entities import *
148
+ from kindling.data_pipes import *
149
+ from kindling.injection import get_kindling_service
150
+
151
+ # Define data entity (a plain call — registration happens immediately)
152
+ DataEntities.entity(
153
+ entityid="customers.raw",
154
+ name="Raw Customer Data",
155
+ partition_columns=["country"],
156
+ merge_columns=["customer_id"],
157
+ tags={"domain": "customer", "layer": "bronze"},
158
+ schema=customer_schema,
159
+ )
160
+
161
+ # Create data transformation pipe
162
+ @DataPipes.pipe(
163
+ pipeid="customers.transform",
164
+ name="Transform Customers",
165
+ tags={"layer": "silver"},
166
+ input_entity_ids=["customers.raw"],
167
+ output_entity_id="customers.silver",
168
+ output_type="table"
169
+ )
170
+ def transform_customers(customers_raw):
171
+ # Transformation logic here
172
+ return customers_raw.filter(...)
173
+
174
+ # Execute pipeline
175
+ executer = get_kindling_service(DataPipesExecution)
176
+ executer.run_datapipes(["customers.transform"])
177
+ ```
178
+
179
+ ## Key Features
180
+
181
+ ### Core Framework
182
+ - **Declarative definitions** for data entities and transformations (tags-first conventions)
183
+ - **Dependency injection** for loose coupling and testability
184
+ - **Delta Lake integration** for reliable storage and time travel
185
+ - **Built-in entity providers** - `delta`, `parquet`, `csv`, `memory`, `sql` (views), `current_view` (SCD2), `eventhub`, `adx-api`; ADX (connector) and Cosmos DB via extensions
186
+ - **Watermarking** for change tracking and incremental loads
187
+ - **Streaming pipes** with per-micro-batch SCD1/SCD2 merge sinks (`write.mode` tag)
188
+ - **Config-first execution options** - parallelism, error strategies (incl. `skip_dependents`), per-pipe retry via `kindling.execution.*`
189
+ - **Schema reconciliation** - `kindling migrate plan|apply` converges live tables to declared entities
190
+ - **Notebook round-tripping** - `kindling notebook list|pull|push` workspace notebooks as git-friendly `.py` source
191
+ - **Observability** through logging and tracing (JVM-free fallback on UC shared clusters)
192
+
193
+ ### Multi-Platform Support
194
+ - **Unified API** across Microsoft Fabric, Azure Synapse, and Databricks
195
+ - **Platform abstraction** - write once, deploy anywhere
196
+ - **Platform-specific optimizations** via configuration
197
+ - **Smart platform detection** - automatic platform identification
198
+
199
+ ### Data Apps & Deployment
200
+ - **Package apps** as .kda archives (Kindling Data Apps)
201
+ - **Run through the durable runner** - apps submit to one managed platform runner
202
+ - **Cross-platform deployment** - same app runs on all platforms
203
+ - **Artifact management** - wheels, configs, and dependencies
204
+
205
+ ### Configuration System
206
+ - **Hierarchical YAML configs** - settings, platform, workspace, environment layers
207
+ - **Auto-detection** - platform and workspace ID discovery
208
+ - **Flexible overrides** - bootstrap config for runtime changes
209
+ - **Multi-team support** - workspace-specific configurations
210
+
211
+ ### Extensibility
212
+ - **Extension system** - load custom packages via configuration
213
+ - **Azure Monitor integration** - via spark-kindling-ext-otel-azure extension
214
+ - **Declarative engines** - SDP declaration engine with a Databricks Lakeflow adapter (spark-kindling-ext-sdp / spark-kindling-ext-databricks)
215
+ - **Temporal primitives** - events, conditions, and episode lifecycle (spark-kindling-ext-temporal)
216
+ - **Custom providers** - implement your own storage backends
217
+ - **Signal/event system** - blinker-based pub/sub for custom workflows
218
+
219
+ ## Requirements
220
+ - Python 3.10+
221
+ - Apache Spark 3.4+
222
+ - Microsoft Fabric, Azure Synapse Analytics, Databricks — or local/standalone Spark for development
223
+
224
+ ## Dependencies
225
+ This framework builds upon several excellent open source projects:
226
+
227
+ - **Apache Spark** - Unified analytics engine for large-scale data processing (Apache 2.0)
228
+ - **Delta Lake** - Storage framework for reliable data lakes (Apache 2.0)
229
+ - **injector** - Python dependency injection framework (BSD)
230
+ - **blinker** - Python signal/event framework for pub/sub (MIT)
231
+ - **dynaconf** - Configuration management for Python (MIT)
232
+ - **pytest** - Testing framework (MIT)
233
+
234
+ ### Platform SDKs (auto-detected, only one required):
235
+ - **notebookutils** / **mssparkutils** - Microsoft Fabric & Synapse (MIT)
236
+ - **dbutils** - Databricks utilities (Databricks)
237
+
238
+ ### Extensions (optional):
239
+ - **azure-monitor-opentelemetry** - Azure Monitor integration (MIT)
240
+ - **opentelemetry-api/sdk** - OpenTelemetry tracing (Apache 2.0)
241
+
242
+
243
+ ## Contributing
244
+ We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/sep/spark-kindling-framework/blob/main/CONTRIBUTING.md) for guidelines.
245
+
246
+ ## Support
247
+ This is open source software provided without warranty or guaranteed support.
248
+
249
+ **Commercial Support:** Professional support, training, and consulting services are available from Software Engineering Professionals, Inc. Contact us at [contact information](mailto:engineering@sep.com).
250
+
251
+ ## License
252
+
253
+ This project is licensed under the MIT License - see the [LICENSE](https://github.com/sep/spark-kindling-framework/blob/main/LICENSE) file for details.
254
+
255
+ ## Developed By
256
+ **Software Engineering Professionals, Inc.**
257
+ 16080 Westfield Blvd.
258
+ Carmel, IN 46033
259
+ [www.sep.com](https://www.sep.com)
260
+
261
+ ## Acknowledgments
262
+ This framework was developed to solve real-world data processing challenges encountered across multiple enterprise engagements. We're grateful to our clients who have helped shape the requirements and validate the approach.
263
+
264
+ ---
265
+ **Note:** This framework is maintained by SEP and used across multiple projects. If you're using this framework and encounter issues or have suggestions, please open an issue or submit a pull request.