spark-kindling 0.14.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- spark_kindling-0.14.0/LICENSE +21 -0
- spark_kindling-0.14.0/PKG-INFO +319 -0
- spark_kindling-0.14.0/README.md +265 -0
- spark_kindling-0.14.0/packages/kindling/__init__.py +174 -0
- spark_kindling-0.14.0/packages/kindling/app_files.py +60 -0
- spark_kindling-0.14.0/packages/kindling/apps.py +59 -0
- spark_kindling-0.14.0/packages/kindling/bootstrap.py +3108 -0
- spark_kindling-0.14.0/packages/kindling/cache_optimizer.py +94 -0
- spark_kindling-0.14.0/packages/kindling/common_transforms.py +28 -0
- spark_kindling-0.14.0/packages/kindling/config_loaders.py +380 -0
- spark_kindling-0.14.0/packages/kindling/config_patterns.py +322 -0
- spark_kindling-0.14.0/packages/kindling/data_apps.py +1427 -0
- spark_kindling-0.14.0/packages/kindling/data_entities.py +2061 -0
- spark_kindling-0.14.0/packages/kindling/data_pipes.py +1075 -0
- spark_kindling-0.14.0/packages/kindling/declaration_derivations.py +437 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider.py +495 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_adx.py +537 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_csv.py +380 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_current_view.py +81 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_delta.py +2646 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_eventhub.py +979 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_eventhub_declaration.py +246 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_memory.py +873 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_parquet.py +243 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_registry.py +266 -0
- spark_kindling-0.14.0/packages/kindling/entity_provider_sql.py +221 -0
- spark_kindling-0.14.0/packages/kindling/entity_resolution.py +354 -0
- spark_kindling-0.14.0/packages/kindling/execution_orchestrator.py +148 -0
- spark_kindling-0.14.0/packages/kindling/execution_strategy.py +675 -0
- spark_kindling-0.14.0/packages/kindling/features.py +224 -0
- spark_kindling-0.14.0/packages/kindling/file_ingestion.py +778 -0
- spark_kindling-0.14.0/packages/kindling/generation_executor.py +1299 -0
- spark_kindling-0.14.0/packages/kindling/injection.py +121 -0
- spark_kindling-0.14.0/packages/kindling/job_deployment.py +545 -0
- spark_kindling-0.14.0/packages/kindling/migration.py +1051 -0
- spark_kindling-0.14.0/packages/kindling/notebook_framework.py +1974 -0
- spark_kindling-0.14.0/packages/kindling/notebook_source.py +125 -0
- spark_kindling-0.14.0/packages/kindling/package_registrations.py +60 -0
- spark_kindling-0.14.0/packages/kindling/pip_manager.py +334 -0
- spark_kindling-0.14.0/packages/kindling/pipe_graph.py +569 -0
- spark_kindling-0.14.0/packages/kindling/pipe_streaming.py +258 -0
- spark_kindling-0.14.0/packages/kindling/plain_telemetry.py +274 -0
- spark_kindling-0.14.0/packages/kindling/platform_databricks.py +1293 -0
- spark_kindling-0.14.0/packages/kindling/platform_fabric.py +1142 -0
- spark_kindling-0.14.0/packages/kindling/platform_provider.py +177 -0
- spark_kindling-0.14.0/packages/kindling/platform_standalone.py +807 -0
- spark_kindling-0.14.0/packages/kindling/platform_synapse.py +1145 -0
- spark_kindling-0.14.0/packages/kindling/sentinels.py +22 -0
- spark_kindling-0.14.0/packages/kindling/signaling.py +620 -0
- spark_kindling-0.14.0/packages/kindling/simple_read_persist_strategy.py +467 -0
- spark_kindling-0.14.0/packages/kindling/simple_stage_processor.py +128 -0
- spark_kindling-0.14.0/packages/kindling/spark_config.py +954 -0
- spark_kindling-0.14.0/packages/kindling/spark_jdbc.py +197 -0
- spark_kindling-0.14.0/packages/kindling/spark_log.py +156 -0
- spark_kindling-0.14.0/packages/kindling/spark_log_provider.py +23 -0
- spark_kindling-0.14.0/packages/kindling/spark_session.py +131 -0
- spark_kindling-0.14.0/packages/kindling/spark_trace.py +558 -0
- spark_kindling-0.14.0/packages/kindling/streaming_health_monitor.py +593 -0
- spark_kindling-0.14.0/packages/kindling/streaming_listener.py +546 -0
- spark_kindling-0.14.0/packages/kindling/streaming_orchestrator.py +428 -0
- spark_kindling-0.14.0/packages/kindling/streaming_query_manager.py +589 -0
- spark_kindling-0.14.0/packages/kindling/streaming_recovery_manager.py +737 -0
- spark_kindling-0.14.0/packages/kindling/surrogate_keys.py +887 -0
- spark_kindling-0.14.0/packages/kindling/test_framework.py +1438 -0
- spark_kindling-0.14.0/packages/kindling/trace_ops.py +265 -0
- spark_kindling-0.14.0/packages/kindling/watermarking.py +684 -0
- spark_kindling-0.14.0/pyproject.toml +525 -0
- spark_kindling-0.14.0/pyproject.toml.orig +264 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Software Engineering Professionals, Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: spark-kindling
|
|
3
|
+
Version: 0.14.0
|
|
4
|
+
Summary: A unified framework for data apps across Fabric, Synapse, and Databricks
|
|
5
|
+
Keywords: spark,pyspark,databricks,microsoft-fabric,azure-synapse,delta-lake,data-engineering
|
|
6
|
+
Author: SEP Engineering
|
|
7
|
+
Author-email: SEP Engineering <engineering@sep.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Topic :: Database
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Requires-Dist: injector>=0.20.1
|
|
18
|
+
Requires-Dist: dynaconf>=3.1.0,!=3.3.4,<3.3.6
|
|
19
|
+
Requires-Dist: pyyaml>=6.0
|
|
20
|
+
Requires-Dist: packaging>=23.0
|
|
21
|
+
Requires-Dist: blinker>=1.6.0
|
|
22
|
+
Requires-Dist: azure-core>=1.30.0
|
|
23
|
+
Requires-Dist: azure-identity>=1.12.0
|
|
24
|
+
Requires-Dist: azure-storage-file-datalake>=12.0.0
|
|
25
|
+
Requires-Dist: azure-kusto-data>=4.0.0 ; extra == 'adx'
|
|
26
|
+
Requires-Dist: azure-kusto-ingest>=4.0.0 ; extra == 'adx'
|
|
27
|
+
Requires-Dist: azure-synapse-artifacts>=0.17.0 ; extra == 'all'
|
|
28
|
+
Requires-Dist: databricks-sdk>=0.12.0 ; extra == 'all'
|
|
29
|
+
Requires-Dist: pyspark>=3.4.0,<4.0.0 ; extra == 'all'
|
|
30
|
+
Requires-Dist: delta-spark>=2.4.0,<4.0.0 ; extra == 'all'
|
|
31
|
+
Requires-Dist: pandas>=2.0.0 ; extra == 'all'
|
|
32
|
+
Requires-Dist: pyarrow>=12.0.0 ; extra == 'all'
|
|
33
|
+
Requires-Dist: azure-kusto-data>=4.0.0 ; extra == 'all'
|
|
34
|
+
Requires-Dist: azure-kusto-ingest>=4.0.0 ; extra == 'all'
|
|
35
|
+
Requires-Dist: databricks-sdk>=0.12.0 ; extra == 'databricks'
|
|
36
|
+
Requires-Dist: azure-core>=1.30.0,<1.31.0 ; extra == 'fabric'
|
|
37
|
+
Requires-Dist: pyspark>=3.4.0,<4.0.0 ; extra == 'standalone'
|
|
38
|
+
Requires-Dist: delta-spark>=2.4.0,<4.0.0 ; extra == 'standalone'
|
|
39
|
+
Requires-Dist: pandas>=2.0.0 ; extra == 'standalone'
|
|
40
|
+
Requires-Dist: pyarrow>=12.0.0 ; extra == 'standalone'
|
|
41
|
+
Requires-Dist: azure-synapse-artifacts>=0.17.0 ; extra == 'synapse'
|
|
42
|
+
Requires-Python: >=3.10
|
|
43
|
+
Project-URL: Homepage, https://github.com/sep/spark-kindling-framework
|
|
44
|
+
Project-URL: Documentation, https://github.com/sep/spark-kindling-framework/tree/main/docs
|
|
45
|
+
Project-URL: Changelog, https://github.com/sep/spark-kindling-framework/blob/main/CHANGELOG.md
|
|
46
|
+
Project-URL: Issues, https://github.com/sep/spark-kindling-framework/issues
|
|
47
|
+
Provides-Extra: adx
|
|
48
|
+
Provides-Extra: all
|
|
49
|
+
Provides-Extra: databricks
|
|
50
|
+
Provides-Extra: fabric
|
|
51
|
+
Provides-Extra: standalone
|
|
52
|
+
Provides-Extra: synapse
|
|
53
|
+
Description-Content-Type: text/markdown
|
|
54
|
+
|
|
55
|
+
<!-- polecat smoke test: workflow verified ok -->
|
|
56
|
+
# Spark Kindling Framework
|
|
57
|
+
|
|
58
|
+
**Import name:** `kindling`
|
|
59
|
+
**Platforms:** Microsoft Fabric, Azure Synapse Analytics, Databricks
|
|
60
|
+
|
|
61
|
+
## Overview
|
|
62
|
+
|
|
63
|
+
Spark Kindling Framework is a comprehensive solution for building robust data pipelines on Apache Spark, specifically designed for cross-platform solutions, via notebook or python-first development. It provides a declarative, dependency-injection-driven approach to defining and executing data transformations while maintaining strong governance and enabling robust observability.
|
|
64
|
+
|
|
65
|
+
### Key Capabilities
|
|
66
|
+
|
|
67
|
+
- **Multi-Platform Support** - Unified API across Fabric, Synapse, and Databricks (including UC shared/standard access mode clusters) plus local/standalone Spark
|
|
68
|
+
- **Data Apps & Job Deployment** - Package and deploy apps as Spark jobs
|
|
69
|
+
- **Batch & Streaming Pipelines** - Incremental watermarked reads, streaming pipes with SCD1/SCD2 merge sinks
|
|
70
|
+
- **Pluggable Entity Providers** - Delta, Parquet, CSV, SQL views, in-memory, Event Hubs, and Azure Data Explorer out of the box
|
|
71
|
+
- **Hierarchical Configuration** - Platform, workspace, and environment-specific configs; execution options resolve config-first
|
|
72
|
+
- **Schema Reconciliation** - `kindling migrate` plans and applies drift between declared entities and live tables
|
|
73
|
+
- **Extensibility** - Plugin system for custom telemetry and integrations
|
|
74
|
+
- **Enterprise Observability** - Built-in logging, tracing, and Azure Monitor integration
|
|
75
|
+
|
|
76
|
+
## Documentation
|
|
77
|
+
|
|
78
|
+
### Core Features
|
|
79
|
+
- [Introduction](https://github.com/sep/spark-kindling-framework/blob/main/docs/intro.md) - Framework overview and architecture
|
|
80
|
+
- [Setup Guide](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/setup_guide.md) - Installation and configuration
|
|
81
|
+
- [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) - Scaffold, run, and test without cloud credentials
|
|
82
|
+
- [CLI Reference](https://github.com/sep/spark-kindling-framework/blob/main/docs/reference/cli_reference.md) - Every `kindling` command (scaffolding, apps, migrate, notebooks, workspace)
|
|
83
|
+
- [Data Entities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_entities.md) - Data entity management system
|
|
84
|
+
- [Entity Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/entity_configuration.md) - Tags-first declaration conventions
|
|
85
|
+
- [Data Pipes](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_pipes.md) - Transformation pipeline system
|
|
86
|
+
- [Derived Datasets](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/derived_datasets.md) - Replacement writes: derived datasets, slice replace, insert-only
|
|
87
|
+
- [Entity Providers](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/entity_providers.md) - Storage abstraction system
|
|
88
|
+
- [Migrating from runMultiple](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/migrating_from_runmultiple.md) - Move Synapse/Fabric notebook DAGs to Kindling pipes
|
|
89
|
+
|
|
90
|
+
### Advanced Features
|
|
91
|
+
- [Job Deployment](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/job_deployment.md) - Deploy apps as Spark jobs
|
|
92
|
+
- [Hierarchical Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_workspace_config.md) - Multi-level YAML config system
|
|
93
|
+
- [Logging & Tracing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/logging_tracing.md) - Observability foundation (including JVM-free telemetry for UC shared clusters)
|
|
94
|
+
- [Watermarking](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/watermarking.md) - Change tracking and incremental processing
|
|
95
|
+
- [File Ingestion](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/file_ingestion.md) - Built-in file ingestion capabilities
|
|
96
|
+
- [Stage Processing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/stage_processing.md) - Pipeline stage orchestration
|
|
97
|
+
- [Temporal End-to-End](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/temporal_end_to_end.md) - Events, conditions, and episodes (spark-kindling-ext-temporal)
|
|
98
|
+
- [Lakeflow App Selection](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/lakeflow_app_selection.md) - Run Kindling data apps inside Databricks Lakeflow pipelines
|
|
99
|
+
- [Dynamic Registration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/dynamic_registration.md) - Register entities and pipes at runtime
|
|
100
|
+
|
|
101
|
+
### Platform & Development
|
|
102
|
+
- [Platform API Architecture](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_api_architecture.md) - Multi-platform abstraction
|
|
103
|
+
- [Platform Storage Utils](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_storage_utils.md) - Storage operations
|
|
104
|
+
- [Utilities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/utilities.md) - Common utilities and helper functions
|
|
105
|
+
- [Build System](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/build_system.md) - How `poe build` produces the runtime, CLI, SDK and extension wheels
|
|
106
|
+
- [CI/CD Setup](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/ci_cd_setup.md) - Continuous integration and deployment
|
|
107
|
+
- [Testing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/testing.md) - Unit, integration, and system testing
|
|
108
|
+
|
|
109
|
+
## Core Modules
|
|
110
|
+
|
|
111
|
+
The framework consists of several modular components:
|
|
112
|
+
|
|
113
|
+
- **Dependency Injection Engine** - Provides IoC container for loose coupling
|
|
114
|
+
- **Data Entities** - Entity registry and storage abstraction
|
|
115
|
+
- **Data Pipes** - Transformation pipeline definition and execution
|
|
116
|
+
- **Data Apps** - Package and deploy apps as .kda archives
|
|
117
|
+
- **Job Deployment** - Deploy apps as Spark jobs across platforms
|
|
118
|
+
- **Configuration System** - Hierarchical YAML configuration (platform/workspace/environment)
|
|
119
|
+
- **Platform Services** - Unified abstraction for Fabric/Synapse/Databricks
|
|
120
|
+
- **Watermarking** - Change tracking for incremental processing
|
|
121
|
+
- **Streaming Orchestration** - Streaming queries with health monitoring, recovery, and merge sinks
|
|
122
|
+
- **Migration** - Plan/apply reconciliation of declared entities against live Delta tables and views
|
|
123
|
+
- **File Ingestion** - File pattern discovery and loading
|
|
124
|
+
- **Stage Processing** - Orchestration of multi-stage pipelines
|
|
125
|
+
- **Common Transforms** - Reusable data transformation utilities
|
|
126
|
+
- **Logging & Tracing** - Comprehensive observability features
|
|
127
|
+
|
|
128
|
+
## Extensions
|
|
129
|
+
|
|
130
|
+
- **[spark-kindling-ext-otel-azure](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_otel_azure)** - Azure Monitor OpenTelemetry integration
|
|
131
|
+
- **[spark-kindling-ext-sdp](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_sdp)** - Spark Declarative Pipelines (SDP) declaration engine
|
|
132
|
+
- **[spark-kindling-ext-databricks](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks)** - Databricks Lakeflow adapter for the SDP declaration engine
|
|
133
|
+
- **[spark-kindling-ext-databricks-autoloader](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks_autoloader)** - Auto Loader (`cloudFiles`) discovery for file ingestion
|
|
134
|
+
- **[spark-kindling-ext-temporal](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_temporal)** - Temporal event, condition, and episode primitives
|
|
135
|
+
- **[spark-kindling-ext-adx](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_adx)** - Azure Data Explorer entity provider (Kusto Spark connector; an API-based `adx-api` provider ships in core)
|
|
136
|
+
- **[spark-kindling-ext-cosmos](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_cosmos)** - Azure Cosmos DB entity provider (idempotent upsert writes)
|
|
137
|
+
- **[spark-kindling-ext-visualization](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_visualization)** - Matplotlib visualization helpers
|
|
138
|
+
|
|
139
|
+
In a Kindling project, add any extension with `kindling env add <name>`. The Databricks, SDP, Cosmos DB, Temporal and OpenTelemetry Azure extensions are also on PyPI (`pip install <name>`). The ADX, Auto Loader and visualization extensions are not; they are attached to each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) as wheels, which `kindling env add` pins by URL (or `pip install` the wheel's release URL).
|
|
140
|
+
|
|
141
|
+
## Install
|
|
142
|
+
|
|
143
|
+
Kindling is on [PyPI](https://pypi.org/project/spark-kindling/). One distribution, platform-specific extras:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
pip install 'spark-kindling[synapse]' # Azure Synapse Analytics
|
|
147
|
+
pip install 'spark-kindling[databricks]' # Databricks
|
|
148
|
+
pip install 'spark-kindling[fabric]' # Microsoft Fabric
|
|
149
|
+
pip install 'spark-kindling[standalone]' # Local development / generic Spark
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
The Python import name is `kindling` (unchanged):
|
|
153
|
+
|
|
154
|
+
```python
|
|
155
|
+
from kindling.data_entities import DataEntities
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
Design-time tooling ships separately:
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
# `kindling` CLI for scaffolding and deploy; it pulls in spark-kindling-sdk (programmatic access to platform APIs)
|
|
162
|
+
pip install spark-kindling-cli
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
The runtime, CLI and SDK are versioned together: pin all three to one release (`spark-kindling[...]==0.14.0 spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`; the CLI alone only requires a minimum SDK). Extensions keep their own versions; each GitHub release lists the extension versions that go with it, and `kindling env add` picks them. Projects scaffolded with `kindling repo init` / `kindling package init` pin these versions in `pyproject.toml` for you, and `kindling env update` moves them to a newer release.
|
|
166
|
+
|
|
167
|
+
### Without PyPI access
|
|
168
|
+
|
|
169
|
+
Each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) also attaches the wheels, so environments that cannot reach PyPI (and releases before 0.14.0, which exist only on GitHub) can install by URL:
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
V=0.14.0 # any release, without the leading v
|
|
173
|
+
BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
|
|
174
|
+
pip install "spark-kindling[databricks] @ $BASE/spark_kindling-$V-py3-none-any.whl"
|
|
175
|
+
pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" "$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
See [docs/release_process.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/release_process.md) for how releases are published and [docs/developer_workflow.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/developer_workflow.md) for local development.
|
|
179
|
+
|
|
180
|
+
## CLI Quick Start
|
|
181
|
+
|
|
182
|
+
Install the CLI (`pip install spark-kindling-cli`; you also need [uv](https://docs.astral.sh/uv/)) and scaffold a repo, package, and app explicitly:
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
kindling repo init my-app --output-dir ./my_app
|
|
186
|
+
cd my_app
|
|
187
|
+
kindling package init my-app
|
|
188
|
+
kindling app init my-app --package my-app
|
|
189
|
+
kindling env bootstrap # pins Kindling at the root and syncs one repo-wide .venv/
|
|
190
|
+
uv run kindling app run my_app --env local
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
Or open the generated repo in its devcontainer (`.devcontainer/devcontainer.json`), which runs `kindling env bootstrap` for you.
|
|
194
|
+
No Azure credentials needed — the scaffold uses in-memory entity providers by default.
|
|
195
|
+
See [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) for the full local workflow.
|
|
196
|
+
|
|
197
|
+
## Notebook Quick Start
|
|
198
|
+
|
|
199
|
+
```python
|
|
200
|
+
# Import Kindling framework
|
|
201
|
+
from kindling.data_entities import *
|
|
202
|
+
from kindling.data_pipes import *
|
|
203
|
+
from kindling.injection import get_kindling_service
|
|
204
|
+
|
|
205
|
+
# Define data entity (a plain call — registration happens immediately)
|
|
206
|
+
DataEntities.entity(
|
|
207
|
+
entityid="customers.raw",
|
|
208
|
+
name="Raw Customer Data",
|
|
209
|
+
partition_columns=["country"],
|
|
210
|
+
merge_columns=["customer_id"],
|
|
211
|
+
tags={"domain": "customer", "layer": "bronze"},
|
|
212
|
+
schema=customer_schema,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
# Create data transformation pipe
|
|
216
|
+
@DataPipes.pipe(
|
|
217
|
+
pipeid="customers.transform",
|
|
218
|
+
name="Transform Customers",
|
|
219
|
+
tags={"layer": "silver"},
|
|
220
|
+
input_entity_ids=["customers.raw"],
|
|
221
|
+
output_entity_id="customers.silver",
|
|
222
|
+
output_type="table"
|
|
223
|
+
)
|
|
224
|
+
def transform_customers(customers_raw):
|
|
225
|
+
# Transformation logic here
|
|
226
|
+
return customers_raw.filter(...)
|
|
227
|
+
|
|
228
|
+
# Execute pipeline
|
|
229
|
+
executer = get_kindling_service(DataPipesExecution)
|
|
230
|
+
executer.run_datapipes(["customers.transform"])
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
## Key Features
|
|
234
|
+
|
|
235
|
+
### Core Framework
|
|
236
|
+
- **Declarative definitions** for data entities and transformations (tags-first conventions)
|
|
237
|
+
- **Dependency injection** for loose coupling and testability
|
|
238
|
+
- **Delta Lake integration** for reliable storage and time travel
|
|
239
|
+
- **Built-in entity providers** - `delta`, `parquet`, `csv`, `memory`, `sql` (views), `current_view` (SCD2), `eventhub`, `adx-api`; ADX (connector) and Cosmos DB via extensions
|
|
240
|
+
- **Watermarking** for change tracking and incremental loads
|
|
241
|
+
- **Streaming pipes** with per-micro-batch SCD1/SCD2 merge sinks (`write.mode` tag)
|
|
242
|
+
- **Config-first execution options** - parallelism, error strategies (incl. `skip_dependents`), per-pipe retry via `kindling.execution.*`
|
|
243
|
+
- **Schema reconciliation** - `kindling migrate plan|apply` converges live tables to declared entities
|
|
244
|
+
- **Notebook round-tripping** - `kindling notebook list|pull|push` workspace notebooks as git-friendly `.py` source
|
|
245
|
+
- **Observability** through logging and tracing (JVM-free fallback on UC shared clusters)
|
|
246
|
+
|
|
247
|
+
### Multi-Platform Support
|
|
248
|
+
- **Unified API** across Microsoft Fabric, Azure Synapse, and Databricks
|
|
249
|
+
- **Platform abstraction** - write once, deploy anywhere
|
|
250
|
+
- **Platform-specific optimizations** via configuration
|
|
251
|
+
- **Smart platform detection** - automatic platform identification
|
|
252
|
+
|
|
253
|
+
### Data Apps & Deployment
|
|
254
|
+
- **Package apps** as .kda archives (Kindling Data Apps)
|
|
255
|
+
- **Run through the durable runner** - apps submit to one managed platform runner
|
|
256
|
+
- **Cross-platform deployment** - same app runs on all platforms
|
|
257
|
+
- **Artifact management** - wheels, configs, and dependencies
|
|
258
|
+
|
|
259
|
+
### Configuration System
|
|
260
|
+
- **Hierarchical YAML configs** - settings, platform, workspace, environment layers
|
|
261
|
+
- **Auto-detection** - platform and workspace ID discovery
|
|
262
|
+
- **Flexible overrides** - bootstrap config for runtime changes
|
|
263
|
+
- **Multi-team support** - workspace-specific configurations
|
|
264
|
+
|
|
265
|
+
### Extensibility
|
|
266
|
+
- **Extension system** - load custom packages via configuration
|
|
267
|
+
- **Azure Monitor integration** - via spark-kindling-ext-otel-azure extension
|
|
268
|
+
- **Declarative engines** - SDP declaration engine with a Databricks Lakeflow adapter (spark-kindling-ext-sdp / spark-kindling-ext-databricks)
|
|
269
|
+
- **Temporal primitives** - events, conditions, and episode lifecycle (spark-kindling-ext-temporal)
|
|
270
|
+
- **Custom providers** - implement your own storage backends
|
|
271
|
+
- **Signal/event system** - blinker-based pub/sub for custom workflows
|
|
272
|
+
|
|
273
|
+
## Requirements
|
|
274
|
+
- Python 3.10+
|
|
275
|
+
- Apache Spark 3.4+
|
|
276
|
+
- Microsoft Fabric, Azure Synapse Analytics, Databricks — or local/standalone Spark for development
|
|
277
|
+
|
|
278
|
+
## Dependencies
|
|
279
|
+
This framework builds upon several excellent open source projects:
|
|
280
|
+
|
|
281
|
+
- **Apache Spark** - Unified analytics engine for large-scale data processing (Apache 2.0)
|
|
282
|
+
- **Delta Lake** - Storage framework for reliable data lakes (Apache 2.0)
|
|
283
|
+
- **injector** - Python dependency injection framework (BSD)
|
|
284
|
+
- **blinker** - Python signal/event framework for pub/sub (MIT)
|
|
285
|
+
- **dynaconf** - Configuration management for Python (MIT)
|
|
286
|
+
- **pytest** - Testing framework (MIT)
|
|
287
|
+
|
|
288
|
+
### Platform SDKs (auto-detected, only one required):
|
|
289
|
+
- **notebookutils** / **mssparkutils** - Microsoft Fabric & Synapse (MIT)
|
|
290
|
+
- **dbutils** - Databricks utilities (Databricks)
|
|
291
|
+
|
|
292
|
+
### Extensions (optional):
|
|
293
|
+
- **azure-monitor-opentelemetry** - Azure Monitor integration (MIT)
|
|
294
|
+
- **opentelemetry-api/sdk** - OpenTelemetry tracing (Apache 2.0)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
## Contributing
|
|
298
|
+
We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/sep/spark-kindling-framework/blob/main/CONTRIBUTING.md) for guidelines.
|
|
299
|
+
|
|
300
|
+
## Support
|
|
301
|
+
This is open source software provided without warranty or guaranteed support.
|
|
302
|
+
|
|
303
|
+
**Commercial Support:** Professional support, training, and consulting services are available from Software Engineering Professionals, Inc. Contact us at [contact information](mailto:engineering@sep.com).
|
|
304
|
+
|
|
305
|
+
## License
|
|
306
|
+
|
|
307
|
+
This project is licensed under the MIT License - see the [LICENSE](https://github.com/sep/spark-kindling-framework/blob/main/LICENSE) file for details.
|
|
308
|
+
|
|
309
|
+
## Developed By
|
|
310
|
+
**Software Engineering Professionals, Inc.**
|
|
311
|
+
16080 Westfield Blvd.
|
|
312
|
+
Carmel, IN 46033
|
|
313
|
+
[www.sep.com](https://www.sep.com)
|
|
314
|
+
|
|
315
|
+
## Acknowledgments
|
|
316
|
+
This framework was developed to solve real-world data processing challenges encountered across multiple enterprise engagements. We're grateful to our clients who have helped shape the requirements and validate the approach.
|
|
317
|
+
|
|
318
|
+
---
|
|
319
|
+
**Note:** This framework is maintained by SEP and used across multiple projects. If you're using this framework and encounter issues or have suggestions, please open an issue or submit a pull request.
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
<!-- polecat smoke test: workflow verified ok -->
|
|
2
|
+
# Spark Kindling Framework
|
|
3
|
+
|
|
4
|
+
**Import name:** `kindling`
|
|
5
|
+
**Platforms:** Microsoft Fabric, Azure Synapse Analytics, Databricks
|
|
6
|
+
|
|
7
|
+
## Overview
|
|
8
|
+
|
|
9
|
+
Spark Kindling Framework is a comprehensive solution for building robust data pipelines on Apache Spark, specifically designed for cross-platform solutions, via notebook or python-first development. It provides a declarative, dependency-injection-driven approach to defining and executing data transformations while maintaining strong governance and enabling robust observability.
|
|
10
|
+
|
|
11
|
+
### Key Capabilities
|
|
12
|
+
|
|
13
|
+
- **Multi-Platform Support** - Unified API across Fabric, Synapse, and Databricks (including UC shared/standard access mode clusters) plus local/standalone Spark
|
|
14
|
+
- **Data Apps & Job Deployment** - Package and deploy apps as Spark jobs
|
|
15
|
+
- **Batch & Streaming Pipelines** - Incremental watermarked reads, streaming pipes with SCD1/SCD2 merge sinks
|
|
16
|
+
- **Pluggable Entity Providers** - Delta, Parquet, CSV, SQL views, in-memory, Event Hubs, and Azure Data Explorer out of the box
|
|
17
|
+
- **Hierarchical Configuration** - Platform, workspace, and environment-specific configs; execution options resolve config-first
|
|
18
|
+
- **Schema Reconciliation** - `kindling migrate` plans and applies drift between declared entities and live tables
|
|
19
|
+
- **Extensibility** - Plugin system for custom telemetry and integrations
|
|
20
|
+
- **Enterprise Observability** - Built-in logging, tracing, and Azure Monitor integration
|
|
21
|
+
|
|
22
|
+
## Documentation
|
|
23
|
+
|
|
24
|
+
### Core Features
|
|
25
|
+
- [Introduction](https://github.com/sep/spark-kindling-framework/blob/main/docs/intro.md) - Framework overview and architecture
|
|
26
|
+
- [Setup Guide](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/setup_guide.md) - Installation and configuration
|
|
27
|
+
- [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) - Scaffold, run, and test without cloud credentials
|
|
28
|
+
- [CLI Reference](https://github.com/sep/spark-kindling-framework/blob/main/docs/reference/cli_reference.md) - Every `kindling` command (scaffolding, apps, migrate, notebooks, workspace)
|
|
29
|
+
- [Data Entities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_entities.md) - Data entity management system
|
|
30
|
+
- [Entity Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/entity_configuration.md) - Tags-first declaration conventions
|
|
31
|
+
- [Data Pipes](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/data_pipes.md) - Transformation pipeline system
|
|
32
|
+
- [Derived Datasets](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/derived_datasets.md) - Replacement writes: derived datasets, slice replace, insert-only
|
|
33
|
+
- [Entity Providers](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/entity_providers.md) - Storage abstraction system
|
|
34
|
+
- [Migrating from runMultiple](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/migrating_from_runmultiple.md) - Move Synapse/Fabric notebook DAGs to Kindling pipes
|
|
35
|
+
|
|
36
|
+
### Advanced Features
|
|
37
|
+
- [Job Deployment](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/job_deployment.md) - Deploy apps as Spark jobs
|
|
38
|
+
- [Hierarchical Configuration](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_workspace_config.md) - Multi-level YAML config system
|
|
39
|
+
- [Logging & Tracing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/logging_tracing.md) - Observability foundation (including JVM-free telemetry for UC shared clusters)
|
|
40
|
+
- [Watermarking](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/watermarking.md) - Change tracking and incremental processing
|
|
41
|
+
- [File Ingestion](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/file_ingestion.md) - Built-in file ingestion capabilities
|
|
42
|
+
- [Stage Processing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/stage_processing.md) - Pipeline stage orchestration
|
|
43
|
+
- [Temporal End-to-End](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/temporal_end_to_end.md) - Events, conditions, and episodes (spark-kindling-ext-temporal)
|
|
44
|
+
- [Lakeflow App Selection](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/lakeflow_app_selection.md) - Run Kindling data apps inside Databricks Lakeflow pipelines
|
|
45
|
+
- [Dynamic Registration](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/dynamic_registration.md) - Register entities and pipes at runtime
|
|
46
|
+
|
|
47
|
+
### Platform & Development
|
|
48
|
+
- [Platform API Architecture](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_api_architecture.md) - Multi-platform abstraction
|
|
49
|
+
- [Platform Storage Utils](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/platform_storage_utils.md) - Storage operations
|
|
50
|
+
- [Utilities](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/utilities.md) - Common utilities and helper functions
|
|
51
|
+
- [Build System](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/build_system.md) - How `poe build` produces the runtime, CLI, SDK and extension wheels
|
|
52
|
+
- [CI/CD Setup](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/ci_cd_setup.md) - Continuous integration and deployment
|
|
53
|
+
- [Testing](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/testing.md) - Unit, integration, and system testing
|
|
54
|
+
|
|
55
|
+
## Core Modules
|
|
56
|
+
|
|
57
|
+
The framework consists of several modular components:
|
|
58
|
+
|
|
59
|
+
- **Dependency Injection Engine** - Provides IoC container for loose coupling
|
|
60
|
+
- **Data Entities** - Entity registry and storage abstraction
|
|
61
|
+
- **Data Pipes** - Transformation pipeline definition and execution
|
|
62
|
+
- **Data Apps** - Package and deploy apps as .kda archives
|
|
63
|
+
- **Job Deployment** - Deploy apps as Spark jobs across platforms
|
|
64
|
+
- **Configuration System** - Hierarchical YAML configuration (platform/workspace/environment)
|
|
65
|
+
- **Platform Services** - Unified abstraction for Fabric/Synapse/Databricks
|
|
66
|
+
- **Watermarking** - Change tracking for incremental processing
|
|
67
|
+
- **Streaming Orchestration** - Streaming queries with health monitoring, recovery, and merge sinks
|
|
68
|
+
- **Migration** - Plan/apply reconciliation of declared entities against live Delta tables and views
|
|
69
|
+
- **File Ingestion** - File pattern discovery and loading
|
|
70
|
+
- **Stage Processing** - Orchestration of multi-stage pipelines
|
|
71
|
+
- **Common Transforms** - Reusable data transformation utilities
|
|
72
|
+
- **Logging & Tracing** - Comprehensive observability features
|
|
73
|
+
|
|
74
|
+
## Extensions
|
|
75
|
+
|
|
76
|
+
- **[spark-kindling-ext-otel-azure](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_otel_azure)** - Azure Monitor OpenTelemetry integration
|
|
77
|
+
- **[spark-kindling-ext-sdp](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_sdp)** - Spark Declarative Pipelines (SDP) declaration engine
|
|
78
|
+
- **[spark-kindling-ext-databricks](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks)** - Databricks Lakeflow adapter for the SDP declaration engine
|
|
79
|
+
- **[spark-kindling-ext-databricks-autoloader](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_databricks_autoloader)** - Auto Loader (`cloudFiles`) discovery for file ingestion
|
|
80
|
+
- **[spark-kindling-ext-temporal](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_temporal)** - Temporal event, condition, and episode primitives
|
|
81
|
+
- **[spark-kindling-ext-adx](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_adx)** - Azure Data Explorer entity provider (Kusto Spark connector; an API-based `adx-api` provider ships in core)
|
|
82
|
+
- **[spark-kindling-ext-cosmos](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_cosmos)** - Azure Cosmos DB entity provider (idempotent upsert writes)
|
|
83
|
+
- **[spark-kindling-ext-visualization](https://github.com/sep/spark-kindling-framework/blob/main/packages/extensions/kindling_ext_visualization)** - Matplotlib visualization helpers
|
|
84
|
+
|
|
85
|
+
In a Kindling project, add any extension with `kindling env add <name>`. The Databricks, SDP, Cosmos DB, Temporal and OpenTelemetry Azure extensions are also on PyPI (`pip install <name>`). The ADX, Auto Loader and visualization extensions are not; they are attached to each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) as wheels, which `kindling env add` pins by URL (or `pip install` the wheel's release URL).
|
|
86
|
+
|
|
87
|
+
## Install
|
|
88
|
+
|
|
89
|
+
Kindling is on [PyPI](https://pypi.org/project/spark-kindling/). One distribution, platform-specific extras:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install 'spark-kindling[synapse]' # Azure Synapse Analytics
|
|
93
|
+
pip install 'spark-kindling[databricks]' # Databricks
|
|
94
|
+
pip install 'spark-kindling[fabric]' # Microsoft Fabric
|
|
95
|
+
pip install 'spark-kindling[standalone]' # Local development / generic Spark
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
The Python import name is `kindling` (unchanged):
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
from kindling.data_entities import DataEntities
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Design-time tooling ships separately:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
# `kindling` CLI for scaffolding and deploy; it pulls in spark-kindling-sdk (programmatic access to platform APIs)
|
|
108
|
+
pip install spark-kindling-cli
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
The runtime, CLI and SDK are versioned together: pin all three to one release (`spark-kindling[...]==0.14.0 spark-kindling-cli==0.14.0 spark-kindling-sdk==0.14.0`; the CLI alone only requires a minimum SDK). Extensions keep their own versions; each GitHub release lists the extension versions that go with it, and `kindling env add` picks them. Projects scaffolded with `kindling repo init` / `kindling package init` pin these versions in `pyproject.toml` for you, and `kindling env update` moves them to a newer release.
|
|
112
|
+
|
|
113
|
+
### Without PyPI access
|
|
114
|
+
|
|
115
|
+
Each [GitHub Release](https://github.com/sep/spark-kindling-framework/releases) also attaches the wheels, so environments that cannot reach PyPI (and releases before 0.14.0, which exist only on GitHub) can install by URL:
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
V=0.14.0 # any release, without the leading v
|
|
119
|
+
BASE=https://github.com/sep/spark-kindling-framework/releases/download/v$V
|
|
120
|
+
pip install "spark-kindling[databricks] @ $BASE/spark_kindling-$V-py3-none-any.whl"
|
|
121
|
+
pip install "$BASE/spark_kindling_cli-$V-py3-none-any.whl" "$BASE/spark_kindling_sdk-$V-py3-none-any.whl"
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
See [docs/release_process.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/release_process.md) for how releases are published and [docs/developer_workflow.md](https://github.com/sep/spark-kindling-framework/blob/main/docs/contributing/developer_workflow.md) for local development.
|
|
125
|
+
|
|
126
|
+
## CLI Quick Start
|
|
127
|
+
|
|
128
|
+
Install the CLI (`pip install spark-kindling-cli`; you also need [uv](https://docs.astral.sh/uv/)) and scaffold a repo, package, and app explicitly:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
kindling repo init my-app --output-dir ./my_app
|
|
132
|
+
cd my_app
|
|
133
|
+
kindling package init my-app
|
|
134
|
+
kindling app init my-app --package my-app
|
|
135
|
+
kindling env bootstrap # pins Kindling at the root and syncs one repo-wide .venv/
|
|
136
|
+
uv run kindling app run my_app --env local
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
Or open the generated repo in its devcontainer (`.devcontainer/devcontainer.json`), which runs `kindling env bootstrap` for you.
|
|
140
|
+
No Azure credentials needed — the scaffold uses in-memory entity providers by default.
|
|
141
|
+
See [Local Python-First Development](https://github.com/sep/spark-kindling-framework/blob/main/docs/guide/local_python_first.md) for the full local workflow.
|
|
142
|
+
|
|
143
|
+
## Notebook Quick Start
|
|
144
|
+
|
|
145
|
+
```python
|
|
146
|
+
# Import Kindling framework
|
|
147
|
+
from kindling.data_entities import *
|
|
148
|
+
from kindling.data_pipes import *
|
|
149
|
+
from kindling.injection import get_kindling_service
|
|
150
|
+
|
|
151
|
+
# Define data entity (a plain call — registration happens immediately)
|
|
152
|
+
DataEntities.entity(
|
|
153
|
+
entityid="customers.raw",
|
|
154
|
+
name="Raw Customer Data",
|
|
155
|
+
partition_columns=["country"],
|
|
156
|
+
merge_columns=["customer_id"],
|
|
157
|
+
tags={"domain": "customer", "layer": "bronze"},
|
|
158
|
+
schema=customer_schema,
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
# Create data transformation pipe
|
|
162
|
+
@DataPipes.pipe(
|
|
163
|
+
pipeid="customers.transform",
|
|
164
|
+
name="Transform Customers",
|
|
165
|
+
tags={"layer": "silver"},
|
|
166
|
+
input_entity_ids=["customers.raw"],
|
|
167
|
+
output_entity_id="customers.silver",
|
|
168
|
+
output_type="table"
|
|
169
|
+
)
|
|
170
|
+
def transform_customers(customers_raw):
|
|
171
|
+
# Transformation logic here
|
|
172
|
+
return customers_raw.filter(...)
|
|
173
|
+
|
|
174
|
+
# Execute pipeline
|
|
175
|
+
executer = get_kindling_service(DataPipesExecution)
|
|
176
|
+
executer.run_datapipes(["customers.transform"])
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Key Features
|
|
180
|
+
|
|
181
|
+
### Core Framework
|
|
182
|
+
- **Declarative definitions** for data entities and transformations (tags-first conventions)
|
|
183
|
+
- **Dependency injection** for loose coupling and testability
|
|
184
|
+
- **Delta Lake integration** for reliable storage and time travel
|
|
185
|
+
- **Built-in entity providers** - `delta`, `parquet`, `csv`, `memory`, `sql` (views), `current_view` (SCD2), `eventhub`, `adx-api`; ADX (connector) and Cosmos DB via extensions
|
|
186
|
+
- **Watermarking** for change tracking and incremental loads
|
|
187
|
+
- **Streaming pipes** with per-micro-batch SCD1/SCD2 merge sinks (`write.mode` tag)
|
|
188
|
+
- **Config-first execution options** - parallelism, error strategies (incl. `skip_dependents`), per-pipe retry via `kindling.execution.*`
|
|
189
|
+
- **Schema reconciliation** - `kindling migrate plan|apply` converges live tables to declared entities
|
|
190
|
+
- **Notebook round-tripping** - `kindling notebook list|pull|push` workspace notebooks as git-friendly `.py` source
|
|
191
|
+
- **Observability** through logging and tracing (JVM-free fallback on UC shared clusters)
|
|
192
|
+
|
|
193
|
+
### Multi-Platform Support
|
|
194
|
+
- **Unified API** across Microsoft Fabric, Azure Synapse, and Databricks
|
|
195
|
+
- **Platform abstraction** - write once, deploy anywhere
|
|
196
|
+
- **Platform-specific optimizations** via configuration
|
|
197
|
+
- **Smart platform detection** - automatic platform identification
|
|
198
|
+
|
|
199
|
+
### Data Apps & Deployment
|
|
200
|
+
- **Package apps** as .kda archives (Kindling Data Apps)
|
|
201
|
+
- **Run through the durable runner** - apps submit to one managed platform runner
|
|
202
|
+
- **Cross-platform deployment** - same app runs on all platforms
|
|
203
|
+
- **Artifact management** - wheels, configs, and dependencies
|
|
204
|
+
|
|
205
|
+
### Configuration System
|
|
206
|
+
- **Hierarchical YAML configs** - settings, platform, workspace, environment layers
|
|
207
|
+
- **Auto-detection** - platform and workspace ID discovery
|
|
208
|
+
- **Flexible overrides** - bootstrap config for runtime changes
|
|
209
|
+
- **Multi-team support** - workspace-specific configurations
|
|
210
|
+
|
|
211
|
+
### Extensibility
|
|
212
|
+
- **Extension system** - load custom packages via configuration
|
|
213
|
+
- **Azure Monitor integration** - via spark-kindling-ext-otel-azure extension
|
|
214
|
+
- **Declarative engines** - SDP declaration engine with a Databricks Lakeflow adapter (spark-kindling-ext-sdp / spark-kindling-ext-databricks)
|
|
215
|
+
- **Temporal primitives** - events, conditions, and episode lifecycle (spark-kindling-ext-temporal)
|
|
216
|
+
- **Custom providers** - implement your own storage backends
|
|
217
|
+
- **Signal/event system** - blinker-based pub/sub for custom workflows
|
|
218
|
+
|
|
219
|
+
## Requirements
|
|
220
|
+
- Python 3.10+
|
|
221
|
+
- Apache Spark 3.4+
|
|
222
|
+
- Microsoft Fabric, Azure Synapse Analytics, Databricks — or local/standalone Spark for development
|
|
223
|
+
|
|
224
|
+
## Dependencies
|
|
225
|
+
This framework builds upon several excellent open source projects:
|
|
226
|
+
|
|
227
|
+
- **Apache Spark** - Unified analytics engine for large-scale data processing (Apache 2.0)
|
|
228
|
+
- **Delta Lake** - Storage framework for reliable data lakes (Apache 2.0)
|
|
229
|
+
- **injector** - Python dependency injection framework (BSD)
|
|
230
|
+
- **blinker** - Python signal/event framework for pub/sub (MIT)
|
|
231
|
+
- **dynaconf** - Configuration management for Python (MIT)
|
|
232
|
+
- **pytest** - Testing framework (MIT)
|
|
233
|
+
|
|
234
|
+
### Platform SDKs (auto-detected, only one required):
|
|
235
|
+
- **notebookutils** / **mssparkutils** - Microsoft Fabric & Synapse (MIT)
|
|
236
|
+
- **dbutils** - Databricks utilities (Databricks)
|
|
237
|
+
|
|
238
|
+
### Extensions (optional):
|
|
239
|
+
- **azure-monitor-opentelemetry** - Azure Monitor integration (MIT)
|
|
240
|
+
- **opentelemetry-api/sdk** - OpenTelemetry tracing (Apache 2.0)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
## Contributing
|
|
244
|
+
We welcome contributions! Please see [CONTRIBUTING.md](https://github.com/sep/spark-kindling-framework/blob/main/CONTRIBUTING.md) for guidelines.
|
|
245
|
+
|
|
246
|
+
## Support
|
|
247
|
+
This is open source software provided without warranty or guaranteed support.
|
|
248
|
+
|
|
249
|
+
**Commercial Support:** Professional support, training, and consulting services are available from Software Engineering Professionals, Inc. Contact us at [contact information](mailto:engineering@sep.com).
|
|
250
|
+
|
|
251
|
+
## License
|
|
252
|
+
|
|
253
|
+
This project is licensed under the MIT License - see the [LICENSE](https://github.com/sep/spark-kindling-framework/blob/main/LICENSE) file for details.
|
|
254
|
+
|
|
255
|
+
## Developed By
|
|
256
|
+
**Software Engineering Professionals, Inc.**
|
|
257
|
+
16080 Westfield Blvd.
|
|
258
|
+
Carmel, IN 46033
|
|
259
|
+
[www.sep.com](https://www.sep.com)
|
|
260
|
+
|
|
261
|
+
## Acknowledgments
|
|
262
|
+
This framework was developed to solve real-world data processing challenges encountered across multiple enterprise engagements. We're grateful to our clients who have helped shape the requirements and validate the approach.
|
|
263
|
+
|
|
264
|
+
---
|
|
265
|
+
**Note:** This framework is maintained by SEP and used across multiple projects. If you're using this framework and encounter issues or have suggestions, please open an issue or submit a pull request.
|