etlantic-sparkforge 0.20.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- etlantic_sparkforge-0.20.0/.gitignore +29 -0
- etlantic_sparkforge-0.20.0/PKG-INFO +117 -0
- etlantic_sparkforge-0.20.0/README.md +94 -0
- etlantic_sparkforge-0.20.0/pyproject.toml +38 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/__init__.py +61 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/adapt.py +572 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/compat.py +155 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/ir.py +197 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/py.typed +0 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/reports.py +341 -0
- etlantic_sparkforge-0.20.0/src/etlantic_sparkforge/runtime_map.py +104 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
*.egg-info/
|
|
6
|
+
.eggs/
|
|
7
|
+
dist/
|
|
8
|
+
build/
|
|
9
|
+
site/
|
|
10
|
+
.venv/
|
|
11
|
+
venv/
|
|
12
|
+
.env
|
|
13
|
+
|
|
14
|
+
# Tools
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
.ruff_cache/
|
|
17
|
+
.mypy_cache/
|
|
18
|
+
.coverage
|
|
19
|
+
htmlcov/
|
|
20
|
+
# macOS
|
|
21
|
+
.DS_Store
|
|
22
|
+
**/.DS_Store
|
|
23
|
+
._*
|
|
24
|
+
**/._*
|
|
25
|
+
|
|
26
|
+
examples/_file_storage_out/
|
|
27
|
+
examples/_generated_*.py
|
|
28
|
+
tmp_cli_contracts/
|
|
29
|
+
tmp_/
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: etlantic-sparkforge
|
|
3
|
+
Version: 0.20.0
|
|
4
|
+
Summary: SparkForge-to-ETLantic migration adapter (medallion facade mapping).
|
|
5
|
+
Project-URL: Homepage, https://github.com/eddiethedean/etlantic
|
|
6
|
+
Project-URL: Documentation, https://github.com/eddiethedean/etlantic/tree/main/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/eddiethedean/etlantic
|
|
8
|
+
Project-URL: Issues, https://github.com/eddiethedean/etlantic/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/eddiethedean/etlantic/blob/main/CHANGELOG.md
|
|
10
|
+
Author-email: Odo Matthews <odosmatthews@gmail.com>
|
|
11
|
+
License-Expression: MIT
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.11
|
|
21
|
+
Requires-Dist: etlantic<0.21,>=0.20.0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# etlantic-sparkforge
|
|
25
|
+
|
|
26
|
+
SparkForge → ETLantic migration adapter for ETLantic 0.20.
|
|
27
|
+
|
|
28
|
+
SparkForge remains the medallion-facing facade (bronze / silver / gold).
|
|
29
|
+
This package maps those conventions onto ordinary ETLantic `Extract` / `Step` /
|
|
30
|
+
`Load`, `Profile`, `RunSelection` / `RunIntent`, and `PipelineRunReport`
|
|
31
|
+
surfaces. **ETLantic core never gains medallion types.**
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install 'etlantic==0.20.0' 'etlantic-sparkforge==0.20.0'
|
|
37
|
+
# or
|
|
38
|
+
pip install 'etlantic[sparkforge]'
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
The shipped adapter is **IR-only**: feed `SparkForgePipelineSpec` (JSON/YAML
|
|
42
|
+
fixtures or hand-built dataclasses). There is **no** live
|
|
43
|
+
`pipeline_builder` / SparkForge Python API bridge in this release.
|
|
44
|
+
|
|
45
|
+
The adapter is registered explicitly by importing `etlantic_sparkforge` and
|
|
46
|
+
calling its conversion helpers. The adapted result supplies an ordinary
|
|
47
|
+
ETLantic pipeline and profile; select execution plugins such as
|
|
48
|
+
`Profile.spark_engine="pyspark"` separately. Production profiles must
|
|
49
|
+
allowlist every trusted execution plugin.
|
|
50
|
+
|
|
51
|
+
## Quick start (IR → Pipeline)
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from etlantic_sparkforge import (
|
|
55
|
+
SparkForgePipelineSpec,
|
|
56
|
+
SparkForgeStepSpec,
|
|
57
|
+
StepKind,
|
|
58
|
+
LayerKind,
|
|
59
|
+
adapt_pipeline,
|
|
60
|
+
debug_request_from_sparkforge,
|
|
61
|
+
enrich_plan,
|
|
62
|
+
)
|
|
63
|
+
from etlantic.plan import plan_pipeline
|
|
64
|
+
|
|
65
|
+
spec = SparkForgePipelineSpec(
|
|
66
|
+
name="ecommerce",
|
|
67
|
+
schema="demo",
|
|
68
|
+
steps=(
|
|
69
|
+
SparkForgeStepSpec(
|
|
70
|
+
name="orders",
|
|
71
|
+
kind=StepKind.BRONZE_RULES,
|
|
72
|
+
layer=LayerKind.BRONZE,
|
|
73
|
+
table_name="bronze_orders",
|
|
74
|
+
),
|
|
75
|
+
SparkForgeStepSpec(
|
|
76
|
+
name="clean_orders",
|
|
77
|
+
kind=StepKind.SILVER_TRANSFORM,
|
|
78
|
+
layer=LayerKind.SILVER,
|
|
79
|
+
source="orders",
|
|
80
|
+
table_name="silver_orders",
|
|
81
|
+
write_mode="overwrite",
|
|
82
|
+
),
|
|
83
|
+
),
|
|
84
|
+
)
|
|
85
|
+
adapted = adapt_pipeline(spec)
|
|
86
|
+
adapted.pipeline_cls.validate(profile=adapted.profile)
|
|
87
|
+
plan = enrich_plan(
|
|
88
|
+
plan_pipeline(adapted.pipeline_cls, profile=adapted.profile),
|
|
89
|
+
adapted,
|
|
90
|
+
)
|
|
91
|
+
request = debug_request_from_sparkforge(mode="incremental", skip_writes=True)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Progressive engine deprecation path
|
|
95
|
+
|
|
96
|
+
1. **Plan-only** — generate/inspect ETLantic plans from SparkForge IR
|
|
97
|
+
(`strict_delta=False` to warn instead of fail when Delta caps are unknown)
|
|
98
|
+
2. **Dual reporting** — `adapt_run_result` → `PipelineRunReport`
|
|
99
|
+
3. **ETLantic planning** — selections/intents via `debug_request_from_sparkforge`
|
|
100
|
+
4. **Plugin execution** — `Profile.spark_engine="pyspark"` / SQL plugins
|
|
101
|
+
5. **Facade** — SparkForge keeps medallion builder; retire duplicated engines
|
|
102
|
+
|
|
103
|
+
`transform_ref` / bronze `rules` emit `PMSF411` warnings: the adapter builds
|
|
104
|
+
passthrough transforms for planning parity; it does not execute SparkForge
|
|
105
|
+
callables. Write intents (including MERGE) are attached via `enrich_plan` for
|
|
106
|
+
orchestration; the local runtime still gates materialization with
|
|
107
|
+
`RunRequest.no_write`.
|
|
108
|
+
|
|
109
|
+
See `docs/11_DEVELOPMENT/MIGRATION_0_9_TO_0_10.md`.
|
|
110
|
+
|
|
111
|
+
## Boundary
|
|
112
|
+
|
|
113
|
+
| Concern | Owner |
|
|
114
|
+
|---|---|
|
|
115
|
+
| bronze / silver / gold APIs | SparkForge |
|
|
116
|
+
| portable graph, plan, reports | ETLantic |
|
|
117
|
+
| mapping + parity fixtures | `etlantic-sparkforge` |
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# etlantic-sparkforge
|
|
2
|
+
|
|
3
|
+
SparkForge → ETLantic migration adapter for ETLantic 0.20.
|
|
4
|
+
|
|
5
|
+
SparkForge remains the medallion-facing facade (bronze / silver / gold).
|
|
6
|
+
This package maps those conventions onto ordinary ETLantic `Extract` / `Step` /
|
|
7
|
+
`Load`, `Profile`, `RunSelection` / `RunIntent`, and `PipelineRunReport`
|
|
8
|
+
surfaces. **ETLantic core never gains medallion types.**
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install 'etlantic==0.20.0' 'etlantic-sparkforge==0.20.0'
|
|
14
|
+
# or
|
|
15
|
+
pip install 'etlantic[sparkforge]'
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
The shipped adapter is **IR-only**: feed `SparkForgePipelineSpec` (JSON/YAML
|
|
19
|
+
fixtures or hand-built dataclasses). There is **no** live
|
|
20
|
+
`pipeline_builder` / SparkForge Python API bridge in this release.
|
|
21
|
+
|
|
22
|
+
The adapter is registered explicitly by importing `etlantic_sparkforge` and
|
|
23
|
+
calling its conversion helpers. The adapted result supplies an ordinary
|
|
24
|
+
ETLantic pipeline and profile; select execution plugins such as
|
|
25
|
+
`Profile.spark_engine="pyspark"` separately. Production profiles must
|
|
26
|
+
allowlist every trusted execution plugin.
|
|
27
|
+
|
|
28
|
+
## Quick start (IR → Pipeline)
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from etlantic_sparkforge import (
|
|
32
|
+
SparkForgePipelineSpec,
|
|
33
|
+
SparkForgeStepSpec,
|
|
34
|
+
StepKind,
|
|
35
|
+
LayerKind,
|
|
36
|
+
adapt_pipeline,
|
|
37
|
+
debug_request_from_sparkforge,
|
|
38
|
+
enrich_plan,
|
|
39
|
+
)
|
|
40
|
+
from etlantic.plan import plan_pipeline
|
|
41
|
+
|
|
42
|
+
spec = SparkForgePipelineSpec(
|
|
43
|
+
name="ecommerce",
|
|
44
|
+
schema="demo",
|
|
45
|
+
steps=(
|
|
46
|
+
SparkForgeStepSpec(
|
|
47
|
+
name="orders",
|
|
48
|
+
kind=StepKind.BRONZE_RULES,
|
|
49
|
+
layer=LayerKind.BRONZE,
|
|
50
|
+
table_name="bronze_orders",
|
|
51
|
+
),
|
|
52
|
+
SparkForgeStepSpec(
|
|
53
|
+
name="clean_orders",
|
|
54
|
+
kind=StepKind.SILVER_TRANSFORM,
|
|
55
|
+
layer=LayerKind.SILVER,
|
|
56
|
+
source="orders",
|
|
57
|
+
table_name="silver_orders",
|
|
58
|
+
write_mode="overwrite",
|
|
59
|
+
),
|
|
60
|
+
),
|
|
61
|
+
)
|
|
62
|
+
adapted = adapt_pipeline(spec)
|
|
63
|
+
adapted.pipeline_cls.validate(profile=adapted.profile)
|
|
64
|
+
plan = enrich_plan(
|
|
65
|
+
plan_pipeline(adapted.pipeline_cls, profile=adapted.profile),
|
|
66
|
+
adapted,
|
|
67
|
+
)
|
|
68
|
+
request = debug_request_from_sparkforge(mode="incremental", skip_writes=True)
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Progressive engine deprecation path
|
|
72
|
+
|
|
73
|
+
1. **Plan-only** — generate/inspect ETLantic plans from SparkForge IR
|
|
74
|
+
(`strict_delta=False` to warn instead of fail when Delta caps are unknown)
|
|
75
|
+
2. **Dual reporting** — `adapt_run_result` → `PipelineRunReport`
|
|
76
|
+
3. **ETLantic planning** — selections/intents via `debug_request_from_sparkforge`
|
|
77
|
+
4. **Plugin execution** — `Profile.spark_engine="pyspark"` / SQL plugins
|
|
78
|
+
5. **Facade** — SparkForge keeps medallion builder; retire duplicated engines
|
|
79
|
+
|
|
80
|
+
`transform_ref` / bronze `rules` emit `PMSF411` warnings: the adapter builds
|
|
81
|
+
passthrough transforms for planning parity; it does not execute SparkForge
|
|
82
|
+
callables. Write intents (including MERGE) are attached via `enrich_plan` for
|
|
83
|
+
orchestration; the local runtime still gates materialization with
|
|
84
|
+
`RunRequest.no_write`.
|
|
85
|
+
|
|
86
|
+
See `docs/11_DEVELOPMENT/MIGRATION_0_9_TO_0_10.md`.
|
|
87
|
+
|
|
88
|
+
## Boundary
|
|
89
|
+
|
|
90
|
+
| Concern | Owner |
|
|
91
|
+
|---|---|
|
|
92
|
+
| bronze / silver / gold APIs | SparkForge |
|
|
93
|
+
| portable graph, plan, reports | ETLantic |
|
|
94
|
+
| mapping + parity fixtures | `etlantic-sparkforge` |
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "etlantic-sparkforge"
|
|
3
|
+
version = "0.20.0"
|
|
4
|
+
description = "SparkForge-to-ETLantic migration adapter (medallion facade mapping)."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
requires-python = ">=3.11"
|
|
8
|
+
authors = [{ name = "Odo Matthews", email = "odosmatthews@gmail.com" }]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 5 - Production/Stable",
|
|
11
|
+
"Intended Audience :: Developers",
|
|
12
|
+
"License :: OSI Approved :: MIT License",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.11",
|
|
15
|
+
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Typing :: Typed",
|
|
18
|
+
]
|
|
19
|
+
dependencies = [
|
|
20
|
+
"etlantic>=0.20.0,<0.21",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
# IR-only adapter in 0.10: no live SparkForge / pipeline_builder dependency.
|
|
24
|
+
# Fixture and parity tests run without installing SparkForge or PySpark.
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://github.com/eddiethedean/etlantic"
|
|
28
|
+
Documentation = "https://github.com/eddiethedean/etlantic/tree/main/docs"
|
|
29
|
+
Repository = "https://github.com/eddiethedean/etlantic"
|
|
30
|
+
Issues = "https://github.com/eddiethedean/etlantic/issues"
|
|
31
|
+
Changelog = "https://github.com/eddiethedean/etlantic/blob/main/CHANGELOG.md"
|
|
32
|
+
|
|
33
|
+
[build-system]
|
|
34
|
+
requires = ["hatchling"]
|
|
35
|
+
build-backend = "hatchling.build"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.wheel]
|
|
38
|
+
packages = ["src/etlantic_sparkforge"]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""etlantic-sparkforge — SparkForge → ETLantic migration adapter."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from etlantic_sparkforge.adapt import (
|
|
6
|
+
AdaptationResult,
|
|
7
|
+
AdaptedRow,
|
|
8
|
+
AdapterError,
|
|
9
|
+
adapt_pipeline,
|
|
10
|
+
adapt_profile,
|
|
11
|
+
adapt_validation_policy,
|
|
12
|
+
enrich_plan,
|
|
13
|
+
)
|
|
14
|
+
from etlantic_sparkforge.compat import (
|
|
15
|
+
COMPATIBILITY_MATRIX,
|
|
16
|
+
assert_delta_capabilities,
|
|
17
|
+
retry_policy_from_sparkforge,
|
|
18
|
+
write_mode_from_sparkforge,
|
|
19
|
+
write_mode_metadata,
|
|
20
|
+
)
|
|
21
|
+
from etlantic_sparkforge.ir import (
|
|
22
|
+
LayerKind,
|
|
23
|
+
SparkForgePipelineSpec,
|
|
24
|
+
SparkForgeStepSpec,
|
|
25
|
+
StepKind,
|
|
26
|
+
)
|
|
27
|
+
from etlantic_sparkforge.reports import adapt_run_result, report_to_sparkforge_explain
|
|
28
|
+
from etlantic_sparkforge.runtime_map import (
|
|
29
|
+
bind_debug_session,
|
|
30
|
+
debug_request_from_sparkforge,
|
|
31
|
+
intent_from_sparkforge,
|
|
32
|
+
selection_from_sparkforge,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
__version__ = "0.20.0"
|
|
36
|
+
|
|
37
|
+
__all__ = [
|
|
38
|
+
"COMPATIBILITY_MATRIX",
|
|
39
|
+
"AdaptationResult",
|
|
40
|
+
"AdaptedRow",
|
|
41
|
+
"AdapterError",
|
|
42
|
+
"LayerKind",
|
|
43
|
+
"SparkForgePipelineSpec",
|
|
44
|
+
"SparkForgeStepSpec",
|
|
45
|
+
"StepKind",
|
|
46
|
+
"__version__",
|
|
47
|
+
"adapt_pipeline",
|
|
48
|
+
"adapt_profile",
|
|
49
|
+
"adapt_run_result",
|
|
50
|
+
"adapt_validation_policy",
|
|
51
|
+
"assert_delta_capabilities",
|
|
52
|
+
"bind_debug_session",
|
|
53
|
+
"debug_request_from_sparkforge",
|
|
54
|
+
"enrich_plan",
|
|
55
|
+
"intent_from_sparkforge",
|
|
56
|
+
"report_to_sparkforge_explain",
|
|
57
|
+
"retry_policy_from_sparkforge",
|
|
58
|
+
"selection_from_sparkforge",
|
|
59
|
+
"write_mode_from_sparkforge",
|
|
60
|
+
"write_mode_metadata",
|
|
61
|
+
]
|