phlo-transform 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_transform/__init__.py +23 -0
- phlo_transform/assets.py +552 -0
- phlo_transform/authorization.py +62 -0
- phlo_transform/binary.py +343 -0
- phlo_transform/cli_plugin.py +402 -0
- phlo_transform/editor.py +679 -0
- phlo_transform/editor_execution.py +467 -0
- phlo_transform/plugin.py +59 -0
- phlo_transform/settings.py +139 -0
- phlo_transform/translate.py +72 -0
- phlo_transform/workspace.py +289 -0
- phlo_transform-0.2.0.dist-info/METADATA +255 -0
- phlo_transform-0.2.0.dist-info/RECORD +16 -0
- phlo_transform-0.2.0.dist-info/WHEEL +5 -0
- phlo_transform-0.2.0.dist-info/entry_points.txt +8 -0
- phlo_transform-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""dbt-to-native translation helpers.
|
|
2
|
+
|
|
3
|
+
The engine owns translation semantics; this module only locates the project's
|
|
4
|
+
dbt project and forwards the documented ``translate --from dbt`` flags. Both
|
|
5
|
+
the CLI and example maintenance scripts use the same helper so a regenerated
|
|
6
|
+
workspace is produced by exactly the command the docs describe.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from phlo.exceptions import PhloCapabilitySetupError
|
|
14
|
+
from phlo_transform.binary import Invocation, require_binary, run_cli
|
|
15
|
+
from phlo_transform.settings import TransformSettings
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def find_dbt_project(project_root: Path) -> Path | None:
|
|
19
|
+
"""Return the shallowest dbt project below ``workflows/``, if any."""
|
|
20
|
+
|
|
21
|
+
markers = [
|
|
22
|
+
marker
|
|
23
|
+
for marker in project_root.glob("workflows/**/dbt_project.yml")
|
|
24
|
+
if "dbt_packages" not in marker.parts
|
|
25
|
+
]
|
|
26
|
+
if not markers:
|
|
27
|
+
return None
|
|
28
|
+
return min(markers, key=lambda path: (len(path.parts), str(path))).parent
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def translate_dbt_project(
|
|
32
|
+
project_root: Path,
|
|
33
|
+
settings: TransformSettings,
|
|
34
|
+
*,
|
|
35
|
+
dbt_project: Path | None = None,
|
|
36
|
+
out: Path | None = None,
|
|
37
|
+
check_only: bool = False,
|
|
38
|
+
overwrite: bool = False,
|
|
39
|
+
verify: bool = False,
|
|
40
|
+
) -> Invocation:
|
|
41
|
+
"""Run ``translate --from dbt`` against the project's dbt project."""
|
|
42
|
+
|
|
43
|
+
source = dbt_project or find_dbt_project(project_root)
|
|
44
|
+
if source is not None and not source.is_absolute():
|
|
45
|
+
source = (project_root / source).resolve()
|
|
46
|
+
if source is None:
|
|
47
|
+
raise PhloCapabilitySetupError(
|
|
48
|
+
capability="transform",
|
|
49
|
+
required=True,
|
|
50
|
+
message="no dbt project found to translate",
|
|
51
|
+
suggestions=[
|
|
52
|
+
"Create workflows/transforms/dbt/dbt_project.yml",
|
|
53
|
+
"Pass --source-root explicitly",
|
|
54
|
+
],
|
|
55
|
+
)
|
|
56
|
+
if not check_only and out is None:
|
|
57
|
+
raise ValueError("translate needs an output directory or check_only=True")
|
|
58
|
+
binary = require_binary(project_root, settings)
|
|
59
|
+
args = ["translate", "--from", "dbt"]
|
|
60
|
+
if check_only:
|
|
61
|
+
args.append("--check")
|
|
62
|
+
if out is not None:
|
|
63
|
+
destination = out if out.is_absolute() else project_root / out
|
|
64
|
+
args.extend(["--out", str(destination)])
|
|
65
|
+
if overwrite:
|
|
66
|
+
args.append("--overwrite")
|
|
67
|
+
if verify:
|
|
68
|
+
args.append("--verify")
|
|
69
|
+
return run_cli(binary, args, settings=settings, root=source)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
__all__ = ["find_dbt_project", "translate_dbt_project"]
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
"""Discovery and typed parsing of Phlo Transform workspaces.
|
|
2
|
+
|
|
3
|
+
A workspace is a directory containing ``phlo.toml``; the CLI owns discovery,
|
|
4
|
+
SQL compilation, and dependency resolution. This module only locates the
|
|
5
|
+
workspace, runs ``list --json``, and exposes the report as typed records so
|
|
6
|
+
the asset provider never parses SQL itself.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import tomllib
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
from phlo.exceptions import PhloCapabilitySetupError
|
|
16
|
+
from phlo.logging import get_logger
|
|
17
|
+
|
|
18
|
+
from phlo_transform.binary import find_binary, require_binary, run_json
|
|
19
|
+
from phlo_transform.settings import TransformSettings
|
|
20
|
+
|
|
21
|
+
logger = get_logger(__name__)
|
|
22
|
+
|
|
23
|
+
WORKSPACE_MARKER = "phlo.toml"
|
|
24
|
+
_SKIPPED_PARTS = {".phlo", ".venv", "dbt_packages", "node_modules", "target"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True, slots=True)
|
|
28
|
+
class WorkspaceModel:
|
|
29
|
+
"""One compiled model from ``phlo-transform list``."""
|
|
30
|
+
|
|
31
|
+
id: str
|
|
32
|
+
name: str
|
|
33
|
+
namespace: str
|
|
34
|
+
materialization: str
|
|
35
|
+
target: str
|
|
36
|
+
version: str
|
|
37
|
+
path: str | None
|
|
38
|
+
tags: tuple[str, ...]
|
|
39
|
+
depends_on: tuple[str, ...]
|
|
40
|
+
sources: tuple[str, ...]
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def leaf_name(self) -> str:
|
|
44
|
+
"""Return the final dotted name segment."""
|
|
45
|
+
|
|
46
|
+
return self.name.rsplit(".", 1)[-1]
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def group(self) -> str:
|
|
50
|
+
"""Return the workflow or namespace that owns the model."""
|
|
51
|
+
|
|
52
|
+
return self.namespace or "transform"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True, slots=True)
|
|
56
|
+
class WorkspaceTest:
|
|
57
|
+
"""One workspace test and the models it targets."""
|
|
58
|
+
|
|
59
|
+
id: str
|
|
60
|
+
name: str
|
|
61
|
+
targets: tuple[str, ...]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True, slots=True)
|
|
65
|
+
class TransformWorkspace:
|
|
66
|
+
"""A discovered workspace and its compiled model graph."""
|
|
67
|
+
|
|
68
|
+
root: Path
|
|
69
|
+
binary: Path
|
|
70
|
+
models: tuple[WorkspaceModel, ...]
|
|
71
|
+
tests: tuple[WorkspaceTest, ...]
|
|
72
|
+
sources: tuple[str, ...]
|
|
73
|
+
source_assets: dict[str, str] = field(default_factory=dict)
|
|
74
|
+
|
|
75
|
+
def model(self, name: str) -> WorkspaceModel | None:
|
|
76
|
+
"""Return a model by logical name."""
|
|
77
|
+
|
|
78
|
+
return next((model for model in self.models if model.name == name), None)
|
|
79
|
+
|
|
80
|
+
def tests_for(self, model_name: str) -> tuple[WorkspaceTest, ...]:
|
|
81
|
+
"""Return every test that targets a model."""
|
|
82
|
+
|
|
83
|
+
return tuple(test for test in self.tests if model_name in test.targets)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _is_candidate(path: Path) -> bool:
|
|
87
|
+
return not _SKIPPED_PARTS.intersection(path.parts)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def discover_workspaces(project_root: Path) -> list[Path]:
|
|
91
|
+
"""Return candidate workspace roots in deterministic precedence order."""
|
|
92
|
+
|
|
93
|
+
candidates: list[Path] = []
|
|
94
|
+
for relative in (Path("transforms"), Path("workflows") / "transforms", Path(".")):
|
|
95
|
+
marker = project_root / relative / WORKSPACE_MARKER
|
|
96
|
+
if marker.is_file():
|
|
97
|
+
candidates.append(marker.parent)
|
|
98
|
+
for pattern in ("workflows/**/" + WORKSPACE_MARKER, "transforms/**/" + WORKSPACE_MARKER):
|
|
99
|
+
for marker in sorted(
|
|
100
|
+
project_root.glob(pattern),
|
|
101
|
+
key=lambda path: (len(path.parts), str(path)),
|
|
102
|
+
):
|
|
103
|
+
if not _is_candidate(marker):
|
|
104
|
+
continue
|
|
105
|
+
if any(
|
|
106
|
+
marker.parent == candidate or marker.parent.is_relative_to(candidate)
|
|
107
|
+
for candidate in candidates
|
|
108
|
+
):
|
|
109
|
+
continue
|
|
110
|
+
candidates.append(marker.parent)
|
|
111
|
+
return candidates
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def find_workspace(project_root: Path, settings: TransformSettings) -> Path | None:
|
|
115
|
+
"""Resolve the configured workspace, or None when the project has none.
|
|
116
|
+
|
|
117
|
+
More than one workspace is an error rather than a silent pick: a project
|
|
118
|
+
with two native workspaces needs ``PHLO_TRANSFORM_ROOT`` because model
|
|
119
|
+
names and targets can collide across them.
|
|
120
|
+
"""
|
|
121
|
+
|
|
122
|
+
if settings.root:
|
|
123
|
+
configured = Path(settings.root).expanduser()
|
|
124
|
+
root = configured if configured.is_absolute() else project_root / configured
|
|
125
|
+
root = root.resolve()
|
|
126
|
+
if not (root / WORKSPACE_MARKER).is_file():
|
|
127
|
+
raise PhloCapabilitySetupError(
|
|
128
|
+
capability="transform",
|
|
129
|
+
required=True,
|
|
130
|
+
message=f"PHLO_TRANSFORM_ROOT does not contain {WORKSPACE_MARKER}: {root}",
|
|
131
|
+
suggestions=["Point PHLO_TRANSFORM_ROOT at a phlo-transform workspace"],
|
|
132
|
+
)
|
|
133
|
+
return root
|
|
134
|
+
|
|
135
|
+
candidates = discover_workspaces(project_root)
|
|
136
|
+
if not candidates:
|
|
137
|
+
return None
|
|
138
|
+
if len(candidates) > 1:
|
|
139
|
+
listed = ", ".join(str(path.relative_to(project_root)) for path in candidates)
|
|
140
|
+
raise PhloCapabilitySetupError(
|
|
141
|
+
capability="transform",
|
|
142
|
+
required=True,
|
|
143
|
+
message=f"multiple phlo-transform workspaces found: {listed}",
|
|
144
|
+
suggestions=[
|
|
145
|
+
"Set PHLO_TRANSFORM_ROOT to the workspace that owns the transform graph",
|
|
146
|
+
],
|
|
147
|
+
)
|
|
148
|
+
return candidates[0]
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def load_source_assets(root: Path) -> dict[str, str]:
|
|
152
|
+
"""Read the provider-owned source-to-ingestion-asset map from ``phlo.toml``.
|
|
153
|
+
|
|
154
|
+
``[phlo_transform.sources]`` maps a workspace source (``raw.c360_orders``)
|
|
155
|
+
to the Phlo asset that produces it (``sling_c360_orders``). The engine
|
|
156
|
+
ignores the table; it exists so orchestrators keep the ingestion ->
|
|
157
|
+
transform dependency edges when a project switches providers.
|
|
158
|
+
"""
|
|
159
|
+
|
|
160
|
+
path = root / WORKSPACE_MARKER
|
|
161
|
+
try:
|
|
162
|
+
payload = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
163
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
164
|
+
logger.warning("phlo_transform_workspace_config_unreadable", path=str(path))
|
|
165
|
+
return {}
|
|
166
|
+
section = payload.get("phlo_transform")
|
|
167
|
+
if not isinstance(section, dict):
|
|
168
|
+
return {}
|
|
169
|
+
sources = section.get("sources")
|
|
170
|
+
if not isinstance(sources, dict):
|
|
171
|
+
return {}
|
|
172
|
+
return {
|
|
173
|
+
str(source): str(asset)
|
|
174
|
+
for source, asset in sources.items()
|
|
175
|
+
if isinstance(source, str) and isinstance(asset, str)
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _parse_models(payload: dict) -> tuple[WorkspaceModel, ...]:
|
|
180
|
+
models: list[WorkspaceModel] = []
|
|
181
|
+
for raw in payload.get("models") or []:
|
|
182
|
+
if not isinstance(raw, dict):
|
|
183
|
+
continue
|
|
184
|
+
name = str(raw.get("name") or "").strip()
|
|
185
|
+
if not name:
|
|
186
|
+
continue
|
|
187
|
+
models.append(
|
|
188
|
+
WorkspaceModel(
|
|
189
|
+
id=str(raw.get("id") or f"model://{name}"),
|
|
190
|
+
name=name,
|
|
191
|
+
namespace=str(raw.get("namespace") or name.rsplit(".", 1)[0]),
|
|
192
|
+
materialization=str(raw.get("materialization") or "table"),
|
|
193
|
+
target=str(raw.get("target") or name),
|
|
194
|
+
version=str(raw.get("version") or ""),
|
|
195
|
+
path=str(raw["path"]) if raw.get("path") else None,
|
|
196
|
+
tags=tuple(str(tag) for tag in raw.get("tags") or []),
|
|
197
|
+
depends_on=tuple(str(dep) for dep in raw.get("depends_on") or []),
|
|
198
|
+
sources=tuple(str(source) for source in raw.get("sources") or []),
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
return tuple(models)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _parse_tests(payload: dict) -> tuple[WorkspaceTest, ...]:
|
|
205
|
+
tests: list[WorkspaceTest] = []
|
|
206
|
+
for raw in payload.get("tests") or []:
|
|
207
|
+
if not isinstance(raw, dict):
|
|
208
|
+
continue
|
|
209
|
+
name = str(raw.get("name") or "").strip()
|
|
210
|
+
if not name:
|
|
211
|
+
continue
|
|
212
|
+
tests.append(
|
|
213
|
+
WorkspaceTest(
|
|
214
|
+
id=str(raw.get("id") or f"test://{name}"),
|
|
215
|
+
name=name,
|
|
216
|
+
targets=tuple(str(target) for target in raw.get("targets") or []),
|
|
217
|
+
)
|
|
218
|
+
)
|
|
219
|
+
return tuple(tests)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def load_workspace(
|
|
223
|
+
project_root: Path,
|
|
224
|
+
settings: TransformSettings,
|
|
225
|
+
*,
|
|
226
|
+
require: bool = True,
|
|
227
|
+
) -> TransformWorkspace | None:
|
|
228
|
+
"""Return the compiled workspace, or None when discovery finds none.
|
|
229
|
+
|
|
230
|
+
``require`` distinguishes asset discovery (a missing workspace is a normal
|
|
231
|
+
"provider not in use" state) from an explicit CLI command (which should
|
|
232
|
+
explain how to create one).
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
root = find_workspace(project_root, settings)
|
|
236
|
+
if root is None:
|
|
237
|
+
return None
|
|
238
|
+
binary = (
|
|
239
|
+
require_binary(project_root, settings) if require else find_binary(project_root, settings)
|
|
240
|
+
)
|
|
241
|
+
if binary is None:
|
|
242
|
+
return None
|
|
243
|
+
payload, invocation = run_json(binary, ["list"], settings=settings, root=root)
|
|
244
|
+
if payload is None or not invocation.ok:
|
|
245
|
+
detail = (invocation.stderr or invocation.stdout).strip().splitlines()
|
|
246
|
+
raise PhloCapabilitySetupError(
|
|
247
|
+
capability="transform",
|
|
248
|
+
required=True,
|
|
249
|
+
message=(
|
|
250
|
+
"phlo-transform could not list the workspace"
|
|
251
|
+
+ (f": {detail[-1]}" if detail else "")
|
|
252
|
+
),
|
|
253
|
+
suggestions=[
|
|
254
|
+
f"Run: phlo-transform --root {root} check",
|
|
255
|
+
"Fix the reported diagnostics and retry",
|
|
256
|
+
],
|
|
257
|
+
)
|
|
258
|
+
sources = tuple(
|
|
259
|
+
str(raw.get("name"))
|
|
260
|
+
for raw in payload.get("sources") or []
|
|
261
|
+
if isinstance(raw, dict) and raw.get("name")
|
|
262
|
+
)
|
|
263
|
+
workspace = TransformWorkspace(
|
|
264
|
+
root=root,
|
|
265
|
+
binary=binary,
|
|
266
|
+
models=_parse_models(payload),
|
|
267
|
+
tests=_parse_tests(payload),
|
|
268
|
+
sources=sources,
|
|
269
|
+
source_assets=load_source_assets(root),
|
|
270
|
+
)
|
|
271
|
+
logger.info(
|
|
272
|
+
"phlo_transform_workspace_loaded",
|
|
273
|
+
workspace_root=str(root),
|
|
274
|
+
model_count=len(workspace.models),
|
|
275
|
+
test_count=len(workspace.tests),
|
|
276
|
+
)
|
|
277
|
+
return workspace
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
__all__ = [
|
|
281
|
+
"TransformWorkspace",
|
|
282
|
+
"WORKSPACE_MARKER",
|
|
283
|
+
"WorkspaceModel",
|
|
284
|
+
"WorkspaceTest",
|
|
285
|
+
"discover_workspaces",
|
|
286
|
+
"find_workspace",
|
|
287
|
+
"load_source_assets",
|
|
288
|
+
"load_workspace",
|
|
289
|
+
]
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phlo-transform
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: SQL transformation capability provider for Phlo, backed by the Phlo Transform engine
|
|
5
|
+
Author-email: Phlo Team <team@phlo.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: phlo<0.20,>=0.19.0
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
12
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
13
|
+
|
|
14
|
+
# phlo-transform
|
|
15
|
+
|
|
16
|
+
`phlo-transform` is Phlo's SQL transformation provider. It exposes two kinds of
|
|
17
|
+
assets to installed orchestrator adapters:
|
|
18
|
+
|
|
19
|
+
- declarations written with `phlo.transform.sql()`;
|
|
20
|
+
- every compiled model in a native [Phlo Transform](https://github.com/phlohouse/phlo-transform)
|
|
21
|
+
workspace (`phlo.toml`).
|
|
22
|
+
|
|
23
|
+
The Rust engine remains the source of truth for compilation, planning,
|
|
24
|
+
execution, lineage, and tests. This package adapts its JSON reports to Phlo
|
|
25
|
+
capability specs; it does not reimplement SQL analysis.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install phlo-transform
|
|
31
|
+
|
|
32
|
+
# The engine binary, downloaded into .phlo/bin with a verified checksum
|
|
33
|
+
phlo transform install
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`phlo transform install` follows the `PHLO_TRANSFORM_RELEASE_VERSION` setting
|
|
37
|
+
(default `0.1.0`) and resolves the portable release for Linux or macOS on
|
|
38
|
+
x86_64/arm64. Alternatively, install the engine from source with
|
|
39
|
+
`cargo install --path crates/phlo-transform-cli` and leave it on `PATH`.
|
|
40
|
+
|
|
41
|
+
## Declare transforms in Python
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
import phlo
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@phlo.transform.sql(
|
|
48
|
+
table="silver.orders",
|
|
49
|
+
depends_on=["bronze.orders"],
|
|
50
|
+
materialized="table",
|
|
51
|
+
)
|
|
52
|
+
def orders() -> str:
|
|
53
|
+
return "select * from bronze.orders"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`phlo.transform.sql()` registers an asset at import time. The package supplies
|
|
57
|
+
both plugin halves - the transformation provider and the asset provider - so
|
|
58
|
+
the declaration is discovered without any extra wiring.
|
|
59
|
+
|
|
60
|
+
## Use a native workspace
|
|
61
|
+
|
|
62
|
+
A workspace is a directory with `phlo.toml`:
|
|
63
|
+
|
|
64
|
+
```text
|
|
65
|
+
workflows/transforms/native/
|
|
66
|
+
phlo.toml
|
|
67
|
+
transforms/stg_orders.sql
|
|
68
|
+
transforms/orders.sql
|
|
69
|
+
tests/generated/...
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
```toml
|
|
73
|
+
[transform]
|
|
74
|
+
default_namespace = "analytics"
|
|
75
|
+
default_materialization = "table"
|
|
76
|
+
|
|
77
|
+
# Optional: keep ingestion -> transform dependency edges in the orchestrator.
|
|
78
|
+
[phlo_transform.sources]
|
|
79
|
+
"raw.orders" = "sling_orders"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Discovery checks `PHLO_TRANSFORM_ROOT`, then `transforms/`,
|
|
83
|
+
`workflows/transforms/`, and `phlo.toml` files under those trees. More than one
|
|
84
|
+
workspace requires an explicit root.
|
|
85
|
+
|
|
86
|
+
Each compiled model becomes one asset. The default asset key is the model's
|
|
87
|
+
last dotted name segment, which matches a dbt model name and keeps asset keys
|
|
88
|
+
stable when a project switches engines. Set
|
|
89
|
+
`PHLO_TRANSFORM_ASSET_KEY=logical` to keep the namespace prefix instead.
|
|
90
|
+
|
|
91
|
+
Compatible workspace assets share one engine run through the Phlo grouped
|
|
92
|
+
execution contract. A selection of two models runs:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
phlo-transform --root <workspace> --json run --select <first-model> --select <second-model>
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Each requested model retains its asset key, dependencies, checks, and
|
|
99
|
+
`MaterializeResult` from its own report entry. Metadata includes status, action,
|
|
100
|
+
reasons, run and plan IDs, versions, SQL hash, query ID, counts, and timings.
|
|
101
|
+
A failed model or owned test fails that asset, not
|
|
102
|
+
healthy siblings. Missing model or test evidence also fails the asset. An
|
|
103
|
+
invocation failure fails the Dagster step after it emits successful outcomes.
|
|
104
|
+
Unchanged and cached models succeed when their owned tests pass.
|
|
105
|
+
|
|
106
|
+
The engine can include upstream dependencies in its plan, even for a subset
|
|
107
|
+
selection. Phlo emits materialisations and checks only for requested assets.
|
|
108
|
+
Per-model `explain` commands still supply column-lineage metadata; grouping
|
|
109
|
+
reduces run commands, not every engine subprocess.
|
|
110
|
+
|
|
111
|
+
Assets with different partitions, resources, or capability overrides form
|
|
112
|
+
separate groups. A model with blocking checks finishes in an earlier group than
|
|
113
|
+
its descendants, so failed checks still prevent downstream execution. A chain
|
|
114
|
+
with checks on every model therefore retains separate invocations. Explicit
|
|
115
|
+
execution policies keep individual runners, as does a graph that would become
|
|
116
|
+
cyclic after grouping.
|
|
117
|
+
|
|
118
|
+
Native `phlo/full_refresh_asset` requests fail explicitly. The engine's `--force`
|
|
119
|
+
rebuilds a plan but does not replace existing incremental data. Use an
|
|
120
|
+
engine-supported replacement plan for that operation, not a forced append.
|
|
121
|
+
|
|
122
|
+
A group is one Dagster step, so its retries repeat the selected models.
|
|
123
|
+
Existing asset selections and schedules need no changes. Set
|
|
124
|
+
`PHLO_TRANSFORM_GROUP_EXECUTION=false` to keep legacy per-model Dagster op
|
|
125
|
+
configuration. Attached checks execute with their owning assets; check-only
|
|
126
|
+
selections are rejected before provider execution.
|
|
127
|
+
|
|
128
|
+
In orchestrated runs the adapter comes from the resolved Phlo query engine:
|
|
129
|
+
the Trino endpoint and catalog come from the query-engine capability, and a
|
|
130
|
+
non-main runtime ref is provisioned to its own catalog through
|
|
131
|
+
`RefQueryCatalogManager` before the engine runs. Without orchestration the
|
|
132
|
+
engine decides: Trino when an endpoint is configured, otherwise DuckDB.
|
|
133
|
+
|
|
134
|
+
## Observatory candidate execution
|
|
135
|
+
|
|
136
|
+
The editor saves immutable SQL revisions and runs them on isolated candidate
|
|
137
|
+
branches. Compilation uses the environment's configured preview reference,
|
|
138
|
+
catalog, endpoint, and identity. Custom primary references are supported when
|
|
139
|
+
that complete binding matches the native provider configuration.
|
|
140
|
+
|
|
141
|
+
A rejected read-only compilation does not imply an uncertain write. After
|
|
142
|
+
correcting its prerequisites, you can confirm a new execution with a fresh
|
|
143
|
+
idempotency key. Once candidate provisioning or execution starts, an uncertain
|
|
144
|
+
outcome remains blocked, including requests with a different key.
|
|
145
|
+
|
|
146
|
+
Successful executions retain their native report. When the candidate result
|
|
147
|
+
hash is recorded, the authorised execution preview endpoint can retry result
|
|
148
|
+
readback without rerunning the model.
|
|
149
|
+
The authorised execution `POST .../executions/{execution_id}/reconcile`
|
|
150
|
+
endpoint rereads the recorded report and pinned provider snapshot. It can
|
|
151
|
+
resolve an uncertain operation response only when a durable native success
|
|
152
|
+
report, output branch hash, source revision, and provider evidence agree.
|
|
153
|
+
It never invokes the engine. A `running` execution, a missing report, or a
|
|
154
|
+
missing output hash remains unresolved. Preserve those records, branches,
|
|
155
|
+
and catalogs for operator investigation. Do not clear idempotency state or
|
|
156
|
+
assume that a timeout means no write occurred.
|
|
157
|
+
|
|
158
|
+
Execution history exposes stale evidence separately from current release
|
|
159
|
+
eligibility. A changed base does not hide a pinned candidate result, but a
|
|
160
|
+
changed candidate hash or revoked source access hides result rows.
|
|
161
|
+
|
|
162
|
+
Draft revisions and terminal executions support authorised `POST .../archive`
|
|
163
|
+
with `{"archived": true, "reason": "Superseded review"}`. Setting `archived` to
|
|
164
|
+
`false` restores visibility. History accepts `include_archived=true`.
|
|
165
|
+
Archiving retains all source bytes, reports, candidate bindings, and operation
|
|
166
|
+
fences. It does not reclaim disk space or reset the global history limits.
|
|
167
|
+
Archived revisions cannot compile, execute, or extend the draft until restored.
|
|
168
|
+
|
|
169
|
+
Archived revisions and terminal executions also support `POST .../reclaim` with
|
|
170
|
+
`confirmed: true`, a reason, and `expected_revision_digest`. This action
|
|
171
|
+
irreversibly deletes the local workspace copy. It retains SQL for source
|
|
172
|
+
handoff, per-file checksums, revision identities, reports, candidate bindings,
|
|
173
|
+
and operation claims. Completed reclamation frees a retained-workspace slot;
|
|
174
|
+
archive alone does not. History still includes the durable tombstone.
|
|
175
|
+
|
|
176
|
+
Reclamation refuses prepared, running, or unknown executions, unknown reference
|
|
177
|
+
metadata, and pending or unknown operation claims. The API holds the operation
|
|
178
|
+
claim transaction lock during local deletion. Repeating confirmed reclamation
|
|
179
|
+
can finish interrupted local cleanup, but never reruns a provider mutation.
|
|
180
|
+
Reclaimed revisions and executions cannot be restored or executed again.
|
|
181
|
+
No remote branches, tables, catalogs, or live data are deleted.
|
|
182
|
+
|
|
183
|
+
The API now applies configured `audit.retention_years` to supporting payloads
|
|
184
|
+
as well as the existing audit-preservation checks. Native workspace age uses
|
|
185
|
+
the later of creation and archival, since native output files can be newer
|
|
186
|
+
than preparation. The minimum expires at the calendar-year anniversary;
|
|
187
|
+
29 February expires on 1 March in a non-leap year. Unset retention adds no
|
|
188
|
+
age minimum. Invalid timestamps or unavailable evidence block reclamation.
|
|
189
|
+
This check does not expire audit records or tombstones.
|
|
190
|
+
|
|
191
|
+
`GET .../drafts/{draft_id}/{revision}/handoff` exports the saved SQL with its
|
|
192
|
+
project-relative path and checksum. The optional `execution_id` includes
|
|
193
|
+
candidate evidence and links to existing branch checks, trial merge, and
|
|
194
|
+
signed merge operations. These links describe POST operations, not automatic
|
|
195
|
+
actions. Source review and a source commit remain separate from the signed
|
|
196
|
+
data release. The handoff never publishes SQL or promotes candidate data.
|
|
197
|
+
|
|
198
|
+
## CLI
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
phlo transform status # active provider, workspace, counts
|
|
202
|
+
phlo transform check # compile and diagnose
|
|
203
|
+
phlo transform plan --select analytics.orders
|
|
204
|
+
phlo transform run --select analytics.orders --force
|
|
205
|
+
phlo transform test --select analytics.orders
|
|
206
|
+
phlo transform list
|
|
207
|
+
phlo transform explain analytics.orders
|
|
208
|
+
phlo transform lineage --format openlineage
|
|
209
|
+
phlo transform doctor
|
|
210
|
+
phlo transform translate --from dbt --out generated/
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Every command forwards `--json` to the engine.
|
|
214
|
+
|
|
215
|
+
## Switching providers in place
|
|
216
|
+
|
|
217
|
+
A project can keep a dbt project and a native workspace side by side:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
phlo transform use phlo-transform # writes .phlo/transformation-provider
|
|
221
|
+
phlo services restart dagster # reload the asset graph
|
|
222
|
+
phlo transform use dbt # switch back
|
|
223
|
+
phlo transform use --unset # follow install defaults
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
Selection order is `PHLO_TRANSFORMATION_PROVIDER` (process or project env),
|
|
227
|
+
then `.phlo/transformation-provider`, then install defaults. Unset means dbt
|
|
228
|
+
owns the graph when a dbt project exists; otherwise the native workspace does.
|
|
229
|
+
Only one provider contributes transform assets at a time, so asset keys cannot
|
|
230
|
+
collide.
|
|
231
|
+
|
|
232
|
+
## Configuration
|
|
233
|
+
|
|
234
|
+
| Variable | Purpose |
|
|
235
|
+
| --- | --- |
|
|
236
|
+
| `PHLO_TRANSFORMATION_PROVIDER` | Active provider: `dbt` or `phlo-transform`. |
|
|
237
|
+
| `PHLO_TRANSFORM_BIN` | Engine executable path. |
|
|
238
|
+
| `PHLO_TRANSFORM_ROOT` | Workspace containing `phlo.toml`. |
|
|
239
|
+
| `PHLO_TRANSFORM_ADAPTER` | `trino` or `duckdb`; inferred otherwise. |
|
|
240
|
+
| `PHLO_TRANSFORM_DUCKDB_PATH` | DuckDB database file for local runs. |
|
|
241
|
+
| `PHLO_TRANSFORM_TRINO_ENDPOINT` | Trino endpoint override. |
|
|
242
|
+
| `PHLO_TRANSFORM_TRINO_USER` / `_PASSWORD` / `_CATALOG` / `_SCHEMA` | Trino connection overrides. |
|
|
243
|
+
| `PHLO_TRANSFORM_JOBS` | Maximum concurrent model builds. |
|
|
244
|
+
| `PHLO_TRANSFORM_GROUP_EXECUTION` | Group compatible workspace assets, `true` by default; `false` retains legacy per-model Dagster ops. |
|
|
245
|
+
| `PHLO_TRANSFORM_ASSET_KEY` | `leaf` (default) or `logical`. |
|
|
246
|
+
| `PHLO_TRANSFORM_RELEASE_VERSION` | Release installed by `phlo transform install`. |
|
|
247
|
+
| `PHLO_TRANSFORM_TIMEOUT_SECONDS` | Per-command timeout. |
|
|
248
|
+
|
|
249
|
+
Values may live in `phlo.yaml` `env:`, `.phlo/.env`, or the process
|
|
250
|
+
environment; earlier sources are overridden by later ones.
|
|
251
|
+
|
|
252
|
+
## Support status
|
|
253
|
+
|
|
254
|
+
This package is `outside_v1` / `development_only` in the support manifest: it is
|
|
255
|
+
production-shaped and tested, but not part of the v1 support contract.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
phlo_transform/__init__.py,sha256=6-gt4_UvD2CTX1R1L0GUMuO3v05c4BCYT2KKEy4JaRQ,704
|
|
2
|
+
phlo_transform/assets.py,sha256=66zIOwmbH3gzX9jzuFWwYR8sHOIqB6ySnQWaVs8UNuA,20038
|
|
3
|
+
phlo_transform/authorization.py,sha256=3AVjzQ9u4nTmlJxAgdxxMqMXAzZTtOnK6fq2d-sftAo,1820
|
|
4
|
+
phlo_transform/binary.py,sha256=ypg53i2ybX-M38J6_f_7hk9xPzPzou9Q2fkV7ij3dx8,11484
|
|
5
|
+
phlo_transform/cli_plugin.py,sha256=mpnlIDPL37FeNghJicVIfvE9CGQqq4t0r0pwBDvdGj4,14895
|
|
6
|
+
phlo_transform/editor.py,sha256=WQWixr9_pW04-76fFKGWV_OsVTGP35Ez1Y-ELvYQk0Q,25819
|
|
7
|
+
phlo_transform/editor_execution.py,sha256=aen4OL7Y8RQKp4-Gzkum5QBkAuL2G4Y3_y2do2S2Sgw,17411
|
|
8
|
+
phlo_transform/plugin.py,sha256=yOHGfY27Q02fJ5T4ffEA59ijdH2wq-t6AK6VyLwMkfs,1767
|
|
9
|
+
phlo_transform/settings.py,sha256=s0ixhcYN_2cslFyOSRDWP63rFz2-Un0n2eZhJPE0t1I,5119
|
|
10
|
+
phlo_transform/translate.py,sha256=q6hmL7Nu1pxHfB2wGTIRuk6TZA3w0I_D6LuSykQNXuc,2487
|
|
11
|
+
phlo_transform/workspace.py,sha256=gq9emvgesxhXgVgoVWu0PJ68uILPFwptK3ktJ2h9DX8,9652
|
|
12
|
+
phlo_transform-0.2.0.dist-info/METADATA,sha256=1DlRu0YdsLEAXD4P9LmI2Ysag2yxIV_xW7JMDq3t-hg,11466
|
|
13
|
+
phlo_transform-0.2.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
14
|
+
phlo_transform-0.2.0.dist-info/entry_points.txt,sha256=8lYZqQF3aF5-V013ozL0oDnmXyI0_2WDa1Nsx2dk6us,249
|
|
15
|
+
phlo_transform-0.2.0.dist-info/top_level.txt,sha256=h4_xxzPA3V7jH0mq2AssRqicOz8hMAKKQFhdJ5ti8n8,15
|
|
16
|
+
phlo_transform-0.2.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
phlo_transform
|