phlo-transform 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_transform/__init__.py +23 -0
- phlo_transform/assets.py +552 -0
- phlo_transform/authorization.py +62 -0
- phlo_transform/binary.py +343 -0
- phlo_transform/cli_plugin.py +402 -0
- phlo_transform/editor.py +679 -0
- phlo_transform/editor_execution.py +467 -0
- phlo_transform/plugin.py +59 -0
- phlo_transform/settings.py +139 -0
- phlo_transform/translate.py +72 -0
- phlo_transform/workspace.py +289 -0
- phlo_transform-0.2.0.dist-info/METADATA +255 -0
- phlo_transform-0.2.0.dist-info/RECORD +16 -0
- phlo_transform-0.2.0.dist-info/WHEEL +5 -0
- phlo_transform-0.2.0.dist-info/entry_points.txt +8 -0
- phlo_transform-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Phlo Transform integration for Phlo.
|
|
2
|
+
|
|
3
|
+
Provides the ``transform`` asset and transformation providers, the
|
|
4
|
+
``phlo transform`` CLI, and typed access to a Phlo Transform workspace. The
|
|
5
|
+
Rust engine is the source of truth for compilation, planning, execution, and
|
|
6
|
+
tests; this package adapts its JSON reports to Phlo capability specs.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from importlib.metadata import version
|
|
12
|
+
|
|
13
|
+
from phlo_transform.assets import build_transform_asset_specs
|
|
14
|
+
from phlo_transform.workspace import TransformWorkspace, load_workspace
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"TransformWorkspace",
|
|
18
|
+
"__version__",
|
|
19
|
+
"build_transform_asset_specs",
|
|
20
|
+
"load_workspace",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
__version__ = version("phlo-transform")
|
phlo_transform/assets.py
ADDED
|
@@ -0,0 +1,552 @@
|
|
|
1
|
+
"""Asset specifications backed by a Phlo Transform workspace.
|
|
2
|
+
|
|
3
|
+
Each compiled model becomes one asset whose dependencies come from the
|
|
4
|
+
engine's own graph report. Compatible assets share one engine run, with
|
|
5
|
+
per-model materialisations and owned test results. Extra dependency models
|
|
6
|
+
in the engine plan do not emit unrequested asset results.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from dataclasses import replace
|
|
12
|
+
from graphlib import TopologicalSorter
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from phlo.capabilities import (
|
|
17
|
+
AssetCheckSpec,
|
|
18
|
+
AssetSpec,
|
|
19
|
+
CheckResult,
|
|
20
|
+
MaterializeResult,
|
|
21
|
+
RunGroupResult,
|
|
22
|
+
RunGroupSpec,
|
|
23
|
+
RunSpec,
|
|
24
|
+
RuntimeContext,
|
|
25
|
+
resolve_capability,
|
|
26
|
+
routing_from_context,
|
|
27
|
+
)
|
|
28
|
+
from phlo.capabilities.interfaces import RefQueryCatalogManager
|
|
29
|
+
from phlo.config.env import resolve_project_root
|
|
30
|
+
from phlo.config.transformation import active_transformation_provider
|
|
31
|
+
from phlo.exceptions import PhloCapabilitySetupError
|
|
32
|
+
from phlo.logging import get_logger
|
|
33
|
+
from phlo.transform import get_transform_assets
|
|
34
|
+
|
|
35
|
+
from phlo_transform.binary import run_json
|
|
36
|
+
from phlo_transform.settings import TransformSettings, get_settings
|
|
37
|
+
from phlo_transform.workspace import TransformWorkspace, WorkspaceModel, load_workspace
|
|
38
|
+
|
|
39
|
+
logger = get_logger(__name__)
|
|
40
|
+
|
|
41
|
+
DEFAULT_TRINO_ENDPOINT = "http://localhost:8080"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def has_dbt_project(project_root: Path) -> bool:
|
|
45
|
+
"""Return True when a dbt project exists in the project tree.
|
|
46
|
+
|
|
47
|
+
Used only to pick the default owner when the project has not selected a
|
|
48
|
+
transformation provider: existing dbt projects keep the graph they had
|
|
49
|
+
before ``phlo-transform`` was installed. Once selected explicitly, the
|
|
50
|
+
provider marker or environment variable always wins.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
if (project_root / "dbt_project.yml").is_file():
|
|
54
|
+
return True
|
|
55
|
+
for marker in project_root.glob("workflows/**/dbt_project.yml"):
|
|
56
|
+
if "dbt_packages" in marker.parts:
|
|
57
|
+
continue
|
|
58
|
+
return True
|
|
59
|
+
return False
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def provider_is_active(project_root: Path) -> bool:
|
|
63
|
+
"""Return True when the transform provider should own the asset graph."""
|
|
64
|
+
|
|
65
|
+
active = active_transformation_provider(project_root)
|
|
66
|
+
if active == "transform":
|
|
67
|
+
return True
|
|
68
|
+
if active == "dbt":
|
|
69
|
+
return False
|
|
70
|
+
return not has_dbt_project(project_root)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _asset_key(model: WorkspaceModel, mode: str) -> str:
|
|
74
|
+
return model.leaf_name if mode == "leaf" else model.name
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _key_map(workspace: TransformWorkspace, mode: str) -> dict[str, str]:
|
|
78
|
+
keys: dict[str, str] = {}
|
|
79
|
+
owners: dict[str, str] = {}
|
|
80
|
+
for model in workspace.models:
|
|
81
|
+
key = _asset_key(model, mode)
|
|
82
|
+
owner = owners.get(key)
|
|
83
|
+
if owner is not None and owner != model.name:
|
|
84
|
+
raise PhloCapabilitySetupError(
|
|
85
|
+
capability="transform",
|
|
86
|
+
required=True,
|
|
87
|
+
message=(
|
|
88
|
+
f"transform asset key collision: '{key}' is produced by both "
|
|
89
|
+
f"'{owner}' and '{model.name}'"
|
|
90
|
+
),
|
|
91
|
+
suggestions=[
|
|
92
|
+
"Set PHLO_TRANSFORM_ASSET_KEY=logical to keep namespace-qualified keys",
|
|
93
|
+
"Rename one of the models",
|
|
94
|
+
],
|
|
95
|
+
)
|
|
96
|
+
owners[key] = model.name
|
|
97
|
+
keys[model.name] = key
|
|
98
|
+
return keys
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _check_name(test_name: str) -> str:
|
|
102
|
+
return test_name.replace(".", "_").replace(" ", "_")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _check_specs(
|
|
106
|
+
workspace: TransformWorkspace, model: WorkspaceModel, key: str
|
|
107
|
+
) -> list[AssetCheckSpec]:
|
|
108
|
+
return [
|
|
109
|
+
AssetCheckSpec(
|
|
110
|
+
name=_check_name(test.name),
|
|
111
|
+
asset_key=key,
|
|
112
|
+
blocking=True,
|
|
113
|
+
description=f"phlo-transform test {test.name}",
|
|
114
|
+
)
|
|
115
|
+
for test in workspace.tests_for(model.name)
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _query_engine_resolution(runtime: RuntimeContext | None):
|
|
120
|
+
if runtime is None:
|
|
121
|
+
return None
|
|
122
|
+
try:
|
|
123
|
+
return resolve_capability("query_engine", runtime=runtime)
|
|
124
|
+
# reason: capability lookup must not break discovery; settings fallback follows.
|
|
125
|
+
except Exception: # noqa: BLE001 - adapter resolution failure falls back to settings
|
|
126
|
+
logger.warning("phlo_transform_query_engine_resolution_failed", exc_info=True)
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _trino_catalog(
|
|
131
|
+
runtime: RuntimeContext | None, settings: TransformSettings
|
|
132
|
+
) -> tuple[str | None, str | None]:
|
|
133
|
+
"""Return the target catalog and environment ref for a run."""
|
|
134
|
+
|
|
135
|
+
ref: str | None = None
|
|
136
|
+
if runtime is not None:
|
|
137
|
+
routing = routing_from_context(runtime)
|
|
138
|
+
ref = routing.ref
|
|
139
|
+
catalog = settings.trino_catalog
|
|
140
|
+
if catalog is None:
|
|
141
|
+
resolution = _query_engine_resolution(runtime)
|
|
142
|
+
if resolution is not None and resolution.metadata.get("default_catalog"):
|
|
143
|
+
catalog = str(resolution.metadata["default_catalog"])
|
|
144
|
+
if ref and ref != "main":
|
|
145
|
+
resolution = _query_engine_resolution(runtime)
|
|
146
|
+
provider = resolution.provider if resolution is not None else None
|
|
147
|
+
if isinstance(provider, RefQueryCatalogManager):
|
|
148
|
+
catalog = provider.provision_ref_query_catalog(ref)
|
|
149
|
+
return catalog, ref
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def adapter_args(
|
|
153
|
+
runtime: RuntimeContext | None,
|
|
154
|
+
settings: TransformSettings,
|
|
155
|
+
*,
|
|
156
|
+
infer: bool = True,
|
|
157
|
+
) -> list[str]:
|
|
158
|
+
"""Build adapter flags from settings and the resolved query engine.
|
|
159
|
+
|
|
160
|
+
``infer`` distinguishes orchestrated asset runs (where the Phlo query
|
|
161
|
+
engine supplies the target) from the CLI (where the engine's own default
|
|
162
|
+
and error guidance should apply unless an adapter was configured).
|
|
163
|
+
"""
|
|
164
|
+
|
|
165
|
+
adapter = (settings.adapter or "").strip().lower() or None
|
|
166
|
+
resolution = _query_engine_resolution(runtime)
|
|
167
|
+
metadata = resolution.metadata if resolution is not None else {}
|
|
168
|
+
endpoint = settings.trino_endpoint
|
|
169
|
+
if endpoint is None and metadata.get("host"):
|
|
170
|
+
endpoint = f"http://{metadata['host']}:{metadata.get('port', 8080)}"
|
|
171
|
+
if adapter is None:
|
|
172
|
+
if endpoint:
|
|
173
|
+
adapter = "trino"
|
|
174
|
+
elif infer:
|
|
175
|
+
adapter = "duckdb"
|
|
176
|
+
else:
|
|
177
|
+
return []
|
|
178
|
+
if adapter == "duckdb":
|
|
179
|
+
args = ["--adapter", "duckdb"]
|
|
180
|
+
if settings.duckdb_path:
|
|
181
|
+
args.extend(["--duckdb-path", settings.duckdb_path])
|
|
182
|
+
return args
|
|
183
|
+
if adapter != "trino":
|
|
184
|
+
raise PhloCapabilitySetupError(
|
|
185
|
+
capability="transform",
|
|
186
|
+
required=True,
|
|
187
|
+
message=f"unsupported phlo-transform adapter '{adapter}'",
|
|
188
|
+
suggestions=["Set PHLO_TRANSFORM_ADAPTER to trino or duckdb"],
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
args = ["--adapter", "trino", "--trino-endpoint", endpoint or DEFAULT_TRINO_ENDPOINT]
|
|
192
|
+
catalog, ref = _trino_catalog(runtime, settings)
|
|
193
|
+
if catalog:
|
|
194
|
+
args.extend(["--catalog", catalog])
|
|
195
|
+
args.extend(["--trino-catalog", catalog])
|
|
196
|
+
if settings.trino_schema:
|
|
197
|
+
args.extend(["--trino-schema", settings.trino_schema])
|
|
198
|
+
if settings.trino_user:
|
|
199
|
+
args.extend(["--trino-user", settings.trino_user])
|
|
200
|
+
if settings.trino_password:
|
|
201
|
+
args.extend(["--trino-password", settings.trino_password])
|
|
202
|
+
if ref and ref != "main":
|
|
203
|
+
args.extend(["--environment", ref])
|
|
204
|
+
return args
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _column_lineage_metadata(
|
|
208
|
+
workspace: TransformWorkspace,
|
|
209
|
+
model: WorkspaceModel,
|
|
210
|
+
settings: TransformSettings,
|
|
211
|
+
) -> dict[str, Any]:
|
|
212
|
+
"""Attach the engine's column inputs as Dagster column-lineage metadata."""
|
|
213
|
+
try:
|
|
214
|
+
from dagster import MetadataValue, TableColumnDep, TableColumnLineage
|
|
215
|
+
except ImportError:
|
|
216
|
+
return {}
|
|
217
|
+
|
|
218
|
+
payload, invocation = run_json(
|
|
219
|
+
workspace.binary,
|
|
220
|
+
["explain", model.name],
|
|
221
|
+
settings=settings,
|
|
222
|
+
root=workspace.root,
|
|
223
|
+
)
|
|
224
|
+
if payload is None or not invocation.ok:
|
|
225
|
+
logger.warning("phlo_transform_column_lineage_unavailable", model=model.name)
|
|
226
|
+
return {}
|
|
227
|
+
|
|
228
|
+
model_payload = payload.get("model")
|
|
229
|
+
model_payload = model_payload if isinstance(model_payload, dict) else {}
|
|
230
|
+
columns = model_payload.get("columns")
|
|
231
|
+
if not isinstance(columns, list):
|
|
232
|
+
nested = model_payload.get("model")
|
|
233
|
+
columns = nested.get("columns") if isinstance(nested, dict) else None
|
|
234
|
+
if not isinstance(columns, list):
|
|
235
|
+
return {}
|
|
236
|
+
|
|
237
|
+
keys = _key_map(workspace, settings.asset_key_mode)
|
|
238
|
+
lineage: dict[str, list[Any]] = {}
|
|
239
|
+
for column in columns:
|
|
240
|
+
if not isinstance(column, dict):
|
|
241
|
+
continue
|
|
242
|
+
name = column.get("name")
|
|
243
|
+
inputs = column.get("inputs")
|
|
244
|
+
if not isinstance(name, str) or not isinstance(inputs, list):
|
|
245
|
+
continue
|
|
246
|
+
dependencies: list[Any] = []
|
|
247
|
+
for input_ref in inputs:
|
|
248
|
+
if not isinstance(input_ref, str):
|
|
249
|
+
continue
|
|
250
|
+
relation, _, column_name = input_ref.rpartition(".")
|
|
251
|
+
asset_key = keys.get(relation) or workspace.source_assets.get(relation)
|
|
252
|
+
if not relation or not column_name or asset_key is None:
|
|
253
|
+
continue
|
|
254
|
+
dependency = TableColumnDep(asset_key=asset_key, column_name=column_name)
|
|
255
|
+
if dependency not in dependencies:
|
|
256
|
+
dependencies.append(dependency)
|
|
257
|
+
if dependencies:
|
|
258
|
+
lineage[name] = dependencies
|
|
259
|
+
|
|
260
|
+
if not lineage:
|
|
261
|
+
return {}
|
|
262
|
+
return {
|
|
263
|
+
"column_lineage": MetadataValue.column_lineage(
|
|
264
|
+
TableColumnLineage(lineage),
|
|
265
|
+
)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _run_model(
|
|
270
|
+
workspace: TransformWorkspace,
|
|
271
|
+
model: WorkspaceModel,
|
|
272
|
+
key: str,
|
|
273
|
+
settings: TransformSettings,
|
|
274
|
+
runtime: RuntimeContext | None,
|
|
275
|
+
) -> list[MaterializeResult | CheckResult]:
|
|
276
|
+
"""Execute one model through the same report contract as a grouped run."""
|
|
277
|
+
result = _run_models(workspace, {key: model}, settings, runtime)
|
|
278
|
+
outcomes = list(result.outcomes[key])
|
|
279
|
+
if result.error:
|
|
280
|
+
outcomes = [
|
|
281
|
+
replace(outcome, status="failure")
|
|
282
|
+
if isinstance(outcome, MaterializeResult)
|
|
283
|
+
else outcome
|
|
284
|
+
for outcome in outcomes
|
|
285
|
+
]
|
|
286
|
+
return outcomes
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _run_models(
|
|
290
|
+
workspace: TransformWorkspace,
|
|
291
|
+
models: dict[str, WorkspaceModel],
|
|
292
|
+
settings: TransformSettings,
|
|
293
|
+
runtime: RuntimeContext | None,
|
|
294
|
+
) -> RunGroupResult:
|
|
295
|
+
"""Run selected models once, preserving node outcomes and invocation errors."""
|
|
296
|
+
if runtime is not None and runtime.tags.get("phlo/full_refresh_asset") in models:
|
|
297
|
+
raise PhloCapabilitySetupError(
|
|
298
|
+
capability="transform",
|
|
299
|
+
required=True,
|
|
300
|
+
message="Native full refresh is unsupported: --force does not replace incremental data",
|
|
301
|
+
suggestions=[
|
|
302
|
+
"Use an engine-supported replacement plan instead of phlo/full_refresh_asset"
|
|
303
|
+
],
|
|
304
|
+
)
|
|
305
|
+
args = ["run"]
|
|
306
|
+
for model in models.values():
|
|
307
|
+
args.extend(["--select", model.name])
|
|
308
|
+
args.extend(["--jobs", str(settings.jobs), *adapter_args(runtime, settings)])
|
|
309
|
+
payload, invocation = run_json(
|
|
310
|
+
workspace.binary,
|
|
311
|
+
args,
|
|
312
|
+
settings=settings,
|
|
313
|
+
root=workspace.root,
|
|
314
|
+
)
|
|
315
|
+
if payload is None:
|
|
316
|
+
detail = (invocation.stderr or invocation.stdout).strip().splitlines()
|
|
317
|
+
raise PhloCapabilitySetupError(
|
|
318
|
+
capability="transform",
|
|
319
|
+
required=True,
|
|
320
|
+
message=(
|
|
321
|
+
"phlo-transform produced no run report" + (f": {detail[-1]}" if detail else "")
|
|
322
|
+
),
|
|
323
|
+
suggestions=["Run the same command with --json to inspect the failure"],
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
status = payload.get("status")
|
|
327
|
+
return RunGroupResult(
|
|
328
|
+
outcomes={
|
|
329
|
+
key: _model_results(workspace, model, key, settings, payload)
|
|
330
|
+
for key, model in models.items()
|
|
331
|
+
},
|
|
332
|
+
error=(
|
|
333
|
+
f"phlo-transform run failed (exit {invocation.returncode}, status {status})"
|
|
334
|
+
if not invocation.ok or status != "passed"
|
|
335
|
+
else None
|
|
336
|
+
),
|
|
337
|
+
)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _model_results(
|
|
341
|
+
workspace: TransformWorkspace,
|
|
342
|
+
model: WorkspaceModel,
|
|
343
|
+
key: str,
|
|
344
|
+
settings: TransformSettings,
|
|
345
|
+
payload: dict,
|
|
346
|
+
) -> list[MaterializeResult | CheckResult]:
|
|
347
|
+
"""Attribute model and owned-test evidence without inheriting sibling failures."""
|
|
348
|
+
model_report = next(
|
|
349
|
+
(
|
|
350
|
+
item
|
|
351
|
+
for item in payload.get("models") or []
|
|
352
|
+
if isinstance(item, dict) and item.get("model") == model.name
|
|
353
|
+
),
|
|
354
|
+
{},
|
|
355
|
+
)
|
|
356
|
+
status = str(model_report.get("status") or "not_reported")
|
|
357
|
+
test_results = {
|
|
358
|
+
str(item.get("test")): item
|
|
359
|
+
for item in payload.get("tests") or []
|
|
360
|
+
if isinstance(item, dict) and item.get("test")
|
|
361
|
+
}
|
|
362
|
+
lineage_metadata = _column_lineage_metadata(workspace, model, settings)
|
|
363
|
+
|
|
364
|
+
results: list[MaterializeResult | CheckResult] = []
|
|
365
|
+
for test in workspace.tests_for(model.name):
|
|
366
|
+
outcome = test_results.get(test.name)
|
|
367
|
+
passed = outcome is not None and str(outcome.get("status")) == "passed"
|
|
368
|
+
results.append(
|
|
369
|
+
CheckResult(
|
|
370
|
+
check_name=_check_name(test.name),
|
|
371
|
+
asset_key=key,
|
|
372
|
+
passed=passed,
|
|
373
|
+
severity="error",
|
|
374
|
+
metadata={
|
|
375
|
+
"test": test.name,
|
|
376
|
+
"status": str(outcome.get("status")) if outcome else "not_reported",
|
|
377
|
+
"row_count": outcome.get("row_count") if outcome else None,
|
|
378
|
+
"error": (
|
|
379
|
+
outcome.get("failure") or outcome.get("error")
|
|
380
|
+
if outcome
|
|
381
|
+
else "test did not report a result"
|
|
382
|
+
),
|
|
383
|
+
},
|
|
384
|
+
)
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
results.append(
|
|
388
|
+
MaterializeResult(
|
|
389
|
+
status=(
|
|
390
|
+
"success"
|
|
391
|
+
if status in {"passed", "skipped", "cached"}
|
|
392
|
+
and all(result.passed for result in results if isinstance(result, CheckResult))
|
|
393
|
+
else "failure"
|
|
394
|
+
),
|
|
395
|
+
metadata={
|
|
396
|
+
"transform_model": model.name,
|
|
397
|
+
"transform_target": model.target,
|
|
398
|
+
"transform_materialization": model.materialization,
|
|
399
|
+
"transform_version": model.version,
|
|
400
|
+
"transform_workspace": str(workspace.root),
|
|
401
|
+
"transform_run_id": payload.get("run_id"),
|
|
402
|
+
"transform_plan_id": payload.get("plan_id"),
|
|
403
|
+
"transform_status": status,
|
|
404
|
+
"transform_action": model_report.get("action"),
|
|
405
|
+
"transform_reasons": model_report.get("reasons") or [],
|
|
406
|
+
"transform_previous_version": model_report.get("previous_version"),
|
|
407
|
+
"transform_desired_version": model_report.get("desired_version"),
|
|
408
|
+
"transform_sql_hash": model_report.get("sql_hash"),
|
|
409
|
+
"transform_counts": payload.get("counts") or {},
|
|
410
|
+
"transform_failure": model_report.get("failure"),
|
|
411
|
+
"transform_duration_ms": model_report.get("duration_ms"),
|
|
412
|
+
"transform_query_id": model_report.get("query_id"),
|
|
413
|
+
"transform_tests": len(workspace.tests_for(model.name)),
|
|
414
|
+
**lineage_metadata,
|
|
415
|
+
},
|
|
416
|
+
)
|
|
417
|
+
)
|
|
418
|
+
return results
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def build_workspace_asset_specs(
|
|
422
|
+
project_root: Path,
|
|
423
|
+
settings: TransformSettings,
|
|
424
|
+
) -> list[AssetSpec]:
|
|
425
|
+
"""Build asset specs for every model in the project's transform workspace."""
|
|
426
|
+
|
|
427
|
+
workspace = load_workspace(project_root, settings)
|
|
428
|
+
if workspace is None:
|
|
429
|
+
return []
|
|
430
|
+
mode = settings.asset_key_mode
|
|
431
|
+
keys = _key_map(workspace, mode)
|
|
432
|
+
models = {keys[model.name]: model for model in workspace.models}
|
|
433
|
+
|
|
434
|
+
def _group_runner(runtime: RuntimeContext, selected: frozenset[str]) -> RunGroupResult:
|
|
435
|
+
return _run_models(
|
|
436
|
+
workspace,
|
|
437
|
+
{key: model for key, model in models.items() if key in selected},
|
|
438
|
+
settings,
|
|
439
|
+
runtime,
|
|
440
|
+
)
|
|
441
|
+
|
|
442
|
+
# Native tests run after all builds, so a checked parent must finish in
|
|
443
|
+
# an earlier Dagster step before its descendants can execute.
|
|
444
|
+
graph = {
|
|
445
|
+
key: {keys[dep] for dep in model.depends_on if dep in keys} for key, model in models.items()
|
|
446
|
+
}
|
|
447
|
+
stages: dict[str, int] = {}
|
|
448
|
+
for key in TopologicalSorter(graph).static_order():
|
|
449
|
+
stages[key] = max(
|
|
450
|
+
(stages[dep] + bool(workspace.tests_for(models[dep].name)) for dep in graph[key]),
|
|
451
|
+
default=0,
|
|
452
|
+
)
|
|
453
|
+
run_groups = {
|
|
454
|
+
stage: RunGroupSpec(name=f"transform_{stage}", fn=_group_runner)
|
|
455
|
+
for stage in set(stages.values())
|
|
456
|
+
}
|
|
457
|
+
specs: list[AssetSpec] = []
|
|
458
|
+
for model in workspace.models:
|
|
459
|
+
key = keys[model.name]
|
|
460
|
+
deps = [keys[dep] for dep in model.depends_on if dep in keys]
|
|
461
|
+
source_assets = [
|
|
462
|
+
workspace.source_assets[source]
|
|
463
|
+
for source in model.sources
|
|
464
|
+
if source in workspace.source_assets
|
|
465
|
+
]
|
|
466
|
+
deps.extend(asset for asset in source_assets if asset not in deps)
|
|
467
|
+
missing = [dep for dep in model.depends_on if dep not in keys]
|
|
468
|
+
if missing:
|
|
469
|
+
logger.warning(
|
|
470
|
+
"phlo_transform_model_dependency_missing",
|
|
471
|
+
model=model.name,
|
|
472
|
+
missing=missing,
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
def _runner(
|
|
476
|
+
runtime: RuntimeContext,
|
|
477
|
+
bound_model: WorkspaceModel = model,
|
|
478
|
+
bound_key: str = key,
|
|
479
|
+
) -> list[MaterializeResult | CheckResult]:
|
|
480
|
+
return _run_model(
|
|
481
|
+
workspace,
|
|
482
|
+
bound_model,
|
|
483
|
+
bound_key,
|
|
484
|
+
settings,
|
|
485
|
+
runtime,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
specs.append(
|
|
489
|
+
AssetSpec(
|
|
490
|
+
key=key,
|
|
491
|
+
group=model.group,
|
|
492
|
+
description=f"Phlo Transform model {model.name}",
|
|
493
|
+
kinds={"transform", "sql", model.materialization},
|
|
494
|
+
tags={"tool": "phlo-transform", "provider": "transform"},
|
|
495
|
+
metadata={
|
|
496
|
+
"transform_engine": "phlo-transform",
|
|
497
|
+
"transform_model": model.name,
|
|
498
|
+
"transform_target": model.target,
|
|
499
|
+
"transform_materialization": model.materialization,
|
|
500
|
+
"transform_version": model.version,
|
|
501
|
+
"transform_path": model.path,
|
|
502
|
+
"transform_tags": list(model.tags),
|
|
503
|
+
"transform_test_names": [test.name for test in workspace.tests_for(model.name)],
|
|
504
|
+
"transform_sources": list(model.sources),
|
|
505
|
+
"transform_source_assets": source_assets,
|
|
506
|
+
"transform_workspace": str(workspace.root),
|
|
507
|
+
},
|
|
508
|
+
deps=deps,
|
|
509
|
+
checks=_check_specs(workspace, model, key),
|
|
510
|
+
run=RunSpec(
|
|
511
|
+
fn=_runner,
|
|
512
|
+
group=run_groups[stages[key]] if settings.group_execution else None,
|
|
513
|
+
),
|
|
514
|
+
)
|
|
515
|
+
)
|
|
516
|
+
logger.info(
|
|
517
|
+
"phlo_transform_asset_specs_built",
|
|
518
|
+
workspace_root=str(workspace.root),
|
|
519
|
+
spec_count=len(specs),
|
|
520
|
+
)
|
|
521
|
+
return specs
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def build_transform_asset_specs(project_root: Path | None = None) -> list[AssetSpec]:
|
|
525
|
+
"""Return every transform asset: Python declarations plus workspace models.
|
|
526
|
+
|
|
527
|
+
``phlo.transform.sql`` declarations are ordinary Python assets and stay
|
|
528
|
+
available regardless of which execution engine owns SQL workspaces. The
|
|
529
|
+
native workspace contributes assets only when it is the active provider.
|
|
530
|
+
"""
|
|
531
|
+
|
|
532
|
+
root = resolve_project_root(project_root)
|
|
533
|
+
specs = list(get_transform_assets())
|
|
534
|
+
settings = get_settings(root)
|
|
535
|
+
if not provider_is_active(root):
|
|
536
|
+
logger.info(
|
|
537
|
+
"phlo_transform_provider_inactive",
|
|
538
|
+
active_provider=active_transformation_provider(root),
|
|
539
|
+
workspace=settings.root,
|
|
540
|
+
)
|
|
541
|
+
return specs
|
|
542
|
+
specs.extend(build_workspace_asset_specs(root, settings))
|
|
543
|
+
return specs
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
__all__ = [
|
|
547
|
+
"adapter_args",
|
|
548
|
+
"build_transform_asset_specs",
|
|
549
|
+
"build_workspace_asset_specs",
|
|
550
|
+
"has_dbt_project",
|
|
551
|
+
"provider_is_active",
|
|
552
|
+
]
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""phlo_transform CLI authorization table.
|
|
2
|
+
|
|
3
|
+
Declares which transform CLI commands mutate versus read so regulated
|
|
4
|
+
deployments gate engine runs, workspace translation, provider switching, and
|
|
5
|
+
binary installation through the same surface contract as other providers.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from phlo.cli.authorization import CliSurfaceAdapter, cli_surface_adapter_class
|
|
11
|
+
|
|
12
|
+
SURFACE_NAME = "phlo-transform"
|
|
13
|
+
FRAMEWORK_TYPE = "cli"
|
|
14
|
+
MUTATION_COMMANDS: frozenset[str] = frozenset(
|
|
15
|
+
{
|
|
16
|
+
"transform.install",
|
|
17
|
+
"transform.run",
|
|
18
|
+
"transform.translate",
|
|
19
|
+
"transform.use",
|
|
20
|
+
}
|
|
21
|
+
)
|
|
22
|
+
READ_COMMANDS: frozenset[str] = frozenset(
|
|
23
|
+
{
|
|
24
|
+
"transform.check",
|
|
25
|
+
"transform.doctor",
|
|
26
|
+
"transform.explain",
|
|
27
|
+
"transform.lineage",
|
|
28
|
+
"transform.list",
|
|
29
|
+
"transform.plan",
|
|
30
|
+
"transform.status",
|
|
31
|
+
"transform.test",
|
|
32
|
+
}
|
|
33
|
+
)
|
|
34
|
+
COMMAND_RESOURCE_MAP: dict[str, str] = {
|
|
35
|
+
"transform.install": "project",
|
|
36
|
+
"transform.run": "dataset",
|
|
37
|
+
"transform.translate": "project",
|
|
38
|
+
"transform.use": "project",
|
|
39
|
+
}
|
|
40
|
+
COMMAND_ACTION_MAP: dict[str, str] = {
|
|
41
|
+
"transform.install": "project.create",
|
|
42
|
+
"transform.run": "dataset.write",
|
|
43
|
+
"transform.translate": "project.create",
|
|
44
|
+
"transform.use": "project.update",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
TransformSurfaceAdapter = cli_surface_adapter_class(
|
|
48
|
+
"TransformSurfaceAdapter",
|
|
49
|
+
surface_name=SURFACE_NAME,
|
|
50
|
+
mutation_commands=MUTATION_COMMANDS,
|
|
51
|
+
read_commands=READ_COMMANDS,
|
|
52
|
+
command_resource_map=COMMAND_RESOURCE_MAP,
|
|
53
|
+
command_action_map=COMMAND_ACTION_MAP,
|
|
54
|
+
default_action_prefix="transform",
|
|
55
|
+
default_resource_prefix="transform",
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def get_transform_adapter() -> CliSurfaceAdapter:
|
|
60
|
+
"""Return the shared TransformSurfaceAdapter instance for this surface."""
|
|
61
|
+
|
|
62
|
+
return TransformSurfaceAdapter.get_instance()
|