phlo-transform 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ """Phlo Transform integration for Phlo.
2
+
3
+ Provides the ``transform`` asset and transformation providers, the
4
+ ``phlo transform`` CLI, and typed access to a Phlo Transform workspace. The
5
+ Rust engine is the source of truth for compilation, planning, execution, and
6
+ tests; this package adapts its JSON reports to Phlo capability specs.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from importlib.metadata import version
12
+
13
+ from phlo_transform.assets import build_transform_asset_specs
14
+ from phlo_transform.workspace import TransformWorkspace, load_workspace
15
+
16
+ __all__ = [
17
+ "TransformWorkspace",
18
+ "__version__",
19
+ "build_transform_asset_specs",
20
+ "load_workspace",
21
+ ]
22
+
23
+ __version__ = version("phlo-transform")
@@ -0,0 +1,552 @@
1
+ """Asset specifications backed by a Phlo Transform workspace.
2
+
3
+ Each compiled model becomes one asset whose dependencies come from the
4
+ engine's own graph report. Compatible assets share one engine run, with
5
+ per-model materialisations and owned test results. Extra dependency models
6
+ in the engine plan do not emit unrequested asset results.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ from dataclasses import replace
12
+ from graphlib import TopologicalSorter
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ from phlo.capabilities import (
17
+ AssetCheckSpec,
18
+ AssetSpec,
19
+ CheckResult,
20
+ MaterializeResult,
21
+ RunGroupResult,
22
+ RunGroupSpec,
23
+ RunSpec,
24
+ RuntimeContext,
25
+ resolve_capability,
26
+ routing_from_context,
27
+ )
28
+ from phlo.capabilities.interfaces import RefQueryCatalogManager
29
+ from phlo.config.env import resolve_project_root
30
+ from phlo.config.transformation import active_transformation_provider
31
+ from phlo.exceptions import PhloCapabilitySetupError
32
+ from phlo.logging import get_logger
33
+ from phlo.transform import get_transform_assets
34
+
35
+ from phlo_transform.binary import run_json
36
+ from phlo_transform.settings import TransformSettings, get_settings
37
+ from phlo_transform.workspace import TransformWorkspace, WorkspaceModel, load_workspace
38
+
39
+ logger = get_logger(__name__)
40
+
41
+ DEFAULT_TRINO_ENDPOINT = "http://localhost:8080"
42
+
43
+
44
+ def has_dbt_project(project_root: Path) -> bool:
45
+ """Return True when a dbt project exists in the project tree.
46
+
47
+ Used only to pick the default owner when the project has not selected a
48
+ transformation provider: existing dbt projects keep the graph they had
49
+ before ``phlo-transform`` was installed. Once selected explicitly, the
50
+ provider marker or environment variable always wins.
51
+ """
52
+
53
+ if (project_root / "dbt_project.yml").is_file():
54
+ return True
55
+ for marker in project_root.glob("workflows/**/dbt_project.yml"):
56
+ if "dbt_packages" in marker.parts:
57
+ continue
58
+ return True
59
+ return False
60
+
61
+
62
+ def provider_is_active(project_root: Path) -> bool:
63
+ """Return True when the transform provider should own the asset graph."""
64
+
65
+ active = active_transformation_provider(project_root)
66
+ if active == "transform":
67
+ return True
68
+ if active == "dbt":
69
+ return False
70
+ return not has_dbt_project(project_root)
71
+
72
+
73
+ def _asset_key(model: WorkspaceModel, mode: str) -> str:
74
+ return model.leaf_name if mode == "leaf" else model.name
75
+
76
+
77
+ def _key_map(workspace: TransformWorkspace, mode: str) -> dict[str, str]:
78
+ keys: dict[str, str] = {}
79
+ owners: dict[str, str] = {}
80
+ for model in workspace.models:
81
+ key = _asset_key(model, mode)
82
+ owner = owners.get(key)
83
+ if owner is not None and owner != model.name:
84
+ raise PhloCapabilitySetupError(
85
+ capability="transform",
86
+ required=True,
87
+ message=(
88
+ f"transform asset key collision: '{key}' is produced by both "
89
+ f"'{owner}' and '{model.name}'"
90
+ ),
91
+ suggestions=[
92
+ "Set PHLO_TRANSFORM_ASSET_KEY=logical to keep namespace-qualified keys",
93
+ "Rename one of the models",
94
+ ],
95
+ )
96
+ owners[key] = model.name
97
+ keys[model.name] = key
98
+ return keys
99
+
100
+
101
+ def _check_name(test_name: str) -> str:
102
+ return test_name.replace(".", "_").replace(" ", "_")
103
+
104
+
105
+ def _check_specs(
106
+ workspace: TransformWorkspace, model: WorkspaceModel, key: str
107
+ ) -> list[AssetCheckSpec]:
108
+ return [
109
+ AssetCheckSpec(
110
+ name=_check_name(test.name),
111
+ asset_key=key,
112
+ blocking=True,
113
+ description=f"phlo-transform test {test.name}",
114
+ )
115
+ for test in workspace.tests_for(model.name)
116
+ ]
117
+
118
+
119
+ def _query_engine_resolution(runtime: RuntimeContext | None):
120
+ if runtime is None:
121
+ return None
122
+ try:
123
+ return resolve_capability("query_engine", runtime=runtime)
124
+ # reason: capability lookup must not break discovery; settings fallback follows.
125
+ except Exception: # noqa: BLE001 - adapter resolution failure falls back to settings
126
+ logger.warning("phlo_transform_query_engine_resolution_failed", exc_info=True)
127
+ return None
128
+
129
+
130
+ def _trino_catalog(
131
+ runtime: RuntimeContext | None, settings: TransformSettings
132
+ ) -> tuple[str | None, str | None]:
133
+ """Return the target catalog and environment ref for a run."""
134
+
135
+ ref: str | None = None
136
+ if runtime is not None:
137
+ routing = routing_from_context(runtime)
138
+ ref = routing.ref
139
+ catalog = settings.trino_catalog
140
+ if catalog is None:
141
+ resolution = _query_engine_resolution(runtime)
142
+ if resolution is not None and resolution.metadata.get("default_catalog"):
143
+ catalog = str(resolution.metadata["default_catalog"])
144
+ if ref and ref != "main":
145
+ resolution = _query_engine_resolution(runtime)
146
+ provider = resolution.provider if resolution is not None else None
147
+ if isinstance(provider, RefQueryCatalogManager):
148
+ catalog = provider.provision_ref_query_catalog(ref)
149
+ return catalog, ref
150
+
151
+
152
+ def adapter_args(
153
+ runtime: RuntimeContext | None,
154
+ settings: TransformSettings,
155
+ *,
156
+ infer: bool = True,
157
+ ) -> list[str]:
158
+ """Build adapter flags from settings and the resolved query engine.
159
+
160
+ ``infer`` distinguishes orchestrated asset runs (where the Phlo query
161
+ engine supplies the target) from the CLI (where the engine's own default
162
+ and error guidance should apply unless an adapter was configured).
163
+ """
164
+
165
+ adapter = (settings.adapter or "").strip().lower() or None
166
+ resolution = _query_engine_resolution(runtime)
167
+ metadata = resolution.metadata if resolution is not None else {}
168
+ endpoint = settings.trino_endpoint
169
+ if endpoint is None and metadata.get("host"):
170
+ endpoint = f"http://{metadata['host']}:{metadata.get('port', 8080)}"
171
+ if adapter is None:
172
+ if endpoint:
173
+ adapter = "trino"
174
+ elif infer:
175
+ adapter = "duckdb"
176
+ else:
177
+ return []
178
+ if adapter == "duckdb":
179
+ args = ["--adapter", "duckdb"]
180
+ if settings.duckdb_path:
181
+ args.extend(["--duckdb-path", settings.duckdb_path])
182
+ return args
183
+ if adapter != "trino":
184
+ raise PhloCapabilitySetupError(
185
+ capability="transform",
186
+ required=True,
187
+ message=f"unsupported phlo-transform adapter '{adapter}'",
188
+ suggestions=["Set PHLO_TRANSFORM_ADAPTER to trino or duckdb"],
189
+ )
190
+
191
+ args = ["--adapter", "trino", "--trino-endpoint", endpoint or DEFAULT_TRINO_ENDPOINT]
192
+ catalog, ref = _trino_catalog(runtime, settings)
193
+ if catalog:
194
+ args.extend(["--catalog", catalog])
195
+ args.extend(["--trino-catalog", catalog])
196
+ if settings.trino_schema:
197
+ args.extend(["--trino-schema", settings.trino_schema])
198
+ if settings.trino_user:
199
+ args.extend(["--trino-user", settings.trino_user])
200
+ if settings.trino_password:
201
+ args.extend(["--trino-password", settings.trino_password])
202
+ if ref and ref != "main":
203
+ args.extend(["--environment", ref])
204
+ return args
205
+
206
+
207
+ def _column_lineage_metadata(
208
+ workspace: TransformWorkspace,
209
+ model: WorkspaceModel,
210
+ settings: TransformSettings,
211
+ ) -> dict[str, Any]:
212
+ """Attach the engine's column inputs as Dagster column-lineage metadata."""
213
+ try:
214
+ from dagster import MetadataValue, TableColumnDep, TableColumnLineage
215
+ except ImportError:
216
+ return {}
217
+
218
+ payload, invocation = run_json(
219
+ workspace.binary,
220
+ ["explain", model.name],
221
+ settings=settings,
222
+ root=workspace.root,
223
+ )
224
+ if payload is None or not invocation.ok:
225
+ logger.warning("phlo_transform_column_lineage_unavailable", model=model.name)
226
+ return {}
227
+
228
+ model_payload = payload.get("model")
229
+ model_payload = model_payload if isinstance(model_payload, dict) else {}
230
+ columns = model_payload.get("columns")
231
+ if not isinstance(columns, list):
232
+ nested = model_payload.get("model")
233
+ columns = nested.get("columns") if isinstance(nested, dict) else None
234
+ if not isinstance(columns, list):
235
+ return {}
236
+
237
+ keys = _key_map(workspace, settings.asset_key_mode)
238
+ lineage: dict[str, list[Any]] = {}
239
+ for column in columns:
240
+ if not isinstance(column, dict):
241
+ continue
242
+ name = column.get("name")
243
+ inputs = column.get("inputs")
244
+ if not isinstance(name, str) or not isinstance(inputs, list):
245
+ continue
246
+ dependencies: list[Any] = []
247
+ for input_ref in inputs:
248
+ if not isinstance(input_ref, str):
249
+ continue
250
+ relation, _, column_name = input_ref.rpartition(".")
251
+ asset_key = keys.get(relation) or workspace.source_assets.get(relation)
252
+ if not relation or not column_name or asset_key is None:
253
+ continue
254
+ dependency = TableColumnDep(asset_key=asset_key, column_name=column_name)
255
+ if dependency not in dependencies:
256
+ dependencies.append(dependency)
257
+ if dependencies:
258
+ lineage[name] = dependencies
259
+
260
+ if not lineage:
261
+ return {}
262
+ return {
263
+ "column_lineage": MetadataValue.column_lineage(
264
+ TableColumnLineage(lineage),
265
+ )
266
+ }
267
+
268
+
269
+ def _run_model(
270
+ workspace: TransformWorkspace,
271
+ model: WorkspaceModel,
272
+ key: str,
273
+ settings: TransformSettings,
274
+ runtime: RuntimeContext | None,
275
+ ) -> list[MaterializeResult | CheckResult]:
276
+ """Execute one model through the same report contract as a grouped run."""
277
+ result = _run_models(workspace, {key: model}, settings, runtime)
278
+ outcomes = list(result.outcomes[key])
279
+ if result.error:
280
+ outcomes = [
281
+ replace(outcome, status="failure")
282
+ if isinstance(outcome, MaterializeResult)
283
+ else outcome
284
+ for outcome in outcomes
285
+ ]
286
+ return outcomes
287
+
288
+
289
+ def _run_models(
290
+ workspace: TransformWorkspace,
291
+ models: dict[str, WorkspaceModel],
292
+ settings: TransformSettings,
293
+ runtime: RuntimeContext | None,
294
+ ) -> RunGroupResult:
295
+ """Run selected models once, preserving node outcomes and invocation errors."""
296
+ if runtime is not None and runtime.tags.get("phlo/full_refresh_asset") in models:
297
+ raise PhloCapabilitySetupError(
298
+ capability="transform",
299
+ required=True,
300
+ message="Native full refresh is unsupported: --force does not replace incremental data",
301
+ suggestions=[
302
+ "Use an engine-supported replacement plan instead of phlo/full_refresh_asset"
303
+ ],
304
+ )
305
+ args = ["run"]
306
+ for model in models.values():
307
+ args.extend(["--select", model.name])
308
+ args.extend(["--jobs", str(settings.jobs), *adapter_args(runtime, settings)])
309
+ payload, invocation = run_json(
310
+ workspace.binary,
311
+ args,
312
+ settings=settings,
313
+ root=workspace.root,
314
+ )
315
+ if payload is None:
316
+ detail = (invocation.stderr or invocation.stdout).strip().splitlines()
317
+ raise PhloCapabilitySetupError(
318
+ capability="transform",
319
+ required=True,
320
+ message=(
321
+ "phlo-transform produced no run report" + (f": {detail[-1]}" if detail else "")
322
+ ),
323
+ suggestions=["Run the same command with --json to inspect the failure"],
324
+ )
325
+
326
+ status = payload.get("status")
327
+ return RunGroupResult(
328
+ outcomes={
329
+ key: _model_results(workspace, model, key, settings, payload)
330
+ for key, model in models.items()
331
+ },
332
+ error=(
333
+ f"phlo-transform run failed (exit {invocation.returncode}, status {status})"
334
+ if not invocation.ok or status != "passed"
335
+ else None
336
+ ),
337
+ )
338
+
339
+
340
+ def _model_results(
341
+ workspace: TransformWorkspace,
342
+ model: WorkspaceModel,
343
+ key: str,
344
+ settings: TransformSettings,
345
+ payload: dict,
346
+ ) -> list[MaterializeResult | CheckResult]:
347
+ """Attribute model and owned-test evidence without inheriting sibling failures."""
348
+ model_report = next(
349
+ (
350
+ item
351
+ for item in payload.get("models") or []
352
+ if isinstance(item, dict) and item.get("model") == model.name
353
+ ),
354
+ {},
355
+ )
356
+ status = str(model_report.get("status") or "not_reported")
357
+ test_results = {
358
+ str(item.get("test")): item
359
+ for item in payload.get("tests") or []
360
+ if isinstance(item, dict) and item.get("test")
361
+ }
362
+ lineage_metadata = _column_lineage_metadata(workspace, model, settings)
363
+
364
+ results: list[MaterializeResult | CheckResult] = []
365
+ for test in workspace.tests_for(model.name):
366
+ outcome = test_results.get(test.name)
367
+ passed = outcome is not None and str(outcome.get("status")) == "passed"
368
+ results.append(
369
+ CheckResult(
370
+ check_name=_check_name(test.name),
371
+ asset_key=key,
372
+ passed=passed,
373
+ severity="error",
374
+ metadata={
375
+ "test": test.name,
376
+ "status": str(outcome.get("status")) if outcome else "not_reported",
377
+ "row_count": outcome.get("row_count") if outcome else None,
378
+ "error": (
379
+ outcome.get("failure") or outcome.get("error")
380
+ if outcome
381
+ else "test did not report a result"
382
+ ),
383
+ },
384
+ )
385
+ )
386
+
387
+ results.append(
388
+ MaterializeResult(
389
+ status=(
390
+ "success"
391
+ if status in {"passed", "skipped", "cached"}
392
+ and all(result.passed for result in results if isinstance(result, CheckResult))
393
+ else "failure"
394
+ ),
395
+ metadata={
396
+ "transform_model": model.name,
397
+ "transform_target": model.target,
398
+ "transform_materialization": model.materialization,
399
+ "transform_version": model.version,
400
+ "transform_workspace": str(workspace.root),
401
+ "transform_run_id": payload.get("run_id"),
402
+ "transform_plan_id": payload.get("plan_id"),
403
+ "transform_status": status,
404
+ "transform_action": model_report.get("action"),
405
+ "transform_reasons": model_report.get("reasons") or [],
406
+ "transform_previous_version": model_report.get("previous_version"),
407
+ "transform_desired_version": model_report.get("desired_version"),
408
+ "transform_sql_hash": model_report.get("sql_hash"),
409
+ "transform_counts": payload.get("counts") or {},
410
+ "transform_failure": model_report.get("failure"),
411
+ "transform_duration_ms": model_report.get("duration_ms"),
412
+ "transform_query_id": model_report.get("query_id"),
413
+ "transform_tests": len(workspace.tests_for(model.name)),
414
+ **lineage_metadata,
415
+ },
416
+ )
417
+ )
418
+ return results
419
+
420
+
421
+ def build_workspace_asset_specs(
422
+ project_root: Path,
423
+ settings: TransformSettings,
424
+ ) -> list[AssetSpec]:
425
+ """Build asset specs for every model in the project's transform workspace."""
426
+
427
+ workspace = load_workspace(project_root, settings)
428
+ if workspace is None:
429
+ return []
430
+ mode = settings.asset_key_mode
431
+ keys = _key_map(workspace, mode)
432
+ models = {keys[model.name]: model for model in workspace.models}
433
+
434
+ def _group_runner(runtime: RuntimeContext, selected: frozenset[str]) -> RunGroupResult:
435
+ return _run_models(
436
+ workspace,
437
+ {key: model for key, model in models.items() if key in selected},
438
+ settings,
439
+ runtime,
440
+ )
441
+
442
+ # Native tests run after all builds, so a checked parent must finish in
443
+ # an earlier Dagster step before its descendants can execute.
444
+ graph = {
445
+ key: {keys[dep] for dep in model.depends_on if dep in keys} for key, model in models.items()
446
+ }
447
+ stages: dict[str, int] = {}
448
+ for key in TopologicalSorter(graph).static_order():
449
+ stages[key] = max(
450
+ (stages[dep] + bool(workspace.tests_for(models[dep].name)) for dep in graph[key]),
451
+ default=0,
452
+ )
453
+ run_groups = {
454
+ stage: RunGroupSpec(name=f"transform_{stage}", fn=_group_runner)
455
+ for stage in set(stages.values())
456
+ }
457
+ specs: list[AssetSpec] = []
458
+ for model in workspace.models:
459
+ key = keys[model.name]
460
+ deps = [keys[dep] for dep in model.depends_on if dep in keys]
461
+ source_assets = [
462
+ workspace.source_assets[source]
463
+ for source in model.sources
464
+ if source in workspace.source_assets
465
+ ]
466
+ deps.extend(asset for asset in source_assets if asset not in deps)
467
+ missing = [dep for dep in model.depends_on if dep not in keys]
468
+ if missing:
469
+ logger.warning(
470
+ "phlo_transform_model_dependency_missing",
471
+ model=model.name,
472
+ missing=missing,
473
+ )
474
+
475
+ def _runner(
476
+ runtime: RuntimeContext,
477
+ bound_model: WorkspaceModel = model,
478
+ bound_key: str = key,
479
+ ) -> list[MaterializeResult | CheckResult]:
480
+ return _run_model(
481
+ workspace,
482
+ bound_model,
483
+ bound_key,
484
+ settings,
485
+ runtime,
486
+ )
487
+
488
+ specs.append(
489
+ AssetSpec(
490
+ key=key,
491
+ group=model.group,
492
+ description=f"Phlo Transform model {model.name}",
493
+ kinds={"transform", "sql", model.materialization},
494
+ tags={"tool": "phlo-transform", "provider": "transform"},
495
+ metadata={
496
+ "transform_engine": "phlo-transform",
497
+ "transform_model": model.name,
498
+ "transform_target": model.target,
499
+ "transform_materialization": model.materialization,
500
+ "transform_version": model.version,
501
+ "transform_path": model.path,
502
+ "transform_tags": list(model.tags),
503
+ "transform_test_names": [test.name for test in workspace.tests_for(model.name)],
504
+ "transform_sources": list(model.sources),
505
+ "transform_source_assets": source_assets,
506
+ "transform_workspace": str(workspace.root),
507
+ },
508
+ deps=deps,
509
+ checks=_check_specs(workspace, model, key),
510
+ run=RunSpec(
511
+ fn=_runner,
512
+ group=run_groups[stages[key]] if settings.group_execution else None,
513
+ ),
514
+ )
515
+ )
516
+ logger.info(
517
+ "phlo_transform_asset_specs_built",
518
+ workspace_root=str(workspace.root),
519
+ spec_count=len(specs),
520
+ )
521
+ return specs
522
+
523
+
524
+ def build_transform_asset_specs(project_root: Path | None = None) -> list[AssetSpec]:
525
+ """Return every transform asset: Python declarations plus workspace models.
526
+
527
+ ``phlo.transform.sql`` declarations are ordinary Python assets and stay
528
+ available regardless of which execution engine owns SQL workspaces. The
529
+ native workspace contributes assets only when it is the active provider.
530
+ """
531
+
532
+ root = resolve_project_root(project_root)
533
+ specs = list(get_transform_assets())
534
+ settings = get_settings(root)
535
+ if not provider_is_active(root):
536
+ logger.info(
537
+ "phlo_transform_provider_inactive",
538
+ active_provider=active_transformation_provider(root),
539
+ workspace=settings.root,
540
+ )
541
+ return specs
542
+ specs.extend(build_workspace_asset_specs(root, settings))
543
+ return specs
544
+
545
+
546
+ __all__ = [
547
+ "adapter_args",
548
+ "build_transform_asset_specs",
549
+ "build_workspace_asset_specs",
550
+ "has_dbt_project",
551
+ "provider_is_active",
552
+ ]
@@ -0,0 +1,62 @@
1
+ """phlo_transform CLI authorization table.
2
+
3
+ Declares which transform CLI commands mutate versus read so regulated
4
+ deployments gate engine runs, workspace translation, provider switching, and
5
+ binary installation through the same surface contract as other providers.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from phlo.cli.authorization import CliSurfaceAdapter, cli_surface_adapter_class
11
+
12
+ SURFACE_NAME = "phlo-transform"
13
+ FRAMEWORK_TYPE = "cli"
14
+ MUTATION_COMMANDS: frozenset[str] = frozenset(
15
+ {
16
+ "transform.install",
17
+ "transform.run",
18
+ "transform.translate",
19
+ "transform.use",
20
+ }
21
+ )
22
+ READ_COMMANDS: frozenset[str] = frozenset(
23
+ {
24
+ "transform.check",
25
+ "transform.doctor",
26
+ "transform.explain",
27
+ "transform.lineage",
28
+ "transform.list",
29
+ "transform.plan",
30
+ "transform.status",
31
+ "transform.test",
32
+ }
33
+ )
34
+ COMMAND_RESOURCE_MAP: dict[str, str] = {
35
+ "transform.install": "project",
36
+ "transform.run": "dataset",
37
+ "transform.translate": "project",
38
+ "transform.use": "project",
39
+ }
40
+ COMMAND_ACTION_MAP: dict[str, str] = {
41
+ "transform.install": "project.create",
42
+ "transform.run": "dataset.write",
43
+ "transform.translate": "project.create",
44
+ "transform.use": "project.update",
45
+ }
46
+
47
+ TransformSurfaceAdapter = cli_surface_adapter_class(
48
+ "TransformSurfaceAdapter",
49
+ surface_name=SURFACE_NAME,
50
+ mutation_commands=MUTATION_COMMANDS,
51
+ read_commands=READ_COMMANDS,
52
+ command_resource_map=COMMAND_RESOURCE_MAP,
53
+ command_action_map=COMMAND_ACTION_MAP,
54
+ default_action_prefix="transform",
55
+ default_resource_prefix="transform",
56
+ )
57
+
58
+
59
+ def get_transform_adapter() -> CliSurfaceAdapter:
60
+ """Return the shared TransformSurfaceAdapter instance for this surface."""
61
+
62
+ return TransformSurfaceAdapter.get_instance()