phlo-transform 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_transform/__init__.py +23 -0
- phlo_transform/assets.py +552 -0
- phlo_transform/authorization.py +62 -0
- phlo_transform/binary.py +343 -0
- phlo_transform/cli_plugin.py +402 -0
- phlo_transform/editor.py +679 -0
- phlo_transform/editor_execution.py +467 -0
- phlo_transform/plugin.py +59 -0
- phlo_transform/settings.py +139 -0
- phlo_transform/translate.py +72 -0
- phlo_transform/workspace.py +289 -0
- phlo_transform-0.2.0.dist-info/METADATA +255 -0
- phlo_transform-0.2.0.dist-info/RECORD +16 -0
- phlo_transform-0.2.0.dist-info/WHEEL +5 -0
- phlo_transform-0.2.0.dist-info/entry_points.txt +8 -0
- phlo_transform-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
"""``phlo transform`` CLI commands.
|
|
2
|
+
|
|
3
|
+
The group is a thin, testable wrapper around the engine's own commands: it
|
|
4
|
+
resolves the workspace and binary, forwards selection and adapter flags, and
|
|
5
|
+
prints the engine's report. Semantic answers always come from the binary, so
|
|
6
|
+
the CLI cannot drift from the engine's compiler or planner.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import click
|
|
15
|
+
|
|
16
|
+
from phlo.cli.authorization_wrappers import enforce_surface_mutation_authorization
|
|
17
|
+
from phlo.cli.output import user_error
|
|
18
|
+
from phlo.config.env import resolve_project_root
|
|
19
|
+
from phlo.config.transformation import (
|
|
20
|
+
KNOWN_PROVIDERS,
|
|
21
|
+
active_transformation_provider,
|
|
22
|
+
normalise_provider,
|
|
23
|
+
set_active_transformation_provider,
|
|
24
|
+
)
|
|
25
|
+
from phlo.plugins.base import cli_command_plugin_class
|
|
26
|
+
from phlo_transform.assets import adapter_args
|
|
27
|
+
from phlo_transform.authorization import get_transform_adapter
|
|
28
|
+
from phlo_transform.binary import (
|
|
29
|
+
binary_version,
|
|
30
|
+
find_binary,
|
|
31
|
+
install_binary,
|
|
32
|
+
run_cli,
|
|
33
|
+
)
|
|
34
|
+
from phlo_transform.settings import TransformSettings, get_settings
|
|
35
|
+
from phlo_transform.translate import translate_dbt_project
|
|
36
|
+
from phlo_transform.workspace import TransformWorkspace, load_workspace
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _project_root() -> Path:
|
|
40
|
+
return resolve_project_root()
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _settings(root: Path | None = None) -> TransformSettings:
|
|
44
|
+
return get_settings(root or _project_root())
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _require_workspace(settings: TransformSettings) -> TransformWorkspace:
|
|
48
|
+
project_root = _project_root()
|
|
49
|
+
workspace = load_workspace(project_root, settings)
|
|
50
|
+
if workspace is None:
|
|
51
|
+
raise user_error(
|
|
52
|
+
"no phlo-transform workspace found",
|
|
53
|
+
missing=project_root / "workflows" / "transforms",
|
|
54
|
+
details=[
|
|
55
|
+
"Create a workspace with: phlo-transform init",
|
|
56
|
+
"Or set PHLO_TRANSFORM_ROOT to an existing workspace",
|
|
57
|
+
],
|
|
58
|
+
run="phlo-transform init",
|
|
59
|
+
)
|
|
60
|
+
return workspace
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _forward(
|
|
64
|
+
workspace: TransformWorkspace,
|
|
65
|
+
command: str,
|
|
66
|
+
args: list[str],
|
|
67
|
+
*,
|
|
68
|
+
settings: TransformSettings,
|
|
69
|
+
as_json: bool,
|
|
70
|
+
include_adapter: bool = False,
|
|
71
|
+
adapter_override: str | None = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
if include_adapter:
|
|
74
|
+
effective = (
|
|
75
|
+
settings.model_copy(update={"adapter": adapter_override})
|
|
76
|
+
if adapter_override
|
|
77
|
+
else settings
|
|
78
|
+
)
|
|
79
|
+
args = [*args, *adapter_args(None, effective, infer=False)]
|
|
80
|
+
invocation = run_cli(
|
|
81
|
+
workspace.binary,
|
|
82
|
+
[command, *args],
|
|
83
|
+
settings=settings,
|
|
84
|
+
root=workspace.root,
|
|
85
|
+
extra_args=["--json"] if as_json else None,
|
|
86
|
+
)
|
|
87
|
+
if invocation.stdout:
|
|
88
|
+
click.echo(invocation.stdout.rstrip())
|
|
89
|
+
if invocation.stderr:
|
|
90
|
+
click.echo(invocation.stderr.rstrip(), err=True)
|
|
91
|
+
if not invocation.ok:
|
|
92
|
+
raise SystemExit(invocation.returncode)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
@click.group("transform")
|
|
96
|
+
def transform_group() -> None:
|
|
97
|
+
"""Phlo Transform workspace commands (check, plan, run, test, lineage)."""
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@transform_group.command("install")
|
|
101
|
+
@click.option("--version", default=None, help="Release version to download (default: settings).")
|
|
102
|
+
@click.option("--force", is_flag=True, help="Replace an already installed binary.")
|
|
103
|
+
def install_cmd(version: str | None, force: bool) -> None:
|
|
104
|
+
"""Download the pinned portable engine binary into .phlo/bin."""
|
|
105
|
+
enforce_surface_mutation_authorization("transform.install", get_transform_adapter)
|
|
106
|
+
settings = _settings()
|
|
107
|
+
binary = install_binary(_project_root(), settings, version=version, force=force)
|
|
108
|
+
reported = binary_version(binary, settings)
|
|
109
|
+
click.echo(f"Installed {binary}")
|
|
110
|
+
if reported:
|
|
111
|
+
click.echo(reported)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@transform_group.command("check")
|
|
115
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
116
|
+
def check_cmd(as_json: bool) -> None:
|
|
117
|
+
"""Compile the workspace and print diagnostics."""
|
|
118
|
+
settings = _settings()
|
|
119
|
+
_forward(_require_workspace(settings), "check", [], settings=settings, as_json=as_json)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@transform_group.command("plan")
|
|
123
|
+
@click.option("--select", "select", multiple=True, help="Model selector (repeatable).")
|
|
124
|
+
@click.option("--exclude", "exclude", multiple=True, help="Exclude selector (repeatable).")
|
|
125
|
+
@click.option("--upstream", is_flag=True, help="Include transitive dependencies.")
|
|
126
|
+
@click.option("--downstream", is_flag=True, help="Include transitive dependents.")
|
|
127
|
+
@click.option("--force", is_flag=True, help="Plan a rebuild regardless of state.")
|
|
128
|
+
@click.option("--ref", default=None, help="Logical environment / Nessie reference.")
|
|
129
|
+
@click.option("--adapter", default=None, help="Execution adapter (trino or duckdb).")
|
|
130
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
131
|
+
def plan_cmd(
|
|
132
|
+
select: tuple[str, ...],
|
|
133
|
+
exclude: tuple[str, ...],
|
|
134
|
+
upstream: bool,
|
|
135
|
+
downstream: bool,
|
|
136
|
+
force: bool,
|
|
137
|
+
ref: str | None,
|
|
138
|
+
adapter: str | None,
|
|
139
|
+
as_json: bool,
|
|
140
|
+
) -> None:
|
|
141
|
+
"""Explain what would change and why."""
|
|
142
|
+
settings = _settings()
|
|
143
|
+
args = _selection_args(select, exclude, upstream, downstream, force, ref)
|
|
144
|
+
_forward(
|
|
145
|
+
_require_workspace(settings),
|
|
146
|
+
"plan",
|
|
147
|
+
args,
|
|
148
|
+
settings=settings,
|
|
149
|
+
as_json=as_json,
|
|
150
|
+
include_adapter=True,
|
|
151
|
+
adapter_override=adapter,
|
|
152
|
+
)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@transform_group.command("run")
|
|
156
|
+
@click.option("--select", "select", multiple=True, help="Model selector (repeatable).")
|
|
157
|
+
@click.option("--exclude", "exclude", multiple=True, help="Exclude selector (repeatable).")
|
|
158
|
+
@click.option("--upstream", is_flag=True, help="Include transitive dependencies.")
|
|
159
|
+
@click.option("--downstream", is_flag=True, help="Include transitive dependents.")
|
|
160
|
+
@click.option("--force", is_flag=True, help="Rebuild selected models regardless of state.")
|
|
161
|
+
@click.option("--ref", default=None, help="Logical environment / Nessie reference.")
|
|
162
|
+
@click.option("--adapter", default=None, help="Execution adapter (trino or duckdb).")
|
|
163
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
164
|
+
def run_cmd(
|
|
165
|
+
select: tuple[str, ...],
|
|
166
|
+
exclude: tuple[str, ...],
|
|
167
|
+
upstream: bool,
|
|
168
|
+
downstream: bool,
|
|
169
|
+
force: bool,
|
|
170
|
+
ref: str | None,
|
|
171
|
+
adapter: str | None,
|
|
172
|
+
as_json: bool,
|
|
173
|
+
) -> None:
|
|
174
|
+
"""Plan, apply, and test the selected models."""
|
|
175
|
+
enforce_surface_mutation_authorization("transform.run", get_transform_adapter)
|
|
176
|
+
settings = _settings()
|
|
177
|
+
args = _selection_args(select, exclude, upstream, downstream, force, ref)
|
|
178
|
+
_forward(
|
|
179
|
+
_require_workspace(settings),
|
|
180
|
+
"run",
|
|
181
|
+
args,
|
|
182
|
+
settings=settings,
|
|
183
|
+
as_json=as_json,
|
|
184
|
+
include_adapter=True,
|
|
185
|
+
adapter_override=adapter,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@transform_group.command("test")
|
|
190
|
+
@click.option("--select", "select", multiple=True, help="Model selector (repeatable).")
|
|
191
|
+
@click.option("--ref", default=None, help="Logical environment / Nessie reference.")
|
|
192
|
+
@click.option("--adapter", default=None, help="Execution adapter (trino or duckdb).")
|
|
193
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
194
|
+
def test_cmd(select: tuple[str, ...], ref: str | None, adapter: str | None, as_json: bool) -> None:
|
|
195
|
+
"""Run workspace tests against the selected models."""
|
|
196
|
+
settings = _settings()
|
|
197
|
+
args = _selection_args(select, (), False, False, False, ref)
|
|
198
|
+
_forward(
|
|
199
|
+
_require_workspace(settings),
|
|
200
|
+
"test",
|
|
201
|
+
args,
|
|
202
|
+
settings=settings,
|
|
203
|
+
as_json=as_json,
|
|
204
|
+
include_adapter=True,
|
|
205
|
+
adapter_override=adapter,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
@transform_group.command("list")
|
|
210
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
211
|
+
def list_cmd(as_json: bool) -> None:
|
|
212
|
+
"""List compiled models, dependencies, and tests."""
|
|
213
|
+
settings = _settings()
|
|
214
|
+
_forward(_require_workspace(settings), "list", [], settings=settings, as_json=as_json)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@transform_group.command("explain")
|
|
218
|
+
@click.argument("model")
|
|
219
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
220
|
+
def explain_cmd(model: str, as_json: bool) -> None:
|
|
221
|
+
"""Explain one model: SQL, columns, lineage, and state."""
|
|
222
|
+
settings = _settings()
|
|
223
|
+
_forward(_require_workspace(settings), "explain", [model], settings=settings, as_json=as_json)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
@transform_group.command("lineage")
|
|
227
|
+
@click.option(
|
|
228
|
+
"--format",
|
|
229
|
+
"lineage_format",
|
|
230
|
+
type=click.Choice(["graph", "openlineage"]),
|
|
231
|
+
default="graph",
|
|
232
|
+
show_default=True,
|
|
233
|
+
help="Export format.",
|
|
234
|
+
)
|
|
235
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
236
|
+
def lineage_cmd(lineage_format: str, as_json: bool) -> None:
|
|
237
|
+
"""Export the workspace lineage graph."""
|
|
238
|
+
settings = _settings()
|
|
239
|
+
args = ["--format", lineage_format]
|
|
240
|
+
_forward(_require_workspace(settings), "lineage", args, settings=settings, as_json=as_json)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@transform_group.command("doctor")
|
|
244
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
245
|
+
def doctor_cmd(as_json: bool) -> None:
|
|
246
|
+
"""Diagnose workspace, adapter, and state health."""
|
|
247
|
+
settings = _settings()
|
|
248
|
+
_forward(
|
|
249
|
+
_require_workspace(settings),
|
|
250
|
+
"doctor",
|
|
251
|
+
[],
|
|
252
|
+
settings=settings,
|
|
253
|
+
as_json=as_json,
|
|
254
|
+
include_adapter=True,
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@transform_group.command("translate")
|
|
259
|
+
@click.option("--from", "source", default="dbt", show_default=True, help="Source format.")
|
|
260
|
+
@click.option("--source-root", type=click.Path(file_okay=False, path_type=Path), default=None)
|
|
261
|
+
@click.option("--out", type=click.Path(file_okay=False, path_type=Path), default=None)
|
|
262
|
+
@click.option("--check", "check_only", is_flag=True, help="Analyse without writing.")
|
|
263
|
+
@click.option("--overwrite", is_flag=True, help="Overwrite existing files.")
|
|
264
|
+
@click.option("--verify", is_flag=True, help="Compile-check the generated workspace.")
|
|
265
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit the engine's JSON report.")
|
|
266
|
+
def translate_cmd(
|
|
267
|
+
source: str,
|
|
268
|
+
source_root: Path | None,
|
|
269
|
+
out: Path | None,
|
|
270
|
+
check_only: bool,
|
|
271
|
+
overwrite: bool,
|
|
272
|
+
verify: bool,
|
|
273
|
+
as_json: bool,
|
|
274
|
+
) -> None:
|
|
275
|
+
"""Translate a dbt project into a native transform workspace."""
|
|
276
|
+
if source != "dbt":
|
|
277
|
+
raise user_error(
|
|
278
|
+
f"unsupported translation source '{source}'",
|
|
279
|
+
details=["Only --from dbt is supported"],
|
|
280
|
+
)
|
|
281
|
+
if not check_only and out is None:
|
|
282
|
+
raise user_error("translate needs --out or --check", run="phlo transform translate --help")
|
|
283
|
+
if not check_only:
|
|
284
|
+
enforce_surface_mutation_authorization("transform.translate", get_transform_adapter)
|
|
285
|
+
settings = _settings()
|
|
286
|
+
invocation = translate_dbt_project(
|
|
287
|
+
_project_root(),
|
|
288
|
+
settings,
|
|
289
|
+
dbt_project=source_root,
|
|
290
|
+
out=out,
|
|
291
|
+
check_only=check_only,
|
|
292
|
+
overwrite=overwrite,
|
|
293
|
+
verify=verify,
|
|
294
|
+
)
|
|
295
|
+
if as_json:
|
|
296
|
+
click.echo(invocation.stdout.rstrip())
|
|
297
|
+
if invocation.stderr:
|
|
298
|
+
click.echo(invocation.stderr.rstrip(), err=True)
|
|
299
|
+
if not invocation.ok:
|
|
300
|
+
raise SystemExit(invocation.returncode)
|
|
301
|
+
return
|
|
302
|
+
click.echo(invocation.stdout.rstrip())
|
|
303
|
+
if invocation.stderr:
|
|
304
|
+
click.echo(invocation.stderr.rstrip(), err=True)
|
|
305
|
+
if not invocation.ok:
|
|
306
|
+
raise SystemExit(invocation.returncode)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
@transform_group.command("use")
|
|
310
|
+
@click.argument("provider", required=False)
|
|
311
|
+
@click.option("--unset", is_flag=True, help="Clear the project choice.")
|
|
312
|
+
def use_cmd(provider: str | None, unset: bool) -> None:
|
|
313
|
+
"""Select the transformation provider that owns the asset graph."""
|
|
314
|
+
enforce_surface_mutation_authorization("transform.use", get_transform_adapter)
|
|
315
|
+
if unset == bool(provider):
|
|
316
|
+
raise user_error("choose one provider or pass --unset", run="phlo transform use dbt")
|
|
317
|
+
if unset:
|
|
318
|
+
set_active_transformation_provider(None)
|
|
319
|
+
click.echo("Transformation provider follows install defaults.")
|
|
320
|
+
return
|
|
321
|
+
|
|
322
|
+
from phlo.plugins.discovery import discover_plugins, get_global_registry
|
|
323
|
+
|
|
324
|
+
discover_plugins(plugin_type="transformation_provider", auto_register=True)
|
|
325
|
+
installed = set(get_global_registry().list("transformation_provider"))
|
|
326
|
+
canonical = normalise_provider(str(provider))
|
|
327
|
+
if canonical not in installed:
|
|
328
|
+
raise user_error(
|
|
329
|
+
f"transformation provider '{provider}' is not installed",
|
|
330
|
+
details=[
|
|
331
|
+
f"Installed providers: {', '.join(sorted(installed)) or 'none'}",
|
|
332
|
+
f"Known names: {', '.join(KNOWN_PROVIDERS)}",
|
|
333
|
+
],
|
|
334
|
+
)
|
|
335
|
+
path = set_active_transformation_provider(canonical, _project_root())
|
|
336
|
+
click.echo(f"Active transformation provider: {canonical}")
|
|
337
|
+
click.echo(f"Wrote {path}")
|
|
338
|
+
click.echo(
|
|
339
|
+
"Restart Dagster (phlo services restart dagster) for the orchestrated graph to reload."
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@transform_group.command("status")
|
|
344
|
+
@click.option("--json", "as_json", is_flag=True, help="Emit machine-readable status.")
|
|
345
|
+
def status_cmd(as_json: bool) -> None:
|
|
346
|
+
"""Show the active provider, workspace, binary, and compiled counts."""
|
|
347
|
+
from phlo_transform.assets import provider_is_active
|
|
348
|
+
|
|
349
|
+
project_root = _project_root()
|
|
350
|
+
settings = _settings(project_root)
|
|
351
|
+
binary = find_binary(project_root, settings)
|
|
352
|
+
workspace = load_workspace(project_root, settings, require=False)
|
|
353
|
+
payload = {
|
|
354
|
+
"active_provider": active_transformation_provider(project_root),
|
|
355
|
+
"provider_active": bool(workspace) and provider_is_active(project_root),
|
|
356
|
+
"project_root": str(project_root),
|
|
357
|
+
"workspace": str(workspace.root) if workspace else None,
|
|
358
|
+
"binary": str(binary) if binary else None,
|
|
359
|
+
"binary_version": binary_version(binary, settings) if binary else None,
|
|
360
|
+
"adapter": settings.adapter,
|
|
361
|
+
"asset_key_mode": settings.asset_key_mode,
|
|
362
|
+
"models": len(workspace.models) if workspace else 0,
|
|
363
|
+
"tests": len(workspace.tests) if workspace else 0,
|
|
364
|
+
}
|
|
365
|
+
if as_json:
|
|
366
|
+
click.echo(json.dumps(payload, indent=2, sort_keys=True))
|
|
367
|
+
return
|
|
368
|
+
for label, value in payload.items():
|
|
369
|
+
click.echo(f"{label}: {value if value is not None else '-'}")
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _selection_args(
|
|
373
|
+
select: tuple[str, ...],
|
|
374
|
+
exclude: tuple[str, ...],
|
|
375
|
+
upstream: bool,
|
|
376
|
+
downstream: bool,
|
|
377
|
+
force: bool,
|
|
378
|
+
ref: str | None,
|
|
379
|
+
) -> list[str]:
|
|
380
|
+
args: list[str] = []
|
|
381
|
+
for term in select:
|
|
382
|
+
args.extend(["--select", term])
|
|
383
|
+
for term in exclude:
|
|
384
|
+
args.extend(["--exclude", term])
|
|
385
|
+
if upstream:
|
|
386
|
+
args.append("--upstream")
|
|
387
|
+
if downstream:
|
|
388
|
+
args.append("--downstream")
|
|
389
|
+
if force:
|
|
390
|
+
args.append("--force")
|
|
391
|
+
if ref:
|
|
392
|
+
args.extend(["--ref", ref])
|
|
393
|
+
return args
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
TransformCliPlugin = cli_command_plugin_class(
|
|
397
|
+
"TransformCliPlugin",
|
|
398
|
+
name="transform",
|
|
399
|
+
version="0.1.0",
|
|
400
|
+
description="Phlo Transform workspace CLI commands",
|
|
401
|
+
commands=[transform_group],
|
|
402
|
+
)
|