phlo-transform 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_transform/__init__.py +23 -0
- phlo_transform/assets.py +552 -0
- phlo_transform/authorization.py +62 -0
- phlo_transform/binary.py +343 -0
- phlo_transform/cli_plugin.py +402 -0
- phlo_transform/editor.py +679 -0
- phlo_transform/editor_execution.py +467 -0
- phlo_transform/plugin.py +59 -0
- phlo_transform/settings.py +139 -0
- phlo_transform/translate.py +72 -0
- phlo_transform/workspace.py +289 -0
- phlo_transform-0.2.0.dist-info/METADATA +255 -0
- phlo_transform-0.2.0.dist-info/RECORD +16 -0
- phlo_transform-0.2.0.dist-info/WHEEL +5 -0
- phlo_transform-0.2.0.dist-info/entry_points.txt +8 -0
- phlo_transform-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,467 @@
|
|
|
1
|
+
"""Native executions pinned to immutable drafts and externally verified candidates.
|
|
2
|
+
|
|
3
|
+
The API's candidate provider owns physical branch/catalog provisioning. This
|
|
4
|
+
module runs only the supplied, verified candidate catalog; it never provisions,
|
|
5
|
+
merges, promotes or rewrites the saved SQL revision.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import shutil
|
|
11
|
+
from copy import deepcopy
|
|
12
|
+
from datetime import UTC, datetime
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from uuid import UUID
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from phlo.plugins.observatory_settings import SettingsScope, get_settings_service
|
|
18
|
+
from phlo_transform.assets import adapter_args
|
|
19
|
+
from phlo_transform.binary import require_binary, run_json
|
|
20
|
+
from phlo_transform.editor import (
|
|
21
|
+
TransformEditorError,
|
|
22
|
+
_artifact,
|
|
23
|
+
_contained,
|
|
24
|
+
_inputs,
|
|
25
|
+
_namespace,
|
|
26
|
+
_records,
|
|
27
|
+
_reclaim_workspace,
|
|
28
|
+
_reclaim_workspace_bytes,
|
|
29
|
+
_digest,
|
|
30
|
+
_uuid,
|
|
31
|
+
_workspace_digest,
|
|
32
|
+
read_model_draft,
|
|
33
|
+
)
|
|
34
|
+
from phlo_transform.settings import get_settings
|
|
35
|
+
|
|
36
|
+
_MAX_RETAINED_EXECUTIONS = 1000
|
|
37
|
+
_EXECUTION_FIELDS = {
|
|
38
|
+
"execution_id",
|
|
39
|
+
"model",
|
|
40
|
+
"actor",
|
|
41
|
+
"env",
|
|
42
|
+
"ref",
|
|
43
|
+
"draft_id",
|
|
44
|
+
"revision",
|
|
45
|
+
"revision_digest",
|
|
46
|
+
"status",
|
|
47
|
+
"created_at",
|
|
48
|
+
"candidate_ref",
|
|
49
|
+
"candidate_catalog",
|
|
50
|
+
"report",
|
|
51
|
+
"operation_claim",
|
|
52
|
+
"archived_at",
|
|
53
|
+
"archived_by",
|
|
54
|
+
"archive_reason",
|
|
55
|
+
"candidate_binding",
|
|
56
|
+
"plan",
|
|
57
|
+
"preview",
|
|
58
|
+
"candidate_result_hash",
|
|
59
|
+
"candidate_snapshot_id",
|
|
60
|
+
"reclaim_started_at",
|
|
61
|
+
"reclaimed_at",
|
|
62
|
+
"reclaimed_by",
|
|
63
|
+
"reclaim_reason",
|
|
64
|
+
"reclaimed_bytes",
|
|
65
|
+
"reclaim_payload_bytes",
|
|
66
|
+
"retained_file_digests",
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _storage(root: Path) -> str:
|
|
71
|
+
return _namespace(root) + ".executions"
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _directory(root: Path, execution_id: str) -> Path:
|
|
75
|
+
return _contained(
|
|
76
|
+
root, root / ".phlo" / "transform-editor" / "executions" / _uuid(execution_id) / "workspace"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _store(
|
|
81
|
+
root: Path,
|
|
82
|
+
record: dict[str, Any],
|
|
83
|
+
*,
|
|
84
|
+
create: bool = False,
|
|
85
|
+
expected_status: str | None = None,
|
|
86
|
+
expected_record: dict[str, Any] | None = None,
|
|
87
|
+
) -> None:
|
|
88
|
+
def update(current):
|
|
89
|
+
records = _records(current, require_known_state=bool(record.get("reclaim_started_at")))
|
|
90
|
+
if record.get("reclaim_started_at") and any(
|
|
91
|
+
set(item) - _EXECUTION_FIELDS for item in records
|
|
92
|
+
):
|
|
93
|
+
raise TransformEditorError(
|
|
94
|
+
"Unknown execution references prevent reclamation.", status=409
|
|
95
|
+
)
|
|
96
|
+
existing = next(
|
|
97
|
+
(item for item in records if item.get("execution_id") == record["execution_id"]), None
|
|
98
|
+
)
|
|
99
|
+
if expected_status and (existing is None or existing.get("status") != expected_status):
|
|
100
|
+
raise TransformEditorError("Execution state changed; do not rerun it.", status=409)
|
|
101
|
+
if expected_record is not None and existing != expected_record:
|
|
102
|
+
raise TransformEditorError(
|
|
103
|
+
"Execution evidence changed; reload before continuing.", status=409
|
|
104
|
+
)
|
|
105
|
+
if (
|
|
106
|
+
create
|
|
107
|
+
and sum(not item.get("reclaimed_at") for item in records) >= _MAX_RETAINED_EXECUTIONS
|
|
108
|
+
):
|
|
109
|
+
raise TransformEditorError("Execution history limit reached.", status=409)
|
|
110
|
+
if create and existing:
|
|
111
|
+
raise TransformEditorError(
|
|
112
|
+
"Execution identity is already recorded; read its result instead of rerunning.",
|
|
113
|
+
status=409,
|
|
114
|
+
)
|
|
115
|
+
return {
|
|
116
|
+
"schema_version": 1,
|
|
117
|
+
"items": [
|
|
118
|
+
*[item for item in records if item.get("execution_id") != record["execution_id"]],
|
|
119
|
+
deepcopy(record),
|
|
120
|
+
],
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
get_settings_service().mutate(SettingsScope.GLOBAL, _storage(root), update)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def prepare_model_execution(
|
|
127
|
+
root: Path,
|
|
128
|
+
model: str,
|
|
129
|
+
*,
|
|
130
|
+
execution_id: str,
|
|
131
|
+
operation_claim: dict[str, str] | None = None,
|
|
132
|
+
**identity: Any,
|
|
133
|
+
) -> dict[str, Any]:
|
|
134
|
+
draft = read_model_draft(root, model, **identity)
|
|
135
|
+
if draft.get("archived_at"):
|
|
136
|
+
raise TransformEditorError("Restore the archived revision before execution.", status=409)
|
|
137
|
+
files = _inputs(_artifact(root, draft["draft_id"], draft["artifact_id"]))
|
|
138
|
+
directory = _directory(root, execution_id)
|
|
139
|
+
if directory.parent.exists():
|
|
140
|
+
raise TransformEditorError(
|
|
141
|
+
"Execution workspace already exists; read its result instead of rerunning.", status=409
|
|
142
|
+
)
|
|
143
|
+
directory.mkdir(parents=True)
|
|
144
|
+
try:
|
|
145
|
+
for name, content in files.items():
|
|
146
|
+
path = _contained(directory, directory / name)
|
|
147
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
148
|
+
path.write_bytes(content)
|
|
149
|
+
record = {
|
|
150
|
+
"execution_id": _uuid(execution_id),
|
|
151
|
+
"model": model,
|
|
152
|
+
**{
|
|
153
|
+
key: draft[key]
|
|
154
|
+
for key in ("actor", "env", "ref", "draft_id", "revision", "revision_digest")
|
|
155
|
+
},
|
|
156
|
+
"status": "prepared",
|
|
157
|
+
"created_at": datetime.now(UTC).isoformat(),
|
|
158
|
+
"candidate_ref": None,
|
|
159
|
+
"candidate_catalog": None,
|
|
160
|
+
"report": None,
|
|
161
|
+
"operation_claim": operation_claim,
|
|
162
|
+
"archived_at": None,
|
|
163
|
+
"archived_by": None,
|
|
164
|
+
"archive_reason": None,
|
|
165
|
+
}
|
|
166
|
+
_store(root, record, create=True)
|
|
167
|
+
except Exception:
|
|
168
|
+
shutil.rmtree(directory.parent, ignore_errors=True)
|
|
169
|
+
raise
|
|
170
|
+
return record
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def read_model_execution(
|
|
174
|
+
root: Path, model: str, *, execution_id: str, **identity: Any
|
|
175
|
+
) -> dict[str, Any]:
|
|
176
|
+
state = get_settings_service().get(SettingsScope.GLOBAL, _storage(root))
|
|
177
|
+
found = next(
|
|
178
|
+
(
|
|
179
|
+
item
|
|
180
|
+
for item in _records(state.settings if state else None)
|
|
181
|
+
if item.get("execution_id") == _uuid(execution_id)
|
|
182
|
+
and item.get("model") == model
|
|
183
|
+
and all(item.get(key) == value for key, value in identity.items())
|
|
184
|
+
),
|
|
185
|
+
None,
|
|
186
|
+
)
|
|
187
|
+
if found is None:
|
|
188
|
+
raise TransformEditorError(
|
|
189
|
+
"Execution was not found in this draft and principal scope.", status=404
|
|
190
|
+
)
|
|
191
|
+
if (
|
|
192
|
+
not found.get("reclaim_started_at")
|
|
193
|
+
and _workspace_digest(_inputs(_directory(root, execution_id))) != found["revision_digest"]
|
|
194
|
+
):
|
|
195
|
+
raise TransformEditorError("Execution source integrity verification failed.", status=409)
|
|
196
|
+
return deepcopy(found)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def execute_model_draft(
|
|
200
|
+
root: Path,
|
|
201
|
+
model: str,
|
|
202
|
+
*,
|
|
203
|
+
execution_id: str,
|
|
204
|
+
candidate_ref: str,
|
|
205
|
+
candidate_catalog: str,
|
|
206
|
+
connection: dict[str, str],
|
|
207
|
+
candidate_binding: dict[str, Any],
|
|
208
|
+
**identity: Any,
|
|
209
|
+
) -> dict[str, Any]:
|
|
210
|
+
record = read_model_execution(root, model, execution_id=execution_id, **identity)
|
|
211
|
+
if record.get("archived_at"):
|
|
212
|
+
raise TransformEditorError("Archived executions cannot be run.", status=409)
|
|
213
|
+
if record["status"] != "prepared":
|
|
214
|
+
raise TransformEditorError(
|
|
215
|
+
"Execution already started; read its recorded outcome instead of rerunning.", status=409
|
|
216
|
+
)
|
|
217
|
+
operation_id = UUID(execution_id).hex
|
|
218
|
+
if candidate_ref != f"{record['env']}-transform-{operation_id}" or candidate_ref in {
|
|
219
|
+
"main",
|
|
220
|
+
record["ref"],
|
|
221
|
+
}:
|
|
222
|
+
raise TransformEditorError(
|
|
223
|
+
"A verified owned non-main candidate reference is required.", status=422
|
|
224
|
+
)
|
|
225
|
+
if (
|
|
226
|
+
candidate_catalog != candidate_binding.get("catalog")
|
|
227
|
+
or candidate_binding.get("execution_id") != operation_id
|
|
228
|
+
or not candidate_catalog.endswith(f"_candidate_{operation_id}")
|
|
229
|
+
or not all(character.isalnum() or character == "_" for character in candidate_catalog)
|
|
230
|
+
):
|
|
231
|
+
raise TransformEditorError("A separate verified candidate catalog is required.", status=422)
|
|
232
|
+
settings = get_settings(root).model_copy(
|
|
233
|
+
update={
|
|
234
|
+
"adapter": "trino",
|
|
235
|
+
"trino_endpoint": connection["endpoint"],
|
|
236
|
+
"trino_user": connection["user"],
|
|
237
|
+
"trino_password": None,
|
|
238
|
+
"trino_catalog": candidate_catalog,
|
|
239
|
+
"timeout_seconds": min(get_settings(root).timeout_seconds, 90.0),
|
|
240
|
+
}
|
|
241
|
+
)
|
|
242
|
+
binary = require_binary(root, settings)
|
|
243
|
+
directory = _directory(root, execution_id)
|
|
244
|
+
args = [
|
|
245
|
+
"--jobs",
|
|
246
|
+
"1",
|
|
247
|
+
"--model-timeout",
|
|
248
|
+
"60s",
|
|
249
|
+
"--select",
|
|
250
|
+
model,
|
|
251
|
+
"--environment",
|
|
252
|
+
candidate_ref,
|
|
253
|
+
*adapter_args(None, settings, infer=False),
|
|
254
|
+
]
|
|
255
|
+
environment = {
|
|
256
|
+
"PHLO_TRINO_PASSWORD": connection.get("password"),
|
|
257
|
+
# The verified API provider owns provisioning. Never let inherited
|
|
258
|
+
# engine configuration silently invoke its independent provisioner.
|
|
259
|
+
"PHLO_NESSIE_ENDPOINT": None,
|
|
260
|
+
"PHLO_NESSIE_CATALOG_URI": None,
|
|
261
|
+
"PHLO_NESSIE_TOKEN": None,
|
|
262
|
+
}
|
|
263
|
+
plan, planned = run_json(
|
|
264
|
+
binary, ["plan", *args], settings=settings, root=directory, environment=environment
|
|
265
|
+
)
|
|
266
|
+
models = plan.get("models") or [] if plan else []
|
|
267
|
+
if (
|
|
268
|
+
not plan
|
|
269
|
+
or not planned.ok
|
|
270
|
+
or plan.get("blocked", True)
|
|
271
|
+
or plan.get("seeds")
|
|
272
|
+
or len(models) != 1
|
|
273
|
+
or models[0].get("id") not in {model, "model://" + model.replace(".", "/")}
|
|
274
|
+
or not str(models[0].get("target", "")).startswith(candidate_catalog + ".")
|
|
275
|
+
):
|
|
276
|
+
raise TransformEditorError(
|
|
277
|
+
"The native plan must contain only the selected model in the verified candidate catalog.",
|
|
278
|
+
status=422,
|
|
279
|
+
)
|
|
280
|
+
if _workspace_digest(_inputs(directory)) != record["revision_digest"]:
|
|
281
|
+
raise TransformEditorError("Execution source changed before native run.", status=409)
|
|
282
|
+
record.update(
|
|
283
|
+
status="running",
|
|
284
|
+
candidate_ref=candidate_ref,
|
|
285
|
+
candidate_catalog=candidate_catalog,
|
|
286
|
+
candidate_binding=candidate_binding,
|
|
287
|
+
plan=plan,
|
|
288
|
+
)
|
|
289
|
+
_store(root, record, expected_status="prepared")
|
|
290
|
+
try:
|
|
291
|
+
report, invoked = run_json(
|
|
292
|
+
binary, ["run", *args], settings=settings, root=directory, environment=environment
|
|
293
|
+
)
|
|
294
|
+
if report is None:
|
|
295
|
+
raise TransformEditorError(
|
|
296
|
+
"The engine did not return a run report; the execution outcome is unknown.",
|
|
297
|
+
status=502,
|
|
298
|
+
)
|
|
299
|
+
selected = [item for item in report.get("models") or [] if item.get("model") == model]
|
|
300
|
+
if (
|
|
301
|
+
len(selected) != 1
|
|
302
|
+
or selected[0].get("target") != models[0]["target"]
|
|
303
|
+
or report.get("environment") != candidate_ref
|
|
304
|
+
):
|
|
305
|
+
raise TransformEditorError(
|
|
306
|
+
"The engine returned execution evidence outside the pinned candidate plan.",
|
|
307
|
+
status=502,
|
|
308
|
+
)
|
|
309
|
+
record.update(
|
|
310
|
+
status="passed" if invoked.ok and report.get("status") == "passed" else "failed",
|
|
311
|
+
report=report,
|
|
312
|
+
)
|
|
313
|
+
_store(root, record)
|
|
314
|
+
return record
|
|
315
|
+
except Exception:
|
|
316
|
+
record.update(status="unknown")
|
|
317
|
+
_store(root, record)
|
|
318
|
+
raise
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def list_model_executions(
|
|
322
|
+
root: Path, model: str, *, include_archived: bool = False, **identity: Any
|
|
323
|
+
) -> list[dict[str, Any]]:
|
|
324
|
+
state = get_settings_service().get(SettingsScope.GLOBAL, _storage(root))
|
|
325
|
+
return [
|
|
326
|
+
read_model_execution(root, model, execution_id=item["execution_id"], **identity)
|
|
327
|
+
for item in _records(state.settings if state else None)
|
|
328
|
+
if item.get("model") == model
|
|
329
|
+
and all(item.get(key) == value for key, value in identity.items())
|
|
330
|
+
and (include_archived or not item.get("archived_at"))
|
|
331
|
+
]
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def archive_model_execution(
|
|
335
|
+
root: Path, model: str, *, execution_id: str, archived: bool, reason: str, **identity: Any
|
|
336
|
+
) -> dict[str, Any]:
|
|
337
|
+
record = read_model_execution(root, model, execution_id=execution_id, **identity)
|
|
338
|
+
if record.get("reclaim_started_at"):
|
|
339
|
+
raise TransformEditorError(
|
|
340
|
+
"Reclamation is irreversible; execution cannot be restored.", status=409
|
|
341
|
+
)
|
|
342
|
+
if record["status"] in {"prepared", "running"}:
|
|
343
|
+
raise TransformEditorError("An active execution cannot be archived.", status=409)
|
|
344
|
+
updated = {
|
|
345
|
+
**record,
|
|
346
|
+
"archived_at": datetime.now(UTC).isoformat() if archived else None,
|
|
347
|
+
"archived_by": identity["actor"] if archived else None,
|
|
348
|
+
"archive_reason": reason if archived else None,
|
|
349
|
+
}
|
|
350
|
+
_store(root, updated, expected_record=record)
|
|
351
|
+
return updated
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def reclaim_model_execution(
|
|
355
|
+
root: Path,
|
|
356
|
+
model: str,
|
|
357
|
+
*,
|
|
358
|
+
execution_id: str,
|
|
359
|
+
expected_revision_digest: str,
|
|
360
|
+
reason: str,
|
|
361
|
+
expected_record: dict[str, Any] | None = None,
|
|
362
|
+
**identity: Any,
|
|
363
|
+
) -> dict[str, Any]:
|
|
364
|
+
"""Dispose only terminal archived execution workspaces, retaining recorded evidence."""
|
|
365
|
+
record = read_model_execution(root, model, execution_id=execution_id, **identity)
|
|
366
|
+
if expected_record is not None and record != expected_record:
|
|
367
|
+
raise TransformEditorError("Execution changed after retention validation.", status=409)
|
|
368
|
+
if set(record) - _EXECUTION_FIELDS:
|
|
369
|
+
raise TransformEditorError("Unknown execution evidence prevents reclamation.", status=409)
|
|
370
|
+
if not record.get("archived_at") or record["revision_digest"] != expected_revision_digest:
|
|
371
|
+
raise TransformEditorError(
|
|
372
|
+
"An archived execution and exact digest are required.", status=409
|
|
373
|
+
)
|
|
374
|
+
if record["status"] not in {"passed", "failed"}:
|
|
375
|
+
raise TransformEditorError("Unresolved executions must retain their workspace.", status=409)
|
|
376
|
+
if record.get("reclaimed_at"):
|
|
377
|
+
return record
|
|
378
|
+
directory = (
|
|
379
|
+
root.resolve() / ".phlo/transform-editor/executions" / _uuid(execution_id) / "workspace"
|
|
380
|
+
)
|
|
381
|
+
started = {**record}
|
|
382
|
+
if not record.get("reclaim_started_at"):
|
|
383
|
+
started.update(
|
|
384
|
+
reclaim_started_at=datetime.now(UTC).isoformat(),
|
|
385
|
+
reclaimed_by=identity["actor"],
|
|
386
|
+
reclaim_reason=reason,
|
|
387
|
+
reclaim_payload_bytes=_reclaim_workspace_bytes(root, directory),
|
|
388
|
+
retained_file_digests={
|
|
389
|
+
name: _digest(data) for name, data in _inputs(directory).items()
|
|
390
|
+
},
|
|
391
|
+
)
|
|
392
|
+
size = started.get("reclaim_payload_bytes")
|
|
393
|
+
if type(size) is not int or size < 0:
|
|
394
|
+
raise TransformEditorError("Original reclamation byte evidence is unavailable.", status=409)
|
|
395
|
+
_store(root, started, expected_record=record)
|
|
396
|
+
_reclaim_workspace(root, directory)
|
|
397
|
+
finished = {**started, "reclaimed_at": datetime.now(UTC).isoformat(), "reclaimed_bytes": size}
|
|
398
|
+
_store(root, finished, expected_record=started)
|
|
399
|
+
return finished
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def reconcile_model_execution(
|
|
403
|
+
root: Path, model: str, *, execution_id: str, preview: dict[str, Any], **identity: Any
|
|
404
|
+
) -> dict[str, Any]:
|
|
405
|
+
"""Record provider readback only for an already attested immutable native success."""
|
|
406
|
+
record = read_model_execution(root, model, execution_id=execution_id, **identity)
|
|
407
|
+
report = record.get("report") or {}
|
|
408
|
+
planned = (record.get("plan") or {}).get("models") or []
|
|
409
|
+
results = report.get("models") or []
|
|
410
|
+
if (
|
|
411
|
+
record["status"] not in {"passed", "unknown"}
|
|
412
|
+
or not record.get("candidate_result_hash")
|
|
413
|
+
or record["candidate_result_hash"] != preview.get("ref_hash")
|
|
414
|
+
or (
|
|
415
|
+
record.get("candidate_snapshot_id") is not None
|
|
416
|
+
and str(record["candidate_snapshot_id"]) != str(preview.get("snapshot_id"))
|
|
417
|
+
)
|
|
418
|
+
or not preview.get("snapshot_id")
|
|
419
|
+
or report.get("status") != "passed"
|
|
420
|
+
or not report.get("run_id")
|
|
421
|
+
or report.get("environment") != record["candidate_ref"]
|
|
422
|
+
or len(planned) != 1
|
|
423
|
+
or len(results) != 1
|
|
424
|
+
or results[0].get("model") != model
|
|
425
|
+
or results[0].get("target") != planned[0].get("target")
|
|
426
|
+
or not str(results[0].get("target", "")).startswith(record["candidate_catalog"] + ".")
|
|
427
|
+
):
|
|
428
|
+
raise TransformEditorError(
|
|
429
|
+
"Native success and pinned provider evidence do not agree; the operation stays fenced.",
|
|
430
|
+
status=409,
|
|
431
|
+
)
|
|
432
|
+
updated = {
|
|
433
|
+
**record,
|
|
434
|
+
"status": "passed",
|
|
435
|
+
"preview": preview,
|
|
436
|
+
"candidate_snapshot_id": str(preview["snapshot_id"]),
|
|
437
|
+
}
|
|
438
|
+
_store(root, updated, expected_record=record)
|
|
439
|
+
return updated
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def record_execution_preview(
|
|
443
|
+
root: Path, model: str, *, execution_id: str, preview: dict[str, Any], **identity: Any
|
|
444
|
+
) -> dict[str, Any]:
|
|
445
|
+
record = read_model_execution(root, model, execution_id=execution_id, **identity)
|
|
446
|
+
if record["status"] != "passed":
|
|
447
|
+
raise TransformEditorError(
|
|
448
|
+
"Only successful native executions have a materialized result preview.", status=409
|
|
449
|
+
)
|
|
450
|
+
record["preview"] = preview
|
|
451
|
+
record["candidate_result_hash"] = preview["ref_hash"]
|
|
452
|
+
record["candidate_snapshot_id"] = preview["snapshot_id"]
|
|
453
|
+
_store(root, record, expected_status="passed")
|
|
454
|
+
return record
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
def record_execution_observation(
|
|
458
|
+
root: Path, model: str, *, execution_id: str, candidate_result_hash: str, **identity: Any
|
|
459
|
+
) -> dict[str, Any]:
|
|
460
|
+
record = read_model_execution(root, model, execution_id=execution_id, **identity)
|
|
461
|
+
if record["status"] != "passed":
|
|
462
|
+
raise TransformEditorError(
|
|
463
|
+
"Only successful executions can record an output branch hash.", status=409
|
|
464
|
+
)
|
|
465
|
+
record["candidate_result_hash"] = candidate_result_hash
|
|
466
|
+
_store(root, record, expected_status="passed")
|
|
467
|
+
return record
|
phlo_transform/plugin.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Plugins that expose Phlo Transform to the Phlo capability system."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Iterable
|
|
6
|
+
|
|
7
|
+
from phlo.capabilities import AssetSpec
|
|
8
|
+
from phlo.plugins.base import AssetProviderPlugin, PluginMetadata, TransformationProviderPlugin
|
|
9
|
+
|
|
10
|
+
from phlo_transform import __version__
|
|
11
|
+
from phlo_transform.assets import build_transform_asset_specs
|
|
12
|
+
from phlo_transform.cli_plugin import TransformCliPlugin
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _metadata() -> PluginMetadata:
|
|
16
|
+
return PluginMetadata(
|
|
17
|
+
name="transform",
|
|
18
|
+
version=__version__,
|
|
19
|
+
description="SQL transforms from phlo.transform.sql and Phlo Transform workspaces",
|
|
20
|
+
homepage="https://github.com/phlohouse/phlo-transform",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class TransformAssetProvider(AssetProviderPlugin):
|
|
25
|
+
"""Expose transform declarations and compiled workspace models as assets."""
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def metadata(self) -> PluginMetadata:
|
|
29
|
+
"""Return plugin metadata."""
|
|
30
|
+
|
|
31
|
+
return _metadata()
|
|
32
|
+
|
|
33
|
+
def get_assets(self) -> Iterable[AssetSpec]:
|
|
34
|
+
"""Return every transform-backed asset spec."""
|
|
35
|
+
|
|
36
|
+
return build_transform_asset_specs()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class TransformProvider(TransformationProviderPlugin):
|
|
40
|
+
"""Supply the ``transform`` transformation provider contract."""
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def metadata(self) -> PluginMetadata:
|
|
44
|
+
"""Return plugin metadata."""
|
|
45
|
+
|
|
46
|
+
return _metadata()
|
|
47
|
+
|
|
48
|
+
def get_asset_retriever(self) -> Callable[[], list[AssetSpec]]:
|
|
49
|
+
"""Return the function that builds transform asset specs."""
|
|
50
|
+
|
|
51
|
+
return build_transform_asset_specs
|
|
52
|
+
|
|
53
|
+
def get_cli_plugin(self) -> type:
|
|
54
|
+
"""Return the ``phlo transform`` command plugin."""
|
|
55
|
+
|
|
56
|
+
return TransformCliPlugin
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
__all__ = ["TransformAssetProvider", "TransformProvider"]
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Settings for the Phlo Transform provider package.
|
|
2
|
+
|
|
3
|
+
All values are optional so a project works with zero configuration once the
|
|
4
|
+
CLI binary is on ``PATH`` or installed into ``.phlo/bin``. Environment names
|
|
5
|
+
follow the ``PHLO_TRANSFORM_*`` prefix used by the CLI itself.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from pydantic import AliasChoices, Field
|
|
13
|
+
|
|
14
|
+
from phlo.config.base import BaseConfig
|
|
15
|
+
from phlo.config.env import load_project_env, resolve_project_root
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TransformSettings(BaseConfig):
|
|
19
|
+
"""Configuration for locating and running the Phlo Transform engine."""
|
|
20
|
+
|
|
21
|
+
binary: str | None = Field(
|
|
22
|
+
default=None,
|
|
23
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_BIN"),
|
|
24
|
+
description="Path to the phlo-transform executable; overrides project and PATH lookup.",
|
|
25
|
+
)
|
|
26
|
+
root: str | None = Field(
|
|
27
|
+
default=None,
|
|
28
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_ROOT"),
|
|
29
|
+
description="Transform workspace containing phlo.toml; auto-discovered when unset.",
|
|
30
|
+
)
|
|
31
|
+
adapter: str | None = Field(
|
|
32
|
+
default=None,
|
|
33
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_ADAPTER"),
|
|
34
|
+
description="Execution adapter (trino or duckdb); inferred from the query engine when unset.",
|
|
35
|
+
)
|
|
36
|
+
duckdb_path: str | None = Field(
|
|
37
|
+
default=None,
|
|
38
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_DUCKDB_PATH"),
|
|
39
|
+
description="DuckDB database file used when the adapter is duckdb.",
|
|
40
|
+
)
|
|
41
|
+
trino_endpoint: str | None = Field(
|
|
42
|
+
default=None,
|
|
43
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_TRINO_ENDPOINT"),
|
|
44
|
+
description="Trino HTTP endpoint; defaults to the resolved query engine.",
|
|
45
|
+
)
|
|
46
|
+
trino_user: str | None = Field(
|
|
47
|
+
default=None,
|
|
48
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_TRINO_USER"),
|
|
49
|
+
description="Trino user name.",
|
|
50
|
+
)
|
|
51
|
+
trino_password: str | None = Field(
|
|
52
|
+
default=None,
|
|
53
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_TRINO_PASSWORD"),
|
|
54
|
+
description="Trino password; prefer the environment over checked-in files.",
|
|
55
|
+
)
|
|
56
|
+
trino_catalog: str | None = Field(
|
|
57
|
+
default=None,
|
|
58
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_TRINO_CATALOG"),
|
|
59
|
+
description="Trino catalog for model targets; defaults to the query engine catalog.",
|
|
60
|
+
)
|
|
61
|
+
trino_schema: str | None = Field(
|
|
62
|
+
default=None,
|
|
63
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_TRINO_SCHEMA"),
|
|
64
|
+
description="Trino session schema.",
|
|
65
|
+
)
|
|
66
|
+
jobs: int = Field(
|
|
67
|
+
default=4,
|
|
68
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_JOBS"),
|
|
69
|
+
description="Maximum concurrent model builds.",
|
|
70
|
+
)
|
|
71
|
+
group_execution: bool = Field(
|
|
72
|
+
default=True,
|
|
73
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_GROUP_EXECUTION"),
|
|
74
|
+
description="Run compatible workspace assets together; disable for legacy per-model Dagster op configuration.",
|
|
75
|
+
)
|
|
76
|
+
asset_key: str = Field(
|
|
77
|
+
default="leaf",
|
|
78
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_ASSET_KEY"),
|
|
79
|
+
description=(
|
|
80
|
+
"Asset key shape for workspace models: leaf uses the last dotted "
|
|
81
|
+
"name segment (dbt-compatible); logical keeps the namespace prefix."
|
|
82
|
+
),
|
|
83
|
+
)
|
|
84
|
+
release_version: str = Field(
|
|
85
|
+
default="0.1.0",
|
|
86
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_RELEASE_VERSION"),
|
|
87
|
+
description="Release whose portable binary phlo transform install downloads.",
|
|
88
|
+
)
|
|
89
|
+
timeout_seconds: float = Field(
|
|
90
|
+
default=900.0,
|
|
91
|
+
validation_alias=AliasChoices("PHLO_TRANSFORM_TIMEOUT_SECONDS"),
|
|
92
|
+
description="Upper bound for one phlo-transform command.",
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
@property
|
|
96
|
+
def asset_key_mode(self) -> str:
|
|
97
|
+
"""Return the validated asset-key mode."""
|
|
98
|
+
|
|
99
|
+
mode = self.asset_key.strip().lower()
|
|
100
|
+
if mode not in {"leaf", "logical"}:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
f"PHLO_TRANSFORM_ASSET_KEY must be 'leaf' or 'logical', got '{self.asset_key}'"
|
|
103
|
+
)
|
|
104
|
+
return mode
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
_ENV_KEYS = (
|
|
108
|
+
"PHLO_TRANSFORM_BIN",
|
|
109
|
+
"PHLO_TRANSFORM_ROOT",
|
|
110
|
+
"PHLO_TRANSFORM_ADAPTER",
|
|
111
|
+
"PHLO_TRANSFORM_DUCKDB_PATH",
|
|
112
|
+
"PHLO_TRANSFORM_TRINO_ENDPOINT",
|
|
113
|
+
"PHLO_TRANSFORM_TRINO_USER",
|
|
114
|
+
"PHLO_TRANSFORM_TRINO_PASSWORD",
|
|
115
|
+
"PHLO_TRANSFORM_TRINO_CATALOG",
|
|
116
|
+
"PHLO_TRANSFORM_TRINO_SCHEMA",
|
|
117
|
+
"PHLO_TRANSFORM_JOBS",
|
|
118
|
+
"PHLO_TRANSFORM_GROUP_EXECUTION",
|
|
119
|
+
"PHLO_TRANSFORM_ASSET_KEY",
|
|
120
|
+
"PHLO_TRANSFORM_RELEASE_VERSION",
|
|
121
|
+
"PHLO_TRANSFORM_TIMEOUT_SECONDS",
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def get_settings(project_root: Path | str | None = None) -> TransformSettings:
|
|
126
|
+
"""Return provider settings for a project root.
|
|
127
|
+
|
|
128
|
+
``phlo.yaml`` ``env:`` values are part of the project environment layered
|
|
129
|
+
over ``.phlo/.env`` files, so they are read here too; a project can point
|
|
130
|
+
at its workspace without exporting a shell variable.
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
root = resolve_project_root(project_root)
|
|
134
|
+
env = load_project_env(root)
|
|
135
|
+
values = {key: env[key] for key in _ENV_KEYS if env.get(key)}
|
|
136
|
+
return TransformSettings.model_validate(values)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
__all__ = ["TransformSettings", "get_settings"]
|