phlo-transform 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_transform/__init__.py +23 -0
- phlo_transform/assets.py +552 -0
- phlo_transform/authorization.py +62 -0
- phlo_transform/binary.py +343 -0
- phlo_transform/cli_plugin.py +402 -0
- phlo_transform/editor.py +679 -0
- phlo_transform/editor_execution.py +467 -0
- phlo_transform/plugin.py +59 -0
- phlo_transform/settings.py +139 -0
- phlo_transform/translate.py +72 -0
- phlo_transform/workspace.py +289 -0
- phlo_transform-0.2.0.dist-info/METADATA +255 -0
- phlo_transform-0.2.0.dist-info/RECORD +16 -0
- phlo_transform-0.2.0.dist-info/WHEEL +5 -0
- phlo_transform-0.2.0.dist-info/entry_points.txt +8 -0
- phlo_transform-0.2.0.dist-info/top_level.txt +1 -0
phlo_transform/editor.py
ADDED
|
@@ -0,0 +1,679 @@
|
|
|
1
|
+
"""Contained SQL draft revisions and genuine read-only engine compilation.
|
|
2
|
+
|
|
3
|
+
Drafts are immutable workspace snapshots, not Git branches or published source
|
|
4
|
+
files. The Rust engine owns compilation and planning; this adapter only keeps
|
|
5
|
+
source identity, storage and invocation boundaries intact.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import shutil
|
|
13
|
+
from datetime import UTC, datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
from uuid import UUID, uuid4
|
|
17
|
+
|
|
18
|
+
from phlo.plugins.observatory_settings import SettingsScope, get_settings_service
|
|
19
|
+
from phlo_transform.assets import adapter_args, provider_is_active
|
|
20
|
+
from phlo_transform.binary import require_binary, run_json
|
|
21
|
+
from phlo_transform.settings import TransformSettings, get_settings
|
|
22
|
+
from phlo_transform.workspace import TransformWorkspace, load_workspace
|
|
23
|
+
|
|
24
|
+
_MAX_SOURCE_BYTES = 256 * 1024
|
|
25
|
+
_MAX_WORKSPACE_BYTES = 16 * 1024 * 1024
|
|
26
|
+
_MAX_RETAINED_REVISIONS = 1000
|
|
27
|
+
_INPUT_SUFFIXES = {".sql", ".toml", ".yaml", ".yml", ".json", ".csv"}
|
|
28
|
+
_SKIPPED = {".git", ".phlo", ".venv", "node_modules", "target", "dbt_packages", "__pycache__"}
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class TransformEditorError(RuntimeError):
|
|
32
|
+
"""A client-safe failure to read, save or compile a specific SQL revision."""
|
|
33
|
+
|
|
34
|
+
def __init__(self, message: str, *, reason: str = "invalid", status: int = 422) -> None:
|
|
35
|
+
super().__init__(message)
|
|
36
|
+
self.reason = reason
|
|
37
|
+
self.status = status
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _digest(content: bytes) -> str:
|
|
41
|
+
return hashlib.sha256(content).hexdigest()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _workspace(root: Path) -> TransformWorkspace:
|
|
45
|
+
if not provider_is_active(root):
|
|
46
|
+
raise TransformEditorError("Phlo Transform is not the active provider.", status=503)
|
|
47
|
+
workspace = load_workspace(root, get_settings(root))
|
|
48
|
+
if workspace is None:
|
|
49
|
+
raise TransformEditorError("No native Phlo Transform workspace is configured.", status=503)
|
|
50
|
+
_contained(root, workspace.root)
|
|
51
|
+
return workspace
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _contained(root: Path, path: Path) -> Path:
|
|
55
|
+
resolved = path.resolve()
|
|
56
|
+
if not resolved.is_relative_to(root.resolve()):
|
|
57
|
+
raise TransformEditorError("The model source is outside the project workspace.")
|
|
58
|
+
return resolved
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _inputs(root: Path) -> dict[str, bytes]:
|
|
62
|
+
files: dict[str, bytes] = {}
|
|
63
|
+
total = 0
|
|
64
|
+
for path in sorted(root.rglob("*")):
|
|
65
|
+
relative = path.relative_to(root)
|
|
66
|
+
if _SKIPPED.intersection(relative.parts) or path.suffix.lower() not in _INPUT_SUFFIXES:
|
|
67
|
+
continue
|
|
68
|
+
if path.is_symlink():
|
|
69
|
+
raise TransformEditorError(
|
|
70
|
+
"Symbolic links are not supported in editable transform inputs."
|
|
71
|
+
)
|
|
72
|
+
if not path.is_file():
|
|
73
|
+
continue
|
|
74
|
+
content = _contained(root, path).read_bytes()
|
|
75
|
+
total += len(content)
|
|
76
|
+
if total > _MAX_WORKSPACE_BYTES or len(files) >= 1000:
|
|
77
|
+
raise TransformEditorError("The transform workspace exceeds the editor snapshot limit.")
|
|
78
|
+
files[relative.as_posix()] = content
|
|
79
|
+
return files
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _workspace_digest(files: dict[str, bytes]) -> str:
|
|
83
|
+
return _digest(
|
|
84
|
+
json.dumps(
|
|
85
|
+
[(name, _digest(content)) for name, content in sorted(files.items())],
|
|
86
|
+
separators=(",", ":"),
|
|
87
|
+
).encode()
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _source(root: Path, model: str) -> tuple[TransformWorkspace, str, dict[str, bytes]]:
|
|
92
|
+
workspace = _workspace(root)
|
|
93
|
+
selected = workspace.model(model)
|
|
94
|
+
if selected is None or not selected.path:
|
|
95
|
+
raise TransformEditorError("The selected model has no editable SQL source.", status=404)
|
|
96
|
+
path = _contained(workspace.root, workspace.root / selected.path)
|
|
97
|
+
if path.suffix.lower() != ".sql" or path.is_symlink():
|
|
98
|
+
raise TransformEditorError("Only contained SQL model files can be edited.")
|
|
99
|
+
relative = path.relative_to(workspace.root.resolve()).as_posix()
|
|
100
|
+
files = _inputs(workspace.root)
|
|
101
|
+
if relative not in files or len(files[relative]) > _MAX_SOURCE_BYTES:
|
|
102
|
+
raise TransformEditorError("The selected SQL source is unavailable or too large.")
|
|
103
|
+
return workspace, relative, files
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def read_model_source(root: Path, model: str) -> dict[str, Any]:
|
|
107
|
+
workspace, relative, files = _source(root, model)
|
|
108
|
+
selected = workspace.model(model)
|
|
109
|
+
assert selected is not None
|
|
110
|
+
return {
|
|
111
|
+
"model": model,
|
|
112
|
+
"path": relative,
|
|
113
|
+
"project_path": (workspace.root / relative).relative_to(root.resolve()).as_posix(),
|
|
114
|
+
"sql": files[relative].decode("utf-8"),
|
|
115
|
+
"source_digest": _digest(files[relative]),
|
|
116
|
+
"workspace_digest": _workspace_digest(files),
|
|
117
|
+
"model_version": selected.version,
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _namespace(root: Path) -> str:
|
|
122
|
+
return "transform.editor." + _digest(str(root.resolve()).encode())
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _records(
|
|
126
|
+
state: dict[str, Any] | None, *, require_known_state: bool = False
|
|
127
|
+
) -> list[dict[str, Any]]:
|
|
128
|
+
if state is None:
|
|
129
|
+
return []
|
|
130
|
+
if require_known_state and set(state) - {"schema_version", "items"}:
|
|
131
|
+
raise TransformEditorError(
|
|
132
|
+
"Unknown mutation or reference metadata prevents reclamation.", status=409
|
|
133
|
+
)
|
|
134
|
+
if state.get("schema_version") != 1 or not isinstance(state.get("items"), list):
|
|
135
|
+
raise TransformEditorError("Stored transform draft metadata is unavailable.", status=503)
|
|
136
|
+
return state["items"]
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _uuid(value: str) -> str:
|
|
140
|
+
try:
|
|
141
|
+
return str(UUID(value))
|
|
142
|
+
except (ValueError, AttributeError) as exc:
|
|
143
|
+
raise TransformEditorError("Invalid transform draft identity.") from exc
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _artifact(root: Path, draft_id: str, artifact_id: str) -> Path:
|
|
147
|
+
return _contained(
|
|
148
|
+
root,
|
|
149
|
+
root / ".phlo" / "transform-editor" / _uuid(draft_id) / _uuid(artifact_id) / "workspace",
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def save_model_draft(
|
|
154
|
+
root: Path,
|
|
155
|
+
model: str,
|
|
156
|
+
*,
|
|
157
|
+
actor: str,
|
|
158
|
+
env: str,
|
|
159
|
+
ref: str,
|
|
160
|
+
sql: str,
|
|
161
|
+
source_digest: str,
|
|
162
|
+
workspace_digest: str,
|
|
163
|
+
draft_id: str | None = None,
|
|
164
|
+
expected_revision: int = 0,
|
|
165
|
+
) -> dict[str, Any]:
|
|
166
|
+
if len(sql.encode("utf-8")) > _MAX_SOURCE_BYTES or not sql.strip():
|
|
167
|
+
raise TransformEditorError("Provide non-empty SQL within the 256 KiB editor limit.")
|
|
168
|
+
workspace, relative, files = _source(root, model)
|
|
169
|
+
if _digest(files[relative]) != source_digest or _workspace_digest(files) != workspace_digest:
|
|
170
|
+
raise TransformEditorError(
|
|
171
|
+
"The source workspace changed. Reload before saving a draft.",
|
|
172
|
+
reason="conflict",
|
|
173
|
+
status=409,
|
|
174
|
+
)
|
|
175
|
+
if (draft_id is None) != (expected_revision == 0):
|
|
176
|
+
raise TransformEditorError(
|
|
177
|
+
"New drafts require revision zero; updates require a draft identity."
|
|
178
|
+
)
|
|
179
|
+
identity = _uuid(draft_id) if draft_id else str(uuid4())
|
|
180
|
+
artifact_id = str(uuid4())
|
|
181
|
+
destination = _artifact(root, identity, artifact_id)
|
|
182
|
+
files[relative] = sql.encode("utf-8")
|
|
183
|
+
record = {
|
|
184
|
+
"draft_id": identity,
|
|
185
|
+
"artifact_id": artifact_id,
|
|
186
|
+
"revision": expected_revision + 1,
|
|
187
|
+
"model": model,
|
|
188
|
+
"actor": actor,
|
|
189
|
+
"env": env,
|
|
190
|
+
"ref": ref,
|
|
191
|
+
"path": relative,
|
|
192
|
+
"project_path": (workspace.root / relative).relative_to(root.resolve()).as_posix(),
|
|
193
|
+
"source_digest": source_digest,
|
|
194
|
+
"workspace_digest": workspace_digest,
|
|
195
|
+
"revision_digest": _workspace_digest(files),
|
|
196
|
+
"sql_digest": _digest(files[relative]),
|
|
197
|
+
"created_at": datetime.now(UTC).isoformat(),
|
|
198
|
+
"archived_at": None,
|
|
199
|
+
"archived_by": None,
|
|
200
|
+
"archive_reason": None,
|
|
201
|
+
}
|
|
202
|
+
try:
|
|
203
|
+
for name, content in files.items():
|
|
204
|
+
path = _contained(destination, destination / name)
|
|
205
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
206
|
+
path.write_bytes(content)
|
|
207
|
+
|
|
208
|
+
def change(current: dict[str, Any] | None) -> dict[str, Any]:
|
|
209
|
+
records = _records(current)
|
|
210
|
+
history = [item for item in records if item.get("draft_id") == identity]
|
|
211
|
+
if history and any(
|
|
212
|
+
item.get(key) != record[key]
|
|
213
|
+
for item in history
|
|
214
|
+
for key in ("actor", "env", "ref", "model")
|
|
215
|
+
):
|
|
216
|
+
raise TransformEditorError(
|
|
217
|
+
"Draft revision was not found in this scope.", status=404
|
|
218
|
+
)
|
|
219
|
+
if max((item.get("revision", 0) for item in history), default=0) != expected_revision:
|
|
220
|
+
raise TransformEditorError(
|
|
221
|
+
"The draft changed. Reload its latest revision before saving.",
|
|
222
|
+
reason="conflict",
|
|
223
|
+
status=409,
|
|
224
|
+
)
|
|
225
|
+
if history and any(
|
|
226
|
+
item.get("archived_at") for item in history if item["revision"] == expected_revision
|
|
227
|
+
):
|
|
228
|
+
raise TransformEditorError(
|
|
229
|
+
"Restore the latest revision before editing this draft.", status=409
|
|
230
|
+
)
|
|
231
|
+
if sum(not item.get("reclaimed_at") for item in records) >= _MAX_RETAINED_REVISIONS:
|
|
232
|
+
raise TransformEditorError("The transform draft revision limit was reached.")
|
|
233
|
+
return {"schema_version": 1, "items": [*records, record]}
|
|
234
|
+
|
|
235
|
+
get_settings_service().mutate(SettingsScope.GLOBAL, _namespace(root), change)
|
|
236
|
+
except Exception:
|
|
237
|
+
shutil.rmtree(destination.parent, ignore_errors=True)
|
|
238
|
+
raise
|
|
239
|
+
return {**record, "sql": sql}
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def read_model_draft(
|
|
243
|
+
root: Path, model: str, *, draft_id: str, revision: int, actor: str, env: str, ref: str
|
|
244
|
+
) -> dict[str, Any]:
|
|
245
|
+
state = get_settings_service().get(SettingsScope.GLOBAL, _namespace(root))
|
|
246
|
+
found = next(
|
|
247
|
+
(
|
|
248
|
+
item
|
|
249
|
+
for item in _records(state.settings if state else None)
|
|
250
|
+
if item.get("draft_id") == _uuid(draft_id)
|
|
251
|
+
and item.get("revision") == revision
|
|
252
|
+
and item.get("actor") == actor
|
|
253
|
+
and item.get("env") == env
|
|
254
|
+
and item.get("ref") == ref
|
|
255
|
+
and item.get("model") == model
|
|
256
|
+
),
|
|
257
|
+
None,
|
|
258
|
+
)
|
|
259
|
+
if found is None:
|
|
260
|
+
raise TransformEditorError("Draft revision was not found in this scope.", status=404)
|
|
261
|
+
if found.get("reclaim_started_at"):
|
|
262
|
+
sql = found["retained_sql"]
|
|
263
|
+
if _digest(sql.encode()) != found["sql_digest"]:
|
|
264
|
+
raise TransformEditorError("Retained source checksum changed.", status=409)
|
|
265
|
+
return {**found, "sql": sql}
|
|
266
|
+
directory = _artifact(root, found["draft_id"], found["artifact_id"])
|
|
267
|
+
files = _inputs(directory)
|
|
268
|
+
if (
|
|
269
|
+
_workspace_digest(files) != found["revision_digest"]
|
|
270
|
+
or found["path"] not in files
|
|
271
|
+
or _digest(files[found["path"]]) != found["sql_digest"]
|
|
272
|
+
):
|
|
273
|
+
raise TransformEditorError(
|
|
274
|
+
"Draft revision integrity verification failed.", reason="tampered", status=409
|
|
275
|
+
)
|
|
276
|
+
return {**found, "sql": files[found["path"]].decode("utf-8")}
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def archive_model_draft(
|
|
280
|
+
root: Path, model: str, *, archived: bool, reason: str, **identity: Any
|
|
281
|
+
) -> dict[str, Any]:
|
|
282
|
+
"""Preserve immutable source and its identity while changing retention visibility."""
|
|
283
|
+
saved = read_model_draft(root, model, **identity)
|
|
284
|
+
if saved.get("reclaim_started_at"):
|
|
285
|
+
raise TransformEditorError(
|
|
286
|
+
"Reclamation is irreversible; revision cannot be restored.", status=409
|
|
287
|
+
)
|
|
288
|
+
changes = {
|
|
289
|
+
"archived_at": datetime.now(UTC).isoformat() if archived else None,
|
|
290
|
+
"archived_by": identity["actor"] if archived else None,
|
|
291
|
+
"archive_reason": reason if archived else None,
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
def update(current):
|
|
295
|
+
records = _records(current)
|
|
296
|
+
record = next(
|
|
297
|
+
item
|
|
298
|
+
for item in records
|
|
299
|
+
if item["draft_id"] == saved["draft_id"] and item["revision"] == saved["revision"]
|
|
300
|
+
)
|
|
301
|
+
if any(saved[key] != value for key, value in record.items()):
|
|
302
|
+
raise TransformEditorError("Revision changed during archival.", status=409)
|
|
303
|
+
return {
|
|
304
|
+
"schema_version": 1,
|
|
305
|
+
"items": [{**item, **changes} if item is record else item for item in records],
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
get_settings_service().mutate(SettingsScope.GLOBAL, _namespace(root), update)
|
|
309
|
+
return {**saved, **changes}
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _reclaim_workspace_bytes(root: Path, directory: Path) -> int:
|
|
313
|
+
"""Measure only the literal retained workspace, never a symlink destination."""
|
|
314
|
+
if directory.absolute() != directory.resolve() or not directory.is_relative_to(
|
|
315
|
+
root.resolve() / ".phlo" / "transform-editor"
|
|
316
|
+
):
|
|
317
|
+
raise TransformEditorError(
|
|
318
|
+
"Reclamation requires a contained non-symlink workspace.", status=409
|
|
319
|
+
)
|
|
320
|
+
return sum(
|
|
321
|
+
path.stat().st_size
|
|
322
|
+
for path in directory.rglob("*")
|
|
323
|
+
if path.is_file() and not path.is_symlink()
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _reclaim_workspace(root: Path, directory: Path) -> int:
|
|
328
|
+
"""Remove only the literal retained workspace, never a symlink destination."""
|
|
329
|
+
size = _reclaim_workspace_bytes(root, directory)
|
|
330
|
+
if directory.exists():
|
|
331
|
+
shutil.rmtree(directory)
|
|
332
|
+
return size
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def reclaim_model_draft(
|
|
336
|
+
root: Path,
|
|
337
|
+
model: str,
|
|
338
|
+
*,
|
|
339
|
+
expected_revision_digest: str,
|
|
340
|
+
reason: str,
|
|
341
|
+
expected_record: dict[str, Any] | None = None,
|
|
342
|
+
**identity: Any,
|
|
343
|
+
) -> dict[str, Any]:
|
|
344
|
+
"""Reclaim an archived snapshot but retain SQL, file digests and all revision identities."""
|
|
345
|
+
from phlo_transform.editor_execution import _EXECUTION_FIELDS, _storage
|
|
346
|
+
|
|
347
|
+
saved = read_model_draft(root, model, **identity)
|
|
348
|
+
if expected_record is not None and saved != expected_record:
|
|
349
|
+
raise TransformEditorError("Revision changed after retention validation.", status=409)
|
|
350
|
+
known = {
|
|
351
|
+
"draft_id",
|
|
352
|
+
"artifact_id",
|
|
353
|
+
"revision",
|
|
354
|
+
"model",
|
|
355
|
+
"actor",
|
|
356
|
+
"env",
|
|
357
|
+
"ref",
|
|
358
|
+
"path",
|
|
359
|
+
"project_path",
|
|
360
|
+
"source_digest",
|
|
361
|
+
"workspace_digest",
|
|
362
|
+
"revision_digest",
|
|
363
|
+
"sql_digest",
|
|
364
|
+
"created_at",
|
|
365
|
+
"sql",
|
|
366
|
+
"archived_at",
|
|
367
|
+
"archived_by",
|
|
368
|
+
"archive_reason",
|
|
369
|
+
"reclaim_started_at",
|
|
370
|
+
"reclaimed_at",
|
|
371
|
+
"reclaimed_by",
|
|
372
|
+
"reclaim_reason",
|
|
373
|
+
"reclaimed_bytes",
|
|
374
|
+
"reclaim_payload_bytes",
|
|
375
|
+
"retained_sql",
|
|
376
|
+
"retained_file_digests",
|
|
377
|
+
}
|
|
378
|
+
if set(saved) - known:
|
|
379
|
+
raise TransformEditorError("Unknown revision evidence prevents reclamation.", status=409)
|
|
380
|
+
if not saved.get("archived_at") or saved["revision_digest"] != expected_revision_digest:
|
|
381
|
+
raise TransformEditorError(
|
|
382
|
+
"An archived revision and exact digest are required.", status=409
|
|
383
|
+
)
|
|
384
|
+
if saved.get("reclaimed_at"):
|
|
385
|
+
return saved
|
|
386
|
+
state = get_settings_service().get(SettingsScope.GLOBAL, _storage(root))
|
|
387
|
+
if any(
|
|
388
|
+
set(item) - _EXECUTION_FIELDS
|
|
389
|
+
or (
|
|
390
|
+
item.get("draft_id") == saved["draft_id"]
|
|
391
|
+
and item.get("revision") == saved["revision"]
|
|
392
|
+
and item.get("status") not in {"passed", "failed"}
|
|
393
|
+
)
|
|
394
|
+
for item in _records(state.settings if state else None, require_known_state=True)
|
|
395
|
+
):
|
|
396
|
+
raise TransformEditorError(
|
|
397
|
+
"Unresolved executions must retain their source snapshot.", status=409
|
|
398
|
+
)
|
|
399
|
+
directory = (
|
|
400
|
+
root.resolve()
|
|
401
|
+
/ ".phlo/transform-editor"
|
|
402
|
+
/ saved["draft_id"]
|
|
403
|
+
/ saved["artifact_id"]
|
|
404
|
+
/ "workspace"
|
|
405
|
+
)
|
|
406
|
+
started = {key: value for key, value in saved.items() if key != "sql"}
|
|
407
|
+
if not saved.get("reclaim_started_at"):
|
|
408
|
+
started.update(
|
|
409
|
+
reclaim_started_at=datetime.now(UTC).isoformat(),
|
|
410
|
+
reclaimed_by=identity["actor"],
|
|
411
|
+
reclaim_reason=reason,
|
|
412
|
+
reclaim_payload_bytes=_reclaim_workspace_bytes(root, directory),
|
|
413
|
+
retained_sql=saved["sql"],
|
|
414
|
+
retained_file_digests={
|
|
415
|
+
name: _digest(data) for name, data in _inputs(directory).items()
|
|
416
|
+
},
|
|
417
|
+
)
|
|
418
|
+
size = started.get("reclaim_payload_bytes")
|
|
419
|
+
if type(size) is not int or size < 0:
|
|
420
|
+
raise TransformEditorError("Original reclamation byte evidence is unavailable.", status=409)
|
|
421
|
+
|
|
422
|
+
def store(record):
|
|
423
|
+
def update(current):
|
|
424
|
+
records = _records(current, require_known_state=True)
|
|
425
|
+
if any(set(item) - known for item in records):
|
|
426
|
+
raise TransformEditorError(
|
|
427
|
+
"Unknown revision references prevent reclamation.", status=409
|
|
428
|
+
)
|
|
429
|
+
existing = next(item for item in records if item["artifact_id"] == saved["artifact_id"])
|
|
430
|
+
expected = {key: value for key, value in saved.items() if key != "sql"}
|
|
431
|
+
if existing != expected and existing != started:
|
|
432
|
+
raise TransformEditorError("Revision changed during reclamation.", status=409)
|
|
433
|
+
return {
|
|
434
|
+
"schema_version": 1,
|
|
435
|
+
"items": [record if item is existing else item for item in records],
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
get_settings_service().mutate(SettingsScope.GLOBAL, _namespace(root), update)
|
|
439
|
+
|
|
440
|
+
store(started)
|
|
441
|
+
_reclaim_workspace(root, directory)
|
|
442
|
+
finished = {**started, "reclaimed_at": datetime.now(UTC).isoformat(), "reclaimed_bytes": size}
|
|
443
|
+
store(finished)
|
|
444
|
+
return {**finished, "sql": saved["sql"]}
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def preview_settings(
|
|
448
|
+
root: Path,
|
|
449
|
+
*,
|
|
450
|
+
ref: str,
|
|
451
|
+
required_ref: str | None = None,
|
|
452
|
+
required_catalog: str | None = None,
|
|
453
|
+
required_endpoint: str | None = None,
|
|
454
|
+
required_user: str | None = None,
|
|
455
|
+
) -> TransformSettings:
|
|
456
|
+
"""Validate local native prerequisites against the caller's attested preview binding."""
|
|
457
|
+
if not provider_is_active(root):
|
|
458
|
+
raise TransformEditorError("Phlo Transform is not the active provider.", status=503)
|
|
459
|
+
if required_ref is not None and required_ref != ref:
|
|
460
|
+
raise TransformEditorError(
|
|
461
|
+
"The preview binding does not match the draft reference.", status=503
|
|
462
|
+
)
|
|
463
|
+
if ref != "main" and not all(
|
|
464
|
+
(required_ref, required_catalog, required_endpoint, required_user)
|
|
465
|
+
):
|
|
466
|
+
raise TransformEditorError(
|
|
467
|
+
"A complete preview binding is required for this reference.", status=503
|
|
468
|
+
)
|
|
469
|
+
settings = get_settings(root)
|
|
470
|
+
require_binary(root, settings)
|
|
471
|
+
if settings.adapter != "trino" or not settings.trino_endpoint or not settings.trino_catalog:
|
|
472
|
+
raise TransformEditorError(
|
|
473
|
+
"Read-only planning requires an explicitly configured Trino endpoint and catalog.",
|
|
474
|
+
status=503,
|
|
475
|
+
)
|
|
476
|
+
if required_catalog is not None and settings.trino_catalog != required_catalog:
|
|
477
|
+
raise TransformEditorError(
|
|
478
|
+
"The engine catalog does not match the selected environment.", status=503
|
|
479
|
+
)
|
|
480
|
+
if required_endpoint is not None and (
|
|
481
|
+
settings.trino_endpoint.rstrip("/") != required_endpoint.rstrip("/")
|
|
482
|
+
or settings.trino_user != required_user
|
|
483
|
+
):
|
|
484
|
+
raise TransformEditorError(
|
|
485
|
+
"The engine connection does not match the environment's read-only preview identity.",
|
|
486
|
+
status=503,
|
|
487
|
+
)
|
|
488
|
+
return settings
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def compile_model_draft(
|
|
492
|
+
root: Path,
|
|
493
|
+
model: str,
|
|
494
|
+
*,
|
|
495
|
+
allowed_sources: set[str],
|
|
496
|
+
required_ref: str | None = None,
|
|
497
|
+
required_catalog: str | None = None,
|
|
498
|
+
required_endpoint: str | None = None,
|
|
499
|
+
required_user: str | None = None,
|
|
500
|
+
trusted_ca_file: str | None = None,
|
|
501
|
+
**identity: Any,
|
|
502
|
+
) -> dict[str, Any]:
|
|
503
|
+
record = read_model_draft(root, model, **identity)
|
|
504
|
+
if record.get("archived_at"):
|
|
505
|
+
raise TransformEditorError("Restore the archived revision before compiling it.", status=409)
|
|
506
|
+
directory = _artifact(root, record["draft_id"], record["artifact_id"])
|
|
507
|
+
settings = preview_settings(
|
|
508
|
+
root,
|
|
509
|
+
ref=record["ref"],
|
|
510
|
+
required_ref=required_ref,
|
|
511
|
+
required_catalog=required_catalog,
|
|
512
|
+
required_endpoint=required_endpoint,
|
|
513
|
+
required_user=required_user,
|
|
514
|
+
)
|
|
515
|
+
binary = require_binary(root, settings)
|
|
516
|
+
check, checked = run_json(binary, ["check"], settings=settings, root=directory)
|
|
517
|
+
if check is None:
|
|
518
|
+
raise TransformEditorError(
|
|
519
|
+
"The engine did not return a valid compilation report.", status=502
|
|
520
|
+
)
|
|
521
|
+
result = {
|
|
522
|
+
"draft_id": record["draft_id"],
|
|
523
|
+
"revision": record["revision"],
|
|
524
|
+
"revision_digest": record["revision_digest"],
|
|
525
|
+
"ref": record["ref"],
|
|
526
|
+
"status": "failed",
|
|
527
|
+
"diagnostics": check.get("diagnostics") or [],
|
|
528
|
+
"plan_id": None,
|
|
529
|
+
"compiled_sql": None,
|
|
530
|
+
"target": None,
|
|
531
|
+
"sql_hash": None,
|
|
532
|
+
"desired_version": None,
|
|
533
|
+
"materialization_available": False,
|
|
534
|
+
}
|
|
535
|
+
if not checked.ok:
|
|
536
|
+
return result
|
|
537
|
+
listed, listed_invocation = run_json(binary, ["list"], settings=settings, root=directory)
|
|
538
|
+
if listed is None or not listed_invocation.ok:
|
|
539
|
+
raise TransformEditorError("The engine did not report the draft source graph.", status=502)
|
|
540
|
+
for source in listed.get("sources") or []:
|
|
541
|
+
name = str(source.get("name") or "")
|
|
542
|
+
parts = name.split(".")
|
|
543
|
+
if len(parts) == 3 and parts[0] == settings.trino_catalog:
|
|
544
|
+
name = ".".join(parts[1:])
|
|
545
|
+
if name not in allowed_sources:
|
|
546
|
+
raise TransformEditorError(
|
|
547
|
+
"Every draft source must be an authorized asset in the configured environment catalog.",
|
|
548
|
+
reason="forbidden",
|
|
549
|
+
status=403,
|
|
550
|
+
)
|
|
551
|
+
plan, planned = run_json(
|
|
552
|
+
binary,
|
|
553
|
+
[
|
|
554
|
+
"plan",
|
|
555
|
+
"--catalogue",
|
|
556
|
+
"--select",
|
|
557
|
+
model,
|
|
558
|
+
*adapter_args(None, settings.model_copy(update={"trino_password": None}), infer=False),
|
|
559
|
+
],
|
|
560
|
+
settings=settings,
|
|
561
|
+
root=directory,
|
|
562
|
+
environment={
|
|
563
|
+
"PHLO_TRINO_PASSWORD": settings.trino_password or "",
|
|
564
|
+
**({"SSL_CERT_FILE": trusted_ca_file} if trusted_ca_file else {}),
|
|
565
|
+
},
|
|
566
|
+
)
|
|
567
|
+
if plan is None:
|
|
568
|
+
raise TransformEditorError("The engine did not return a valid read-only plan.", status=502)
|
|
569
|
+
selected = next(
|
|
570
|
+
(
|
|
571
|
+
item
|
|
572
|
+
for item in plan.get("models") or []
|
|
573
|
+
if item.get("id") in {model, "model://" + model.replace(".", "/")}
|
|
574
|
+
),
|
|
575
|
+
None,
|
|
576
|
+
)
|
|
577
|
+
result.update(
|
|
578
|
+
{
|
|
579
|
+
"status": "ready"
|
|
580
|
+
if planned.ok and not plan.get("blocked", True) and selected
|
|
581
|
+
else "failed",
|
|
582
|
+
"diagnostics": [*result["diagnostics"], *(plan.get("diagnostics") or [])],
|
|
583
|
+
"plan_id": plan.get("id"),
|
|
584
|
+
"compiled_sql": selected.get("compiled_sql") if selected else None,
|
|
585
|
+
"target": selected.get("target") if selected else None,
|
|
586
|
+
"sql_hash": selected.get("sql_hash") if selected else None,
|
|
587
|
+
"desired_version": selected.get("desired_version") if selected else None,
|
|
588
|
+
}
|
|
589
|
+
)
|
|
590
|
+
return result
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def list_model_drafts(
|
|
594
|
+
root: Path, model: str, *, actor: str, env: str, ref: str, include_archived: bool = False
|
|
595
|
+
) -> dict[str, Any]:
|
|
596
|
+
"""Return bounded, verified revision metadata from this exact principal scope."""
|
|
597
|
+
state = get_settings_service().get(SettingsScope.GLOBAL, _namespace(root))
|
|
598
|
+
records = [
|
|
599
|
+
item
|
|
600
|
+
for item in _records(state.settings if state else None)
|
|
601
|
+
if item.get("model") == model
|
|
602
|
+
and item.get("actor") == actor
|
|
603
|
+
and item.get("env") == env
|
|
604
|
+
and item.get("ref") == ref
|
|
605
|
+
and (include_archived or not item.get("archived_at"))
|
|
606
|
+
]
|
|
607
|
+
records.sort(key=lambda item: (item["created_at"], item["revision"]), reverse=True)
|
|
608
|
+
summaries = []
|
|
609
|
+
keys = (
|
|
610
|
+
"draft_id",
|
|
611
|
+
"revision",
|
|
612
|
+
"model",
|
|
613
|
+
"env",
|
|
614
|
+
"ref",
|
|
615
|
+
"path",
|
|
616
|
+
"created_at",
|
|
617
|
+
"source_digest",
|
|
618
|
+
"revision_digest",
|
|
619
|
+
"sql_digest",
|
|
620
|
+
"archived_at",
|
|
621
|
+
"archived_by",
|
|
622
|
+
"archive_reason",
|
|
623
|
+
"reclaim_started_at",
|
|
624
|
+
"reclaimed_at",
|
|
625
|
+
"reclaimed_by",
|
|
626
|
+
"reclaim_reason",
|
|
627
|
+
"reclaimed_bytes",
|
|
628
|
+
)
|
|
629
|
+
for record in records[:100]:
|
|
630
|
+
verified = read_model_draft(
|
|
631
|
+
root,
|
|
632
|
+
model,
|
|
633
|
+
actor=actor,
|
|
634
|
+
env=env,
|
|
635
|
+
ref=ref,
|
|
636
|
+
draft_id=record["draft_id"],
|
|
637
|
+
revision=record["revision"],
|
|
638
|
+
)
|
|
639
|
+
summaries.append(
|
|
640
|
+
{key: verified.get(key, 0 if key == "reclaimed_bytes" else None) for key in keys}
|
|
641
|
+
)
|
|
642
|
+
return {"items": summaries, "total": len(records), "truncated": len(records) > 100}
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def compare_model_draft(root: Path, model: str, **identity: Any) -> dict[str, Any]:
|
|
646
|
+
"""Compare an immutable SQL revision with actual current workspace source."""
|
|
647
|
+
from difflib import SequenceMatcher, unified_diff
|
|
648
|
+
|
|
649
|
+
saved = read_model_draft(root, model, **identity)
|
|
650
|
+
source = read_model_source(root, model)
|
|
651
|
+
before = source["sql"].split("\n")
|
|
652
|
+
after = saved["sql"].split("\n")
|
|
653
|
+
added = removed = 0
|
|
654
|
+
for operation, start_before, end_before, start_after, end_after in SequenceMatcher(
|
|
655
|
+
None, before, after, autojunk=False
|
|
656
|
+
).get_opcodes():
|
|
657
|
+
if operation in {"replace", "delete"}:
|
|
658
|
+
removed += end_before - start_before
|
|
659
|
+
if operation in {"replace", "insert"}:
|
|
660
|
+
added += end_after - start_after
|
|
661
|
+
return {
|
|
662
|
+
"draft_id": saved["draft_id"],
|
|
663
|
+
"revision": saved["revision"],
|
|
664
|
+
"revision_digest": saved["revision_digest"],
|
|
665
|
+
"source_digest": source["source_digest"],
|
|
666
|
+
"source_path": source["path"],
|
|
667
|
+
"baseline_matches_saved_source": source["source_digest"] == saved["source_digest"],
|
|
668
|
+
"added_lines": added,
|
|
669
|
+
"removed_lines": removed,
|
|
670
|
+
"unified_diff": list(
|
|
671
|
+
unified_diff(
|
|
672
|
+
before,
|
|
673
|
+
after,
|
|
674
|
+
fromfile="Current workspace source",
|
|
675
|
+
tofile=f"Saved SQL revision {saved['revision']}",
|
|
676
|
+
lineterm="",
|
|
677
|
+
)
|
|
678
|
+
),
|
|
679
|
+
}
|