phlo-transform 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- phlo_transform-0.2.0/PKG-INFO +255 -0
- phlo_transform-0.2.0/README.md +242 -0
- phlo_transform-0.2.0/pyproject.toml +49 -0
- phlo_transform-0.2.0/setup.cfg +4 -0
- phlo_transform-0.2.0/src/phlo_transform/__init__.py +23 -0
- phlo_transform-0.2.0/src/phlo_transform/assets.py +552 -0
- phlo_transform-0.2.0/src/phlo_transform/authorization.py +62 -0
- phlo_transform-0.2.0/src/phlo_transform/binary.py +343 -0
- phlo_transform-0.2.0/src/phlo_transform/cli_plugin.py +402 -0
- phlo_transform-0.2.0/src/phlo_transform/editor.py +679 -0
- phlo_transform-0.2.0/src/phlo_transform/editor_execution.py +467 -0
- phlo_transform-0.2.0/src/phlo_transform/plugin.py +59 -0
- phlo_transform-0.2.0/src/phlo_transform/settings.py +139 -0
- phlo_transform-0.2.0/src/phlo_transform/translate.py +72 -0
- phlo_transform-0.2.0/src/phlo_transform/workspace.py +289 -0
- phlo_transform-0.2.0/src/phlo_transform.egg-info/PKG-INFO +255 -0
- phlo_transform-0.2.0/src/phlo_transform.egg-info/SOURCES.txt +28 -0
- phlo_transform-0.2.0/src/phlo_transform.egg-info/dependency_links.txt +1 -0
- phlo_transform-0.2.0/src/phlo_transform.egg-info/entry_points.txt +8 -0
- phlo_transform-0.2.0/src/phlo_transform.egg-info/requires.txt +5 -0
- phlo_transform-0.2.0/src/phlo_transform.egg-info/top_level.txt +1 -0
- phlo_transform-0.2.0/tests/test_assets.py +439 -0
- phlo_transform-0.2.0/tests/test_binary.py +182 -0
- phlo_transform-0.2.0/tests/test_cli.py +169 -0
- phlo_transform-0.2.0/tests/test_editor.py +423 -0
- phlo_transform-0.2.0/tests/test_editor_execution.py +556 -0
- phlo_transform-0.2.0/tests/test_integration_dagster.py +132 -0
- phlo_transform-0.2.0/tests/test_plugin.py +78 -0
- phlo_transform-0.2.0/tests/test_translate.py +101 -0
- phlo_transform-0.2.0/tests/test_workspace.py +120 -0
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: phlo-transform
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: SQL transformation capability provider for Phlo, backed by the Phlo Transform engine
|
|
5
|
+
Author-email: Phlo Team <team@phlo.dev>
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Requires-Dist: phlo<0.20,>=0.19.0
|
|
10
|
+
Provides-Extra: dev
|
|
11
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
12
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
13
|
+
|
|
14
|
+
# phlo-transform
|
|
15
|
+
|
|
16
|
+
`phlo-transform` is Phlo's SQL transformation provider. It exposes two kinds of
|
|
17
|
+
assets to installed orchestrator adapters:
|
|
18
|
+
|
|
19
|
+
- declarations written with `phlo.transform.sql()`;
|
|
20
|
+
- every compiled model in a native [Phlo Transform](https://github.com/phlohouse/phlo-transform)
|
|
21
|
+
workspace (`phlo.toml`).
|
|
22
|
+
|
|
23
|
+
The Rust engine remains the source of truth for compilation, planning,
|
|
24
|
+
execution, lineage, and tests. This package adapts its JSON reports to Phlo
|
|
25
|
+
capability specs; it does not reimplement SQL analysis.
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install phlo-transform
|
|
31
|
+
|
|
32
|
+
# The engine binary, downloaded into .phlo/bin with a verified checksum
|
|
33
|
+
phlo transform install
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
`phlo transform install` follows the `PHLO_TRANSFORM_RELEASE_VERSION` setting
|
|
37
|
+
(default `0.1.0`) and resolves the portable release for Linux or macOS on
|
|
38
|
+
x86_64/arm64. Alternatively, install the engine from source with
|
|
39
|
+
`cargo install --path crates/phlo-transform-cli` and leave it on `PATH`.
|
|
40
|
+
|
|
41
|
+
## Declare transforms in Python
|
|
42
|
+
|
|
43
|
+
```python
|
|
44
|
+
import phlo
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@phlo.transform.sql(
|
|
48
|
+
table="silver.orders",
|
|
49
|
+
depends_on=["bronze.orders"],
|
|
50
|
+
materialized="table",
|
|
51
|
+
)
|
|
52
|
+
def orders() -> str:
|
|
53
|
+
return "select * from bronze.orders"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`phlo.transform.sql()` registers an asset at import time. The package supplies
|
|
57
|
+
both plugin halves - the transformation provider and the asset provider - so
|
|
58
|
+
the declaration is discovered without any extra wiring.
|
|
59
|
+
|
|
60
|
+
## Use a native workspace
|
|
61
|
+
|
|
62
|
+
A workspace is a directory with `phlo.toml`:
|
|
63
|
+
|
|
64
|
+
```text
|
|
65
|
+
workflows/transforms/native/
|
|
66
|
+
phlo.toml
|
|
67
|
+
transforms/stg_orders.sql
|
|
68
|
+
transforms/orders.sql
|
|
69
|
+
tests/generated/...
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
```toml
|
|
73
|
+
[transform]
|
|
74
|
+
default_namespace = "analytics"
|
|
75
|
+
default_materialization = "table"
|
|
76
|
+
|
|
77
|
+
# Optional: keep ingestion -> transform dependency edges in the orchestrator.
|
|
78
|
+
[phlo_transform.sources]
|
|
79
|
+
"raw.orders" = "sling_orders"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Discovery checks `PHLO_TRANSFORM_ROOT`, then `transforms/`,
|
|
83
|
+
`workflows/transforms/`, and `phlo.toml` files under those trees. More than one
|
|
84
|
+
workspace requires an explicit root.
|
|
85
|
+
|
|
86
|
+
Each compiled model becomes one asset. The default asset key is the model's
|
|
87
|
+
last dotted name segment, which matches a dbt model name and keeps asset keys
|
|
88
|
+
stable when a project switches engines. Set
|
|
89
|
+
`PHLO_TRANSFORM_ASSET_KEY=logical` to keep the namespace prefix instead.
|
|
90
|
+
|
|
91
|
+
Compatible workspace assets share one engine run through the Phlo grouped
|
|
92
|
+
execution contract. A selection of two models runs:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
phlo-transform --root <workspace> --json run --select <first-model> --select <second-model>
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Each requested model retains its asset key, dependencies, checks, and
|
|
99
|
+
`MaterializeResult` from its own report entry. Metadata includes status, action,
|
|
100
|
+
reasons, run and plan IDs, versions, SQL hash, query ID, counts, and timings.
|
|
101
|
+
A failed model or owned test fails that asset, not
|
|
102
|
+
healthy siblings. Missing model or test evidence also fails the asset. An
|
|
103
|
+
invocation failure fails the Dagster step after it emits successful outcomes.
|
|
104
|
+
Unchanged and cached models succeed when their owned tests pass.
|
|
105
|
+
|
|
106
|
+
The engine can include upstream dependencies in its plan, even for a subset
|
|
107
|
+
selection. Phlo emits materialisations and checks only for requested assets.
|
|
108
|
+
Per-model `explain` commands still supply column-lineage metadata; grouping
|
|
109
|
+
reduces run commands, not every engine subprocess.
|
|
110
|
+
|
|
111
|
+
Assets with different partitions, resources, or capability overrides form
|
|
112
|
+
separate groups. A model with blocking checks finishes in an earlier group than
|
|
113
|
+
its descendants, so failed checks still prevent downstream execution. A chain
|
|
114
|
+
with checks on every model therefore retains separate invocations. Explicit
|
|
115
|
+
execution policies keep individual runners, as does a graph that would become
|
|
116
|
+
cyclic after grouping.
|
|
117
|
+
|
|
118
|
+
Native `phlo/full_refresh_asset` requests fail explicitly. The engine's `--force`
|
|
119
|
+
rebuilds a plan but does not replace existing incremental data. Use an
|
|
120
|
+
engine-supported replacement plan for that operation, not a forced append.
|
|
121
|
+
|
|
122
|
+
A group is one Dagster step, so its retries repeat the selected models.
|
|
123
|
+
Existing asset selections and schedules need no changes. Set
|
|
124
|
+
`PHLO_TRANSFORM_GROUP_EXECUTION=false` to keep legacy per-model Dagster op
|
|
125
|
+
configuration. Attached checks execute with their owning assets; check-only
|
|
126
|
+
selections are rejected before provider execution.
|
|
127
|
+
|
|
128
|
+
In orchestrated runs the adapter comes from the resolved Phlo query engine:
|
|
129
|
+
the Trino endpoint and catalog come from the query-engine capability, and a
|
|
130
|
+
non-main runtime ref is provisioned to its own catalog through
|
|
131
|
+
`RefQueryCatalogManager` before the engine runs. Without orchestration the
|
|
132
|
+
engine decides: Trino when an endpoint is configured, otherwise DuckDB.
|
|
133
|
+
|
|
134
|
+
## Observatory candidate execution
|
|
135
|
+
|
|
136
|
+
The editor saves immutable SQL revisions and runs them on isolated candidate
|
|
137
|
+
branches. Compilation uses the environment's configured preview reference,
|
|
138
|
+
catalog, endpoint, and identity. Custom primary references are supported when
|
|
139
|
+
that complete binding matches the native provider configuration.
|
|
140
|
+
|
|
141
|
+
A rejected read-only compilation does not imply an uncertain write. After
|
|
142
|
+
correcting its prerequisites, you can confirm a new execution with a fresh
|
|
143
|
+
idempotency key. Once candidate provisioning or execution starts, an uncertain
|
|
144
|
+
outcome remains blocked, including requests with a different key.
|
|
145
|
+
|
|
146
|
+
Successful executions retain their native report. When the candidate result
|
|
147
|
+
hash is recorded, the authorised execution preview endpoint can retry result
|
|
148
|
+
readback without rerunning the model.
|
|
149
|
+
The authorised execution `POST .../executions/{execution_id}/reconcile`
|
|
150
|
+
endpoint rereads the recorded report and pinned provider snapshot. It can
|
|
151
|
+
resolve an uncertain operation response only when a durable native success
|
|
152
|
+
report, output branch hash, source revision, and provider evidence agree.
|
|
153
|
+
It never invokes the engine. A `running` execution, a missing report, or a
|
|
154
|
+
missing output hash remains unresolved. Preserve those records, branches,
|
|
155
|
+
and catalogs for operator investigation. Do not clear idempotency state or
|
|
156
|
+
assume that a timeout means no write occurred.
|
|
157
|
+
|
|
158
|
+
Execution history exposes stale evidence separately from current release
|
|
159
|
+
eligibility. A changed base does not hide a pinned candidate result, but a
|
|
160
|
+
changed candidate hash or revoked source access hides result rows.
|
|
161
|
+
|
|
162
|
+
Draft revisions and terminal executions support authorised `POST .../archive`
|
|
163
|
+
with `{"archived": true, "reason": "Superseded review"}`. Setting `archived` to
|
|
164
|
+
`false` restores visibility. History accepts `include_archived=true`.
|
|
165
|
+
Archiving retains all source bytes, reports, candidate bindings, and operation
|
|
166
|
+
fences. It does not reclaim disk space or reset the global history limits.
|
|
167
|
+
Archived revisions cannot compile, execute, or extend the draft until restored.
|
|
168
|
+
|
|
169
|
+
Archived revisions and terminal executions also support `POST .../reclaim` with
|
|
170
|
+
`confirmed: true`, a reason, and `expected_revision_digest`. This action
|
|
171
|
+
irreversibly deletes the local workspace copy. It retains SQL for source
|
|
172
|
+
handoff, per-file checksums, revision identities, reports, candidate bindings,
|
|
173
|
+
and operation claims. Completed reclamation frees a retained-workspace slot;
|
|
174
|
+
archive alone does not. History still includes the durable tombstone.
|
|
175
|
+
|
|
176
|
+
Reclamation refuses prepared, running, or unknown executions, unknown reference
|
|
177
|
+
metadata, and pending or unknown operation claims. The API holds the operation
|
|
178
|
+
claim transaction lock during local deletion. Repeating confirmed reclamation
|
|
179
|
+
can finish interrupted local cleanup, but never reruns a provider mutation.
|
|
180
|
+
Reclaimed revisions and executions cannot be restored or executed again.
|
|
181
|
+
No remote branches, tables, catalogs, or live data are deleted.
|
|
182
|
+
|
|
183
|
+
The API now applies configured `audit.retention_years` to supporting payloads
|
|
184
|
+
as well as the existing audit-preservation checks. Native workspace age uses
|
|
185
|
+
the later of creation and archival, since native output files can be newer
|
|
186
|
+
than preparation. The minimum expires at the calendar-year anniversary;
|
|
187
|
+
29 February expires on 1 March in a non-leap year. Unset retention adds no
|
|
188
|
+
age minimum. Invalid timestamps or unavailable evidence block reclamation.
|
|
189
|
+
This check does not expire audit records or tombstones.
|
|
190
|
+
|
|
191
|
+
`GET .../drafts/{draft_id}/{revision}/handoff` exports the saved SQL with its
|
|
192
|
+
project-relative path and checksum. The optional `execution_id` includes
|
|
193
|
+
candidate evidence and links to existing branch checks, trial merge, and
|
|
194
|
+
signed merge operations. These links describe POST operations, not automatic
|
|
195
|
+
actions. Source review and a source commit remain separate from the signed
|
|
196
|
+
data release. The handoff never publishes SQL or promotes candidate data.
|
|
197
|
+
|
|
198
|
+
## CLI
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
phlo transform status # active provider, workspace, counts
|
|
202
|
+
phlo transform check # compile and diagnose
|
|
203
|
+
phlo transform plan --select analytics.orders
|
|
204
|
+
phlo transform run --select analytics.orders --force
|
|
205
|
+
phlo transform test --select analytics.orders
|
|
206
|
+
phlo transform list
|
|
207
|
+
phlo transform explain analytics.orders
|
|
208
|
+
phlo transform lineage --format openlineage
|
|
209
|
+
phlo transform doctor
|
|
210
|
+
phlo transform translate --from dbt --out generated/
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Every command forwards `--json` to the engine.
|
|
214
|
+
|
|
215
|
+
## Switching providers in place
|
|
216
|
+
|
|
217
|
+
A project can keep a dbt project and a native workspace side by side:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
phlo transform use phlo-transform # writes .phlo/transformation-provider
|
|
221
|
+
phlo services restart dagster # reload the asset graph
|
|
222
|
+
phlo transform use dbt # switch back
|
|
223
|
+
phlo transform use --unset # follow install defaults
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
Selection order is `PHLO_TRANSFORMATION_PROVIDER` (process or project env),
|
|
227
|
+
then `.phlo/transformation-provider`, then install defaults. Unset means dbt
|
|
228
|
+
owns the graph when a dbt project exists; otherwise the native workspace does.
|
|
229
|
+
Only one provider contributes transform assets at a time, so asset keys cannot
|
|
230
|
+
collide.
|
|
231
|
+
|
|
232
|
+
## Configuration
|
|
233
|
+
|
|
234
|
+
| Variable | Purpose |
|
|
235
|
+
| --- | --- |
|
|
236
|
+
| `PHLO_TRANSFORMATION_PROVIDER` | Active provider: `dbt` or `phlo-transform`. |
|
|
237
|
+
| `PHLO_TRANSFORM_BIN` | Engine executable path. |
|
|
238
|
+
| `PHLO_TRANSFORM_ROOT` | Workspace containing `phlo.toml`. |
|
|
239
|
+
| `PHLO_TRANSFORM_ADAPTER` | `trino` or `duckdb`; inferred otherwise. |
|
|
240
|
+
| `PHLO_TRANSFORM_DUCKDB_PATH` | DuckDB database file for local runs. |
|
|
241
|
+
| `PHLO_TRANSFORM_TRINO_ENDPOINT` | Trino endpoint override. |
|
|
242
|
+
| `PHLO_TRANSFORM_TRINO_USER` / `_PASSWORD` / `_CATALOG` / `_SCHEMA` | Trino connection overrides. |
|
|
243
|
+
| `PHLO_TRANSFORM_JOBS` | Maximum concurrent model builds. |
|
|
244
|
+
| `PHLO_TRANSFORM_GROUP_EXECUTION` | Group compatible workspace assets, `true` by default; `false` retains legacy per-model Dagster ops. |
|
|
245
|
+
| `PHLO_TRANSFORM_ASSET_KEY` | `leaf` (default) or `logical`. |
|
|
246
|
+
| `PHLO_TRANSFORM_RELEASE_VERSION` | Release installed by `phlo transform install`. |
|
|
247
|
+
| `PHLO_TRANSFORM_TIMEOUT_SECONDS` | Per-command timeout. |
|
|
248
|
+
|
|
249
|
+
Values may live in `phlo.yaml` `env:`, `.phlo/.env`, or the process
|
|
250
|
+
environment; earlier sources are overridden by later ones.
|
|
251
|
+
|
|
252
|
+
## Support status
|
|
253
|
+
|
|
254
|
+
This package is `outside_v1` / `development_only` in the support manifest: it is
|
|
255
|
+
production-shaped and tested, but not part of the v1 support contract.
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# phlo-transform
|
|
2
|
+
|
|
3
|
+
`phlo-transform` is Phlo's SQL transformation provider. It exposes two kinds of
|
|
4
|
+
assets to installed orchestrator adapters:
|
|
5
|
+
|
|
6
|
+
- declarations written with `phlo.transform.sql()`;
|
|
7
|
+
- every compiled model in a native [Phlo Transform](https://github.com/phlohouse/phlo-transform)
|
|
8
|
+
workspace (`phlo.toml`).
|
|
9
|
+
|
|
10
|
+
The Rust engine remains the source of truth for compilation, planning,
|
|
11
|
+
execution, lineage, and tests. This package adapts its JSON reports to Phlo
|
|
12
|
+
capability specs; it does not reimplement SQL analysis.
|
|
13
|
+
|
|
14
|
+
## Install
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install phlo-transform
|
|
18
|
+
|
|
19
|
+
# The engine binary, downloaded into .phlo/bin with a verified checksum
|
|
20
|
+
phlo transform install
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
`phlo transform install` follows the `PHLO_TRANSFORM_RELEASE_VERSION` setting
|
|
24
|
+
(default `0.1.0`) and resolves the portable release for Linux or macOS on
|
|
25
|
+
x86_64/arm64. Alternatively, install the engine from source with
|
|
26
|
+
`cargo install --path crates/phlo-transform-cli` and leave it on `PATH`.
|
|
27
|
+
|
|
28
|
+
## Declare transforms in Python
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
import phlo
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@phlo.transform.sql(
|
|
35
|
+
table="silver.orders",
|
|
36
|
+
depends_on=["bronze.orders"],
|
|
37
|
+
materialized="table",
|
|
38
|
+
)
|
|
39
|
+
def orders() -> str:
|
|
40
|
+
return "select * from bronze.orders"
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
`phlo.transform.sql()` registers an asset at import time. The package supplies
|
|
44
|
+
both plugin halves - the transformation provider and the asset provider - so
|
|
45
|
+
the declaration is discovered without any extra wiring.
|
|
46
|
+
|
|
47
|
+
## Use a native workspace
|
|
48
|
+
|
|
49
|
+
A workspace is a directory with `phlo.toml`:
|
|
50
|
+
|
|
51
|
+
```text
|
|
52
|
+
workflows/transforms/native/
|
|
53
|
+
phlo.toml
|
|
54
|
+
transforms/stg_orders.sql
|
|
55
|
+
transforms/orders.sql
|
|
56
|
+
tests/generated/...
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
```toml
|
|
60
|
+
[transform]
|
|
61
|
+
default_namespace = "analytics"
|
|
62
|
+
default_materialization = "table"
|
|
63
|
+
|
|
64
|
+
# Optional: keep ingestion -> transform dependency edges in the orchestrator.
|
|
65
|
+
[phlo_transform.sources]
|
|
66
|
+
"raw.orders" = "sling_orders"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Discovery checks `PHLO_TRANSFORM_ROOT`, then `transforms/`,
|
|
70
|
+
`workflows/transforms/`, and `phlo.toml` files under those trees. More than one
|
|
71
|
+
workspace requires an explicit root.
|
|
72
|
+
|
|
73
|
+
Each compiled model becomes one asset. The default asset key is the model's
|
|
74
|
+
last dotted name segment, which matches a dbt model name and keeps asset keys
|
|
75
|
+
stable when a project switches engines. Set
|
|
76
|
+
`PHLO_TRANSFORM_ASSET_KEY=logical` to keep the namespace prefix instead.
|
|
77
|
+
|
|
78
|
+
Compatible workspace assets share one engine run through the Phlo grouped
|
|
79
|
+
execution contract. A selection of two models runs:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
phlo-transform --root <workspace> --json run --select <first-model> --select <second-model>
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Each requested model retains its asset key, dependencies, checks, and
|
|
86
|
+
`MaterializeResult` from its own report entry. Metadata includes status, action,
|
|
87
|
+
reasons, run and plan IDs, versions, SQL hash, query ID, counts, and timings.
|
|
88
|
+
A failed model or owned test fails that asset, not
|
|
89
|
+
healthy siblings. Missing model or test evidence also fails the asset. An
|
|
90
|
+
invocation failure fails the Dagster step after it emits successful outcomes.
|
|
91
|
+
Unchanged and cached models succeed when their owned tests pass.
|
|
92
|
+
|
|
93
|
+
The engine can include upstream dependencies in its plan, even for a subset
|
|
94
|
+
selection. Phlo emits materialisations and checks only for requested assets.
|
|
95
|
+
Per-model `explain` commands still supply column-lineage metadata; grouping
|
|
96
|
+
reduces run commands, not every engine subprocess.
|
|
97
|
+
|
|
98
|
+
Assets with different partitions, resources, or capability overrides form
|
|
99
|
+
separate groups. A model with blocking checks finishes in an earlier group than
|
|
100
|
+
its descendants, so failed checks still prevent downstream execution. A chain
|
|
101
|
+
with checks on every model therefore retains separate invocations. Explicit
|
|
102
|
+
execution policies keep individual runners, as does a graph that would become
|
|
103
|
+
cyclic after grouping.
|
|
104
|
+
|
|
105
|
+
Native `phlo/full_refresh_asset` requests fail explicitly. The engine's `--force`
|
|
106
|
+
rebuilds a plan but does not replace existing incremental data. Use an
|
|
107
|
+
engine-supported replacement plan for that operation, not a forced append.
|
|
108
|
+
|
|
109
|
+
A group is one Dagster step, so its retries repeat the selected models.
|
|
110
|
+
Existing asset selections and schedules need no changes. Set
|
|
111
|
+
`PHLO_TRANSFORM_GROUP_EXECUTION=false` to keep legacy per-model Dagster op
|
|
112
|
+
configuration. Attached checks execute with their owning assets; check-only
|
|
113
|
+
selections are rejected before provider execution.
|
|
114
|
+
|
|
115
|
+
In orchestrated runs the adapter comes from the resolved Phlo query engine:
|
|
116
|
+
the Trino endpoint and catalog come from the query-engine capability, and a
|
|
117
|
+
non-main runtime ref is provisioned to its own catalog through
|
|
118
|
+
`RefQueryCatalogManager` before the engine runs. Without orchestration the
|
|
119
|
+
engine decides: Trino when an endpoint is configured, otherwise DuckDB.
|
|
120
|
+
|
|
121
|
+
## Observatory candidate execution
|
|
122
|
+
|
|
123
|
+
The editor saves immutable SQL revisions and runs them on isolated candidate
|
|
124
|
+
branches. Compilation uses the environment's configured preview reference,
|
|
125
|
+
catalog, endpoint, and identity. Custom primary references are supported when
|
|
126
|
+
that complete binding matches the native provider configuration.
|
|
127
|
+
|
|
128
|
+
A rejected read-only compilation does not imply an uncertain write. After
|
|
129
|
+
correcting its prerequisites, you can confirm a new execution with a fresh
|
|
130
|
+
idempotency key. Once candidate provisioning or execution starts, an uncertain
|
|
131
|
+
outcome remains blocked, including requests with a different key.
|
|
132
|
+
|
|
133
|
+
Successful executions retain their native report. When the candidate result
|
|
134
|
+
hash is recorded, the authorised execution preview endpoint can retry result
|
|
135
|
+
readback without rerunning the model.
|
|
136
|
+
The authorised execution `POST .../executions/{execution_id}/reconcile`
|
|
137
|
+
endpoint rereads the recorded report and pinned provider snapshot. It can
|
|
138
|
+
resolve an uncertain operation response only when a durable native success
|
|
139
|
+
report, output branch hash, source revision, and provider evidence agree.
|
|
140
|
+
It never invokes the engine. A `running` execution, a missing report, or a
|
|
141
|
+
missing output hash remains unresolved. Preserve those records, branches,
|
|
142
|
+
and catalogs for operator investigation. Do not clear idempotency state or
|
|
143
|
+
assume that a timeout means no write occurred.
|
|
144
|
+
|
|
145
|
+
Execution history exposes stale evidence separately from current release
|
|
146
|
+
eligibility. A changed base does not hide a pinned candidate result, but a
|
|
147
|
+
changed candidate hash or revoked source access hides result rows.
|
|
148
|
+
|
|
149
|
+
Draft revisions and terminal executions support authorised `POST .../archive`
|
|
150
|
+
with `{"archived": true, "reason": "Superseded review"}`. Setting `archived` to
|
|
151
|
+
`false` restores visibility. History accepts `include_archived=true`.
|
|
152
|
+
Archiving retains all source bytes, reports, candidate bindings, and operation
|
|
153
|
+
fences. It does not reclaim disk space or reset the global history limits.
|
|
154
|
+
Archived revisions cannot compile, execute, or extend the draft until restored.
|
|
155
|
+
|
|
156
|
+
Archived revisions and terminal executions also support `POST .../reclaim` with
|
|
157
|
+
`confirmed: true`, a reason, and `expected_revision_digest`. This action
|
|
158
|
+
irreversibly deletes the local workspace copy. It retains SQL for source
|
|
159
|
+
handoff, per-file checksums, revision identities, reports, candidate bindings,
|
|
160
|
+
and operation claims. Completed reclamation frees a retained-workspace slot;
|
|
161
|
+
archive alone does not. History still includes the durable tombstone.
|
|
162
|
+
|
|
163
|
+
Reclamation refuses prepared, running, or unknown executions, unknown reference
|
|
164
|
+
metadata, and pending or unknown operation claims. The API holds the operation
|
|
165
|
+
claim transaction lock during local deletion. Repeating confirmed reclamation
|
|
166
|
+
can finish interrupted local cleanup, but never reruns a provider mutation.
|
|
167
|
+
Reclaimed revisions and executions cannot be restored or executed again.
|
|
168
|
+
No remote branches, tables, catalogs, or live data are deleted.
|
|
169
|
+
|
|
170
|
+
The API now applies configured `audit.retention_years` to supporting payloads
|
|
171
|
+
as well as the existing audit-preservation checks. Native workspace age uses
|
|
172
|
+
the later of creation and archival, since native output files can be newer
|
|
173
|
+
than preparation. The minimum expires at the calendar-year anniversary;
|
|
174
|
+
29 February expires on 1 March in a non-leap year. Unset retention adds no
|
|
175
|
+
age minimum. Invalid timestamps or unavailable evidence block reclamation.
|
|
176
|
+
This check does not expire audit records or tombstones.
|
|
177
|
+
|
|
178
|
+
`GET .../drafts/{draft_id}/{revision}/handoff` exports the saved SQL with its
|
|
179
|
+
project-relative path and checksum. The optional `execution_id` includes
|
|
180
|
+
candidate evidence and links to existing branch checks, trial merge, and
|
|
181
|
+
signed merge operations. These links describe POST operations, not automatic
|
|
182
|
+
actions. Source review and a source commit remain separate from the signed
|
|
183
|
+
data release. The handoff never publishes SQL or promotes candidate data.
|
|
184
|
+
|
|
185
|
+
## CLI
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
phlo transform status # active provider, workspace, counts
|
|
189
|
+
phlo transform check # compile and diagnose
|
|
190
|
+
phlo transform plan --select analytics.orders
|
|
191
|
+
phlo transform run --select analytics.orders --force
|
|
192
|
+
phlo transform test --select analytics.orders
|
|
193
|
+
phlo transform list
|
|
194
|
+
phlo transform explain analytics.orders
|
|
195
|
+
phlo transform lineage --format openlineage
|
|
196
|
+
phlo transform doctor
|
|
197
|
+
phlo transform translate --from dbt --out generated/
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
Every command forwards `--json` to the engine.
|
|
201
|
+
|
|
202
|
+
## Switching providers in place
|
|
203
|
+
|
|
204
|
+
A project can keep a dbt project and a native workspace side by side:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
phlo transform use phlo-transform # writes .phlo/transformation-provider
|
|
208
|
+
phlo services restart dagster # reload the asset graph
|
|
209
|
+
phlo transform use dbt # switch back
|
|
210
|
+
phlo transform use --unset # follow install defaults
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Selection order is `PHLO_TRANSFORMATION_PROVIDER` (process or project env),
|
|
214
|
+
then `.phlo/transformation-provider`, then install defaults. Unset means dbt
|
|
215
|
+
owns the graph when a dbt project exists; otherwise the native workspace does.
|
|
216
|
+
Only one provider contributes transform assets at a time, so asset keys cannot
|
|
217
|
+
collide.
|
|
218
|
+
|
|
219
|
+
## Configuration
|
|
220
|
+
|
|
221
|
+
| Variable | Purpose |
|
|
222
|
+
| --- | --- |
|
|
223
|
+
| `PHLO_TRANSFORMATION_PROVIDER` | Active provider: `dbt` or `phlo-transform`. |
|
|
224
|
+
| `PHLO_TRANSFORM_BIN` | Engine executable path. |
|
|
225
|
+
| `PHLO_TRANSFORM_ROOT` | Workspace containing `phlo.toml`. |
|
|
226
|
+
| `PHLO_TRANSFORM_ADAPTER` | `trino` or `duckdb`; inferred otherwise. |
|
|
227
|
+
| `PHLO_TRANSFORM_DUCKDB_PATH` | DuckDB database file for local runs. |
|
|
228
|
+
| `PHLO_TRANSFORM_TRINO_ENDPOINT` | Trino endpoint override. |
|
|
229
|
+
| `PHLO_TRANSFORM_TRINO_USER` / `_PASSWORD` / `_CATALOG` / `_SCHEMA` | Trino connection overrides. |
|
|
230
|
+
| `PHLO_TRANSFORM_JOBS` | Maximum concurrent model builds. |
|
|
231
|
+
| `PHLO_TRANSFORM_GROUP_EXECUTION` | Group compatible workspace assets, `true` by default; `false` retains legacy per-model Dagster ops. |
|
|
232
|
+
| `PHLO_TRANSFORM_ASSET_KEY` | `leaf` (default) or `logical`. |
|
|
233
|
+
| `PHLO_TRANSFORM_RELEASE_VERSION` | Release installed by `phlo transform install`. |
|
|
234
|
+
| `PHLO_TRANSFORM_TIMEOUT_SECONDS` | Per-command timeout. |
|
|
235
|
+
|
|
236
|
+
Values may live in `phlo.yaml` `env:`, `.phlo/.env`, or the process
|
|
237
|
+
environment; earlier sources are overridden by later ones.
|
|
238
|
+
|
|
239
|
+
## Support status
|
|
240
|
+
|
|
241
|
+
This package is `outside_v1` / `development_only` in the support manifest: it is
|
|
242
|
+
production-shaped and tested, but not part of the v1 support contract.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
build-backend = "setuptools.build_meta"
|
|
3
|
+
requires = [
|
|
4
|
+
"setuptools>=45",
|
|
5
|
+
"wheel",
|
|
6
|
+
]
|
|
7
|
+
|
|
8
|
+
[project]
|
|
9
|
+
dependencies = ["phlo>=0.19.0,<0.20"]
|
|
10
|
+
description = "SQL transformation capability provider for Phlo, backed by the Phlo Transform engine"
|
|
11
|
+
name = "phlo-transform"
|
|
12
|
+
requires-python = ">=3.12"
|
|
13
|
+
version = "0.2.0"
|
|
14
|
+
|
|
15
|
+
[[project.authors]]
|
|
16
|
+
email = "team@phlo.dev"
|
|
17
|
+
name = "Phlo Team"
|
|
18
|
+
|
|
19
|
+
[project.entry-points."phlo.plugins.assets"]
|
|
20
|
+
transform = "phlo_transform.plugin:TransformAssetProvider"
|
|
21
|
+
|
|
22
|
+
[project.entry-points."phlo.plugins.cli"]
|
|
23
|
+
transform = "phlo_transform.cli_plugin:TransformCliPlugin"
|
|
24
|
+
|
|
25
|
+
[project.entry-points."phlo.plugins.transformation_providers"]
|
|
26
|
+
transform = "phlo_transform.plugin:TransformProvider"
|
|
27
|
+
|
|
28
|
+
[project.license]
|
|
29
|
+
text = "MIT"
|
|
30
|
+
|
|
31
|
+
[project.optional-dependencies]
|
|
32
|
+
dev = [
|
|
33
|
+
"pytest>=7.0",
|
|
34
|
+
"ruff>=0.1.0",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[project.readme]
|
|
38
|
+
content-type = "text/markdown"
|
|
39
|
+
file = "README.md"
|
|
40
|
+
|
|
41
|
+
[tool.ruff]
|
|
42
|
+
line-length = 100
|
|
43
|
+
target-version = "py312"
|
|
44
|
+
|
|
45
|
+
[tool.setuptools.package-dir]
|
|
46
|
+
"" = "src"
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.packages.find]
|
|
49
|
+
where = ["src"]
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""Phlo Transform integration for Phlo.
|
|
2
|
+
|
|
3
|
+
Provides the ``transform`` asset and transformation providers, the
|
|
4
|
+
``phlo transform`` CLI, and typed access to a Phlo Transform workspace. The
|
|
5
|
+
Rust engine is the source of truth for compilation, planning, execution, and
|
|
6
|
+
tests; this package adapts its JSON reports to Phlo capability specs.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from importlib.metadata import version
|
|
12
|
+
|
|
13
|
+
from phlo_transform.assets import build_transform_asset_specs
|
|
14
|
+
from phlo_transform.workspace import TransformWorkspace, load_workspace
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"TransformWorkspace",
|
|
18
|
+
"__version__",
|
|
19
|
+
"build_transform_asset_specs",
|
|
20
|
+
"load_workspace",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
__version__ = version("phlo-transform")
|