data-gantry 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_gantry-0.5.0.dist-info/METADATA +671 -0
- data_gantry-0.5.0.dist-info/RECORD +63 -0
- data_gantry-0.5.0.dist-info/WHEEL +4 -0
- data_gantry-0.5.0.dist-info/licenses/LICENSE +201 -0
- data_gantry-0.5.0.dist-info/licenses/NOTICE +7 -0
- gantry/__init__.py +77 -0
- gantry/adapter.py +41 -0
- gantry/admission.py +92 -0
- gantry/artifact.py +23 -0
- gantry/batch/__init__.py +7 -0
- gantry/batch/api.py +66 -0
- gantry/batch/capabilities.py +14 -0
- gantry/capabilities.py +24 -0
- gantry/context.py +15 -0
- gantry/execution.py +100 -0
- gantry/failure.py +45 -0
- gantry/flink/__init__.py +2 -0
- gantry/flink/adapter.py +926 -0
- gantry/flink/api.py +388 -0
- gantry/flink/artifact.py +58 -0
- gantry/flink/client.py +318 -0
- gantry/flink/execution.py +51 -0
- gantry/flink/metrics.py +48 -0
- gantry/flink/operation.py +470 -0
- gantry/flink/target.py +111 -0
- gantry/flink/verification.py +154 -0
- gantry/handle.py +31 -0
- gantry/metrics.py +19 -0
- gantry/output.py +29 -0
- gantry/policy.py +30 -0
- gantry/result.py +110 -0
- gantry/runtime.py +495 -0
- gantry/sql/__init__.py +69 -0
- gantry/sql/adapter.py +37 -0
- gantry/sql/adapters/__init__.py +4 -0
- gantry/sql/adapters/bigquery.py +451 -0
- gantry/sql/adapters/duckdb.py +333 -0
- gantry/sql/adapters/postgres.py +354 -0
- gantry/sql/adapters/snowflake.py +335 -0
- gantry/sql/api.py +362 -0
- gantry/sql/bridge.py +109 -0
- gantry/sql/capabilities.py +56 -0
- gantry/sql/classification.py +42 -0
- gantry/sql/dialect.py +146 -0
- gantry/sql/enforcement.py +62 -0
- gantry/sql/explain.py +18 -0
- gantry/sql/materialization.py +878 -0
- gantry/sql/output.py +16 -0
- gantry/sql/policy.py +55 -0
- gantry/sql/providers.py +116 -0
- gantry/sql/query.py +72 -0
- gantry/sql/registry.py +104 -0
- gantry/sql/result.py +36 -0
- gantry/sql/schema.py +32 -0
- gantry/sql/target.py +23 -0
- gantry/store.py +43 -0
- gantry/stream/__init__.py +7 -0
- gantry/stream/api.py +65 -0
- gantry/stream/capabilities.py +14 -0
- gantry/target.py +19 -0
- gantry/tool.py +30 -0
- gantry/verifier.py +66 -0
- gantry/verify.py +181 -0
|
@@ -0,0 +1,671 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: data-gantry
|
|
3
|
+
Version: 0.5.0
|
|
4
|
+
Summary: A common execution control plane for agent-generated data work
|
|
5
|
+
Project-URL: Homepage, https://github.com/arvindram03/gantry
|
|
6
|
+
Project-URL: Source, https://github.com/arvindram03/gantry
|
|
7
|
+
Project-URL: Issues, https://github.com/arvindram03/gantry/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/arvindram03/gantry/blob/main/CHANGELOG.md
|
|
9
|
+
License: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
License-File: NOTICE
|
|
12
|
+
Keywords: agents,duckdb,flink,governance,llm,postgres,sql
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Database
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.12
|
|
22
|
+
Provides-Extra: all-sql
|
|
23
|
+
Requires-Dist: asyncpg>=0.30; extra == 'all-sql'
|
|
24
|
+
Requires-Dist: duckdb>=1.1; extra == 'all-sql'
|
|
25
|
+
Requires-Dist: google-cloud-bigquery>=3.25; extra == 'all-sql'
|
|
26
|
+
Requires-Dist: snowflake-connector-python>=3.12; extra == 'all-sql'
|
|
27
|
+
Provides-Extra: bigquery
|
|
28
|
+
Requires-Dist: google-cloud-bigquery>=3.25; extra == 'bigquery'
|
|
29
|
+
Provides-Extra: duckdb
|
|
30
|
+
Requires-Dist: duckdb>=1.1; extra == 'duckdb'
|
|
31
|
+
Provides-Extra: postgres
|
|
32
|
+
Requires-Dist: asyncpg>=0.30; extra == 'postgres'
|
|
33
|
+
Provides-Extra: snowflake
|
|
34
|
+
Requires-Dist: snowflake-connector-python>=3.12; extra == 'snowflake'
|
|
35
|
+
Description-Content-Type: text/markdown
|
|
36
|
+
|
|
37
|
+
# Gantry
|
|
38
|
+
|
|
39
|
+
**A trust boundary for agents working with real data.**
|
|
40
|
+
|
|
41
|
+
Agents can generate SQL, transformations, and data jobs.
|
|
42
|
+
|
|
43
|
+
But generating work and having the authority to execute it should be two different things.
|
|
44
|
+
|
|
45
|
+
> **Agent output is a proposal, not an instruction.**
|
|
46
|
+
|
|
47
|
+
Gantry sits between agents and data systems. It decides whether proposed work is allowed, executes it under trusted constraints, observes what happens, and gives the caller a structured result.
|
|
48
|
+
|
|
49
|
+
```text
|
|
50
|
+
trusted configuration
|
|
51
|
+
│
|
|
52
|
+
▼
|
|
53
|
+
Agent ──── proposal ────────> Gantry ─────────> Data system
|
|
54
|
+
│
|
|
55
|
+
admission
|
|
56
|
+
enforcement
|
|
57
|
+
execution
|
|
58
|
+
observation
|
|
59
|
+
verification
|
|
60
|
+
│
|
|
61
|
+
▼
|
|
62
|
+
Result
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Start with SQL
|
|
66
|
+
|
|
67
|
+
SQL is the simplest way to use Gantry today.
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install data-gantry
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
> **The distribution is `data-gantry`; the import is `gantry`.** The name
|
|
74
|
+
> `gantry` on PyPI belongs to an unrelated project, so the package installs
|
|
75
|
+
> under `data-gantry` and imports as `gantry`:
|
|
76
|
+
>
|
|
77
|
+
> ```python
|
|
78
|
+
> import gantry
|
|
79
|
+
> ```
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
import os
|
|
84
|
+
|
|
85
|
+
import gantry
|
|
86
|
+
|
|
87
|
+
db = gantry.sql.connect(
|
|
88
|
+
"postgres",
|
|
89
|
+
url=os.environ["DATABASE_URL"],
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
query = db.query(
|
|
93
|
+
read_only=True,
|
|
94
|
+
schemas=["analytics"],
|
|
95
|
+
max_rows=100,
|
|
96
|
+
timeout=30,
|
|
97
|
+
)
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Call the configured operation directly from application code:
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
result = await query(
|
|
104
|
+
"""
|
|
105
|
+
SELECT plan, COUNT(*) AS customers
|
|
106
|
+
FROM analytics.customers
|
|
107
|
+
GROUP BY plan
|
|
108
|
+
"""
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Engines that expose output references, such as BigQuery, can keep the result in the engine and
|
|
113
|
+
return its URI:
|
|
114
|
+
|
|
115
|
+
```python
|
|
116
|
+
warehouse = gantry.sql.connect("bigquery", project="acme")
|
|
117
|
+
warehouse_query = warehouse.query(max_rows=100)
|
|
118
|
+
|
|
119
|
+
uri = (await warehouse_query("SELECT * FROM analytics.customers")).uri
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
`uri` is `None` when the provider returns only bounded inline rows, as PostgreSQL typically does.
|
|
123
|
+
The full query outcome still exposes `inline`, `outputs`, status, metrics, and normalized failures.
|
|
124
|
+
|
|
125
|
+
Or expose its narrower tool form to an agent:
|
|
126
|
+
|
|
127
|
+
```python
|
|
128
|
+
tools = [query.tool()]
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The application keeps `db` and `query`. The agent sees only a `query_sql` tool with one input:
|
|
132
|
+
`sql`. It cannot change schemas, row limits, timeouts, or credentials.
|
|
133
|
+
|
|
134
|
+
The agent can decide:
|
|
135
|
+
|
|
136
|
+
```sql
|
|
137
|
+
SELECT plan, COUNT(*)
|
|
138
|
+
FROM analytics.customers
|
|
139
|
+
GROUP BY plan;
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
It cannot decide that it suddenly needs write access, another schema, different credentials, or a disabled safety check.
|
|
143
|
+
|
|
144
|
+
The authority stays outside the agent.
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
## Why Gantry?
|
|
149
|
+
|
|
150
|
+
Connecting an agent to a database is easy.
|
|
151
|
+
|
|
152
|
+
The harder problem starts after that.
|
|
153
|
+
|
|
154
|
+
An agent may decide to:
|
|
155
|
+
|
|
156
|
+
```sql
|
|
157
|
+
DELETE FROM customers;
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
scan several terabytes in a warehouse,
|
|
161
|
+
|
|
162
|
+
access data it was never supposed to see,
|
|
163
|
+
|
|
164
|
+
or submit a long-running transformation that fails after twenty minutes.
|
|
165
|
+
|
|
166
|
+
Prompting the model to "be careful" is not an execution boundary.
|
|
167
|
+
|
|
168
|
+
Gantry separates:
|
|
169
|
+
|
|
170
|
+
```text
|
|
171
|
+
Agent
|
|
172
|
+
owns
|
|
173
|
+
└── what work to propose
|
|
174
|
+
|
|
175
|
+
Gantry
|
|
176
|
+
owns
|
|
177
|
+
├── whether the work is admissible
|
|
178
|
+
├── under what constraints it executes
|
|
179
|
+
├── how execution is observed
|
|
180
|
+
└── whether the outcome is accepted
|
|
181
|
+
|
|
182
|
+
Data system
|
|
183
|
+
owns
|
|
184
|
+
└── actually executing the data workload
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
The agent proposes.
|
|
188
|
+
|
|
189
|
+
Gantry decides whether the proposal gets authority.
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## Policies are not agent instructions
|
|
194
|
+
|
|
195
|
+
Gantry policies are configured by the application or operator, not generated by the agent performing the work.
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
query = db.query(
|
|
199
|
+
read_only=True,
|
|
200
|
+
schemas=["analytics"],
|
|
201
|
+
max_rows=1000,
|
|
202
|
+
timeout=30,
|
|
203
|
+
)
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
The agent receives only `query.tool()`.
|
|
207
|
+
|
|
208
|
+
It does **not** receive:
|
|
209
|
+
|
|
210
|
+
```text
|
|
211
|
+
database credentials
|
|
212
|
+
unrestricted connections
|
|
213
|
+
policy mutation APIs
|
|
214
|
+
admin APIs
|
|
215
|
+
ways to disable verification
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
So this:
|
|
219
|
+
|
|
220
|
+
```sql
|
|
221
|
+
SELECT *
|
|
222
|
+
FROM analytics.customers
|
|
223
|
+
LIMIT 20;
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
can execute.
|
|
227
|
+
|
|
228
|
+
While this:
|
|
229
|
+
|
|
230
|
+
```sql
|
|
231
|
+
DELETE FROM analytics.customers;
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
is rejected before execution.
|
|
235
|
+
|
|
236
|
+
```text
|
|
237
|
+
REJECTED
|
|
238
|
+
|
|
239
|
+
DELETE is not allowed by read-only policy.
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
---
|
|
243
|
+
|
|
244
|
+
## Create derived data
|
|
245
|
+
|
|
246
|
+
Use a materializer when a caller needs to create a new table from approved sources. This is a
|
|
247
|
+
separate operation from querying; it does not turn the read tool into a general write connection.
|
|
248
|
+
|
|
249
|
+
```python
|
|
250
|
+
warehouse = gantry.sql.connect(
|
|
251
|
+
"bigquery",
|
|
252
|
+
project="acme",
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
materialize = warehouse.materialize(
|
|
256
|
+
sources=["raw.*"],
|
|
257
|
+
destinations=["agent_scratch.*"],
|
|
258
|
+
max_bytes_scanned=10_000_000_000,
|
|
259
|
+
timeout=300,
|
|
260
|
+
verify=[
|
|
261
|
+
gantry.verify.destination_exists(),
|
|
262
|
+
gantry.verify.row_count(min=1),
|
|
263
|
+
gantry.verify.required_columns(["customer_id", "outstanding_balance"]),
|
|
264
|
+
],
|
|
265
|
+
)
|
|
266
|
+
```
|
|
267
|
+
|
|
268
|
+
`db.materialize(...)` requires the application to configure write authority before any SQL is
|
|
269
|
+
supplied. Call the resulting operation directly:
|
|
270
|
+
|
|
271
|
+
```python
|
|
272
|
+
uri = (
|
|
273
|
+
await materialize(
|
|
274
|
+
"""
|
|
275
|
+
CREATE TABLE agent_scratch.high_risk_customers AS
|
|
276
|
+
SELECT customer_id, SUM(balance) AS outstanding_balance
|
|
277
|
+
FROM raw.invoices
|
|
278
|
+
WHERE status = 'unpaid'
|
|
279
|
+
GROUP BY customer_id
|
|
280
|
+
"""
|
|
281
|
+
)
|
|
282
|
+
).uri
|
|
283
|
+
|
|
284
|
+
print(uri) # bigquery://acme/agent_scratch/high_risk_customers
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
`uri` is `None` when no destination was created. Keep the full returned outcome when application
|
|
288
|
+
code needs structured rejection or verification details.
|
|
289
|
+
|
|
290
|
+
Or give both governed operations to an agent:
|
|
291
|
+
|
|
292
|
+
```python
|
|
293
|
+
tools = [
|
|
294
|
+
query.tool(),
|
|
295
|
+
materialize.tool(),
|
|
296
|
+
]
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
Each tool accepts only `{"sql": "..."}`. The agent chooses native SQL; trusted code retains source
|
|
300
|
+
scope, destination scope, execution limits, credentials, and verification.
|
|
301
|
+
|
|
302
|
+
The result returned by `tool.invoke(...)` exposes the same URI:
|
|
303
|
+
|
|
304
|
+
```python
|
|
305
|
+
query_uri = (
|
|
306
|
+
await query.tool().invoke(
|
|
307
|
+
sql="SELECT * FROM analytics.customers",
|
|
308
|
+
)
|
|
309
|
+
).uri
|
|
310
|
+
|
|
311
|
+
materialized_uri = (
|
|
312
|
+
await materialize.tool().invoke(
|
|
313
|
+
sql="CREATE TABLE agent_scratch.customers AS SELECT * FROM raw.customers",
|
|
314
|
+
)
|
|
315
|
+
).uri
|
|
316
|
+
```
|
|
317
|
+
|
|
318
|
+
An agent framework returns these URI values to the model as normal tool output. A query URI is
|
|
319
|
+
`None` when the provider returns only bounded inline rows.
|
|
320
|
+
|
|
321
|
+
> **Configure once. Call directly or expose as a tool.**
|
|
322
|
+
|
|
323
|
+
The engine moves the data directly. Gantry returns an output reference, not the materialized rows.
|
|
324
|
+
Existing destinations, disallowed sources, replacement statements, unsupported limits, and failed
|
|
325
|
+
verification are reported without silently weakening the configuration.
|
|
326
|
+
|
|
327
|
+
For a reconnectable engine, submission and observation can be separated:
|
|
328
|
+
|
|
329
|
+
```python
|
|
330
|
+
handle = await materialize.submit(sql)
|
|
331
|
+
|
|
332
|
+
# A recreated materializer can reconnect with the durable handle.
|
|
333
|
+
result = await materialize.wait(handle)
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
See [SQL materialization](docs/materialization.md) for the complete v0 contract.
|
|
337
|
+
|
|
338
|
+
---
|
|
339
|
+
|
|
340
|
+
## Defense in depth
|
|
341
|
+
|
|
342
|
+
Gantry is not a replacement for database permissions, IAM, network policy, or the security controls of the underlying system.
|
|
343
|
+
|
|
344
|
+
It coordinates them.
|
|
345
|
+
|
|
346
|
+
```text
|
|
347
|
+
Agent
|
|
348
|
+
│
|
|
349
|
+
▼
|
|
350
|
+
Gantry policy
|
|
351
|
+
│
|
|
352
|
+
▼
|
|
353
|
+
Scoped credentials / IAM
|
|
354
|
+
│
|
|
355
|
+
▼
|
|
356
|
+
Native database controls
|
|
357
|
+
│
|
|
358
|
+
▼
|
|
359
|
+
Execution engine
|
|
360
|
+
```
|
|
361
|
+
|
|
362
|
+
Use read-only database roles, scoped service accounts, authorized datasets, statement timeouts, resource quotas, and network restrictions wherever the underlying system supports them.
|
|
363
|
+
|
|
364
|
+
If Gantry cannot satisfy a required policy, execution should fail closed.
|
|
365
|
+
|
|
366
|
+
```text
|
|
367
|
+
Policy requires cost limit
|
|
368
|
+
│
|
|
369
|
+
▼
|
|
370
|
+
Adapter cannot enforce cost limit
|
|
371
|
+
│
|
|
372
|
+
▼
|
|
373
|
+
REJECTED
|
|
374
|
+
```
|
|
375
|
+
|
|
376
|
+
No silent downgrade.
|
|
377
|
+
|
|
378
|
+
---
|
|
379
|
+
|
|
380
|
+
## One model across data systems
|
|
381
|
+
|
|
382
|
+
The proposal changes depending on the engine.
|
|
383
|
+
|
|
384
|
+
The trust boundary does not.
|
|
385
|
+
|
|
386
|
+
### Postgres
|
|
387
|
+
|
|
388
|
+
```text
|
|
389
|
+
Agent
|
|
390
|
+
│ SQL
|
|
391
|
+
▼
|
|
392
|
+
Gantry
|
|
393
|
+
│ admit
|
|
394
|
+
▼
|
|
395
|
+
Postgres
|
|
396
|
+
│
|
|
397
|
+
▼
|
|
398
|
+
Rows
|
|
399
|
+
```
|
|
400
|
+
|
|
401
|
+
### BigQuery
|
|
402
|
+
|
|
403
|
+
```text
|
|
404
|
+
Agent
|
|
405
|
+
│ SQL
|
|
406
|
+
▼
|
|
407
|
+
Gantry
|
|
408
|
+
│ admit + cost constraints
|
|
409
|
+
▼
|
|
410
|
+
BigQuery
|
|
411
|
+
│
|
|
412
|
+
▼
|
|
413
|
+
Query Job
|
|
414
|
+
│
|
|
415
|
+
▼
|
|
416
|
+
Output Reference
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
### Flink SQL
|
|
420
|
+
|
|
421
|
+
Choose the execution model first, then use Flink as the engine. The SQL remains native Flink SQL.
|
|
422
|
+
|
|
423
|
+
```python
|
|
424
|
+
batch = gantry.batch.connect("flink", endpoint=FLINK_ENDPOINT)
|
|
425
|
+
daily_orders = batch.job(
|
|
426
|
+
inputs=["raw.orders"],
|
|
427
|
+
outputs=["analytics.daily_orders"],
|
|
428
|
+
checks=[gantry.verify.output_exists(), gantry.verify.row_count(min=1)],
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
result = await daily_orders("""
|
|
432
|
+
INSERT INTO analytics.daily_orders
|
|
433
|
+
SELECT CAST(order_time AS DATE), COUNT(*)
|
|
434
|
+
FROM raw.orders
|
|
435
|
+
GROUP BY CAST(order_time AS DATE)
|
|
436
|
+
""")
|
|
437
|
+
```
|
|
438
|
+
|
|
439
|
+
Long-running streams use the same configure-once shape, but acceptance means healthy and running
|
|
440
|
+
rather than finished:
|
|
441
|
+
|
|
442
|
+
```python
|
|
443
|
+
stream = gantry.stream.connect("flink", endpoint=FLINK_ENDPOINT)
|
|
444
|
+
clean_events = stream.job(
|
|
445
|
+
inputs=["raw.events"],
|
|
446
|
+
outputs=["clean.events"],
|
|
447
|
+
checks=[
|
|
448
|
+
gantry.verify.running(),
|
|
449
|
+
gantry.verify.restart_count(max=3),
|
|
450
|
+
gantry.verify.watermark_lag(max_seconds=60),
|
|
451
|
+
],
|
|
452
|
+
)
|
|
453
|
+
|
|
454
|
+
result = await clean_events("""
|
|
455
|
+
INSERT INTO clean.events
|
|
456
|
+
SELECT * FROM raw.events WHERE event_type IS NOT NULL
|
|
457
|
+
""")
|
|
458
|
+
|
|
459
|
+
tools = [clean_events.tool()] # the agent sees only {"sql": "..."}
|
|
460
|
+
```
|
|
461
|
+
|
|
462
|
+
A database query may finish in milliseconds.
|
|
463
|
+
|
|
464
|
+
A warehouse query may become an asynchronous job.
|
|
465
|
+
|
|
466
|
+
A Flink SQL statement may create a stream that runs indefinitely.
|
|
467
|
+
|
|
468
|
+
Gantry provides a common execution boundary without pretending those systems have the same computation model.
|
|
469
|
+
|
|
470
|
+
---
|
|
471
|
+
|
|
472
|
+
## Native SQL stays native
|
|
473
|
+
|
|
474
|
+
Gantry is not a SQL abstraction layer.
|
|
475
|
+
|
|
476
|
+
```text
|
|
477
|
+
Postgres SQL ──────> Postgres
|
|
478
|
+
BigQuery SQL ──────> BigQuery
|
|
479
|
+
Snowflake SQL ──────> Snowflake
|
|
480
|
+
Flink SQL ──────> Flink
|
|
481
|
+
```
|
|
482
|
+
|
|
483
|
+
No Gantry query language.
|
|
484
|
+
|
|
485
|
+
No requirement to rewrite your workloads into a common DSL.
|
|
486
|
+
|
|
487
|
+
The engine still owns computation.
|
|
488
|
+
|
|
489
|
+
Gantry owns the boundary around execution.
|
|
490
|
+
|
|
491
|
+
---
|
|
492
|
+
|
|
493
|
+
## Gantry is not in your data path
|
|
494
|
+
|
|
495
|
+
Large datasets should not flow through Gantry.
|
|
496
|
+
|
|
497
|
+
```text
|
|
498
|
+
Agent
|
|
499
|
+
│
|
|
500
|
+
│ proposal
|
|
501
|
+
▼
|
|
502
|
+
Gantry
|
|
503
|
+
│
|
|
504
|
+
│ execution request
|
|
505
|
+
▼
|
|
506
|
+
Data System ───────────────> Data System
|
|
507
|
+
│
|
|
508
|
+
│ reference / bounded result
|
|
509
|
+
▼
|
|
510
|
+
Gantry
|
|
511
|
+
│
|
|
512
|
+
▼
|
|
513
|
+
Agent
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
Small, explicitly bounded query results can be returned inline.
|
|
517
|
+
|
|
518
|
+
Large results remain in the underlying data system and are returned as references.
|
|
519
|
+
|
|
520
|
+
> **References by default. Inline data only when explicitly bounded.**
|
|
521
|
+
|
|
522
|
+
Gantry is a control plane, not a data plane.
|
|
523
|
+
|
|
524
|
+
---
|
|
525
|
+
|
|
526
|
+
## From SQL to data jobs
|
|
527
|
+
|
|
528
|
+
SQL is the first execution surface because it gives agents immediate access to useful data systems.
|
|
529
|
+
|
|
530
|
+
But the execution model is intentionally broader.
|
|
531
|
+
|
|
532
|
+
```text
|
|
533
|
+
Gantry
|
|
534
|
+
|
|
535
|
+
Agent proposal ──> admission
|
|
536
|
+
│
|
|
537
|
+
▼
|
|
538
|
+
execute
|
|
539
|
+
│
|
|
540
|
+
┌─────────┼─────────┐
|
|
541
|
+
▼ ▼ ▼
|
|
542
|
+
Postgres BigQuery Flink
|
|
543
|
+
│ │ │
|
|
544
|
+
▼ ▼ ▼
|
|
545
|
+
rows job/ref stream
|
|
546
|
+
│ │ │
|
|
547
|
+
└─────────┼─────────┘
|
|
548
|
+
▼
|
|
549
|
+
observe
|
|
550
|
+
│
|
|
551
|
+
▼
|
|
552
|
+
verify
|
|
553
|
+
│
|
|
554
|
+
▼
|
|
555
|
+
result
|
|
556
|
+
```
|
|
557
|
+
|
|
558
|
+
The goal is not to standardize how data is computed.
|
|
559
|
+
|
|
560
|
+
The goal is to standardize the contract around agent-generated execution.
|
|
561
|
+
|
|
562
|
+
---
|
|
563
|
+
|
|
564
|
+
## Core principle
|
|
565
|
+
|
|
566
|
+
Gantry follows one rule:
|
|
567
|
+
|
|
568
|
+
> **The agent owns adaptation. Gantry owns acceptance.**
|
|
569
|
+
|
|
570
|
+
Agents are good at deciding what work might solve a problem.
|
|
571
|
+
|
|
572
|
+
Infrastructure is good at enforcing permissions, limits, execution semantics, and guarantees.
|
|
573
|
+
|
|
574
|
+
Gantry keeps those responsibilities separate.
|
|
575
|
+
|
|
576
|
+
---
|
|
577
|
+
|
|
578
|
+
## Supported systems
|
|
579
|
+
|
|
580
|
+
| System | Current Gantry operation |
|
|
581
|
+
| --- | --- |
|
|
582
|
+
| PostgreSQL, Supabase, Neon | Governed queries and bounded inline results |
|
|
583
|
+
| BigQuery | Governed queries, output references, and SQL materialization |
|
|
584
|
+
| Snowflake | Governed queries and reconnectable query jobs |
|
|
585
|
+
| DuckDB | Governed queries and process-local SQL materialization |
|
|
586
|
+
| Flink SQL | Governed batch and stream jobs with durable handles, verification, and cancellation |
|
|
587
|
+
|
|
588
|
+
SQL materialization v0 is enabled for BigQuery and DuckDB. Other providers fail closed if that
|
|
589
|
+
operation is requested; see the [provider support matrix](docs/materialization.md#provider-support).
|
|
590
|
+
|
|
591
|
+
More execution targets can implement the same Gantry execution contract without forcing their computation model into a common abstraction.
|
|
592
|
+
|
|
593
|
+
---
|
|
594
|
+
|
|
595
|
+
## What Gantry owns
|
|
596
|
+
|
|
597
|
+
```text
|
|
598
|
+
admission
|
|
599
|
+
policy enforcement
|
|
600
|
+
execution identity
|
|
601
|
+
submission
|
|
602
|
+
status
|
|
603
|
+
cancellation
|
|
604
|
+
failure normalization
|
|
605
|
+
bounded results
|
|
606
|
+
output references
|
|
607
|
+
observation
|
|
608
|
+
verification
|
|
609
|
+
accepted results
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
## What Gantry does not own
|
|
613
|
+
|
|
614
|
+
```text
|
|
615
|
+
agent planning
|
|
616
|
+
prompting
|
|
617
|
+
code generation
|
|
618
|
+
SQL dialects
|
|
619
|
+
query planning
|
|
620
|
+
data transformation semantics
|
|
621
|
+
distributed execution
|
|
622
|
+
stream processing
|
|
623
|
+
storage
|
|
624
|
+
database permissions
|
|
625
|
+
IAM
|
|
626
|
+
data transport
|
|
627
|
+
```
|
|
628
|
+
|
|
629
|
+
If an existing system already knows how to perform the work, Gantry should use it.
|
|
630
|
+
|
|
631
|
+
---
|
|
632
|
+
|
|
633
|
+
## Try it
|
|
634
|
+
|
|
635
|
+
Seven runnable examples, indexed by what you are trying to do, in
|
|
636
|
+
[`examples/`](examples/). The quickest needs nothing but a Python environment:
|
|
637
|
+
|
|
638
|
+
```bash
|
|
639
|
+
pip install "data-gantry[duckdb]"
|
|
640
|
+
python examples/local_duckdb.py
|
|
641
|
+
```
|
|
642
|
+
|
|
643
|
+
For the rest, one stack brings up PostgreSQL, Flink, and the catalog they read
|
|
644
|
+
through:
|
|
645
|
+
|
|
646
|
+
```bash
|
|
647
|
+
docker compose -f examples/stack/docker-compose.yml up -d --wait
|
|
648
|
+
psql postgresql://gantry:gantry@localhost:5432/gantry -f examples/seed.sql
|
|
649
|
+
python examples/warehouse_rollup.py
|
|
650
|
+
```
|
|
651
|
+
|
|
652
|
+
`warehouse_rollup.py` is the one to read if you want the point rather than the
|
|
653
|
+
API: an agent writes a rollup over 200,000 orders, and of its four attempts one
|
|
654
|
+
is accepted, one runs perfectly and produces a table nobody should read, one
|
|
655
|
+
writes where it was not asked to, and one is refused by the planner.
|
|
656
|
+
|
|
657
|
+
## Documentation
|
|
658
|
+
|
|
659
|
+
- [SQL providers](docs/sql.md)
|
|
660
|
+
- [SQL materialization](docs/materialization.md)
|
|
661
|
+
- [Safety and policies](docs/policy.md)
|
|
662
|
+
- [Flink SQL](docs/flink.md)
|
|
663
|
+
- [Custom SQL adapters](docs/adapters.md)
|
|
664
|
+
|
|
665
|
+
---
|
|
666
|
+
|
|
667
|
+
## License
|
|
668
|
+
|
|
669
|
+
Gantry is licensed under the Apache License 2.0.
|
|
670
|
+
|
|
671
|
+
See [LICENSE](./LICENSE).
|