vgi-python 0.31.4__py3-none-any.whl → 0.32.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vgi/__init__.py +5 -0
- vgi/_test_fixtures/schema_reconcile/worker.py +1 -3
- vgi/_test_fixtures/simple_writable.py +72 -32
- vgi/_test_fixtures/writable/generic.py +14 -10
- vgi/_test_fixtures/writable/table.py +10 -10
- vgi/_test_fixtures/writable/worker.py +1 -4
- vgi/catalog/__init__.py +5 -0
- vgi/catalog/catalog_interface.py +7 -11
- vgi/catalog/descriptors.py +27 -3
- vgi/http/vgi-client.js +17 -18
- vgi/transactor/protocol.py +4 -3
- vgi/transactor/server.py +54 -21
- vgi/write_results.py +73 -0
- {vgi_python-0.31.4.dist-info → vgi_python-0.32.0.dist-info}/METADATA +1 -1
- {vgi_python-0.31.4.dist-info → vgi_python-0.32.0.dist-info}/RECORD +18 -17
- {vgi_python-0.31.4.dist-info → vgi_python-0.32.0.dist-info}/WHEEL +0 -0
- {vgi_python-0.31.4.dist-info → vgi_python-0.32.0.dist-info}/entry_points.txt +0 -0
- {vgi_python-0.31.4.dist-info → vgi_python-0.32.0.dist-info}/licenses/LICENSE +0 -0
vgi/__init__.py
CHANGED
|
@@ -129,10 +129,15 @@ from vgi.table_in_out_function import (
|
|
|
129
129
|
TableInOutGenerator,
|
|
130
130
|
)
|
|
131
131
|
from vgi.worker import Worker
|
|
132
|
+
from vgi.write_results import WriteOperation, WriteResultMode, write_changes_batch, write_result_schema
|
|
132
133
|
|
|
133
134
|
__all__ = [
|
|
134
135
|
"AggregateBindParams",
|
|
135
136
|
"AggregateFunction",
|
|
137
|
+
"WriteOperation",
|
|
138
|
+
"WriteResultMode",
|
|
139
|
+
"write_result_schema",
|
|
140
|
+
"write_changes_batch",
|
|
136
141
|
"AnyArrow",
|
|
137
142
|
"AnyArrowValue",
|
|
138
143
|
"Arg",
|
|
@@ -528,9 +528,7 @@ class SchemaReconcileCatalog(ReadOnlyCatalogInterface):
|
|
|
528
528
|
not_null_constraints=[],
|
|
529
529
|
unique_constraints=[],
|
|
530
530
|
check_constraints=[],
|
|
531
|
-
|
|
532
|
-
supports_update=True,
|
|
533
|
-
supports_delete=True,
|
|
531
|
+
write_result_modes={"insert": "changes", "update": "changes", "delete": "changes"},
|
|
534
532
|
)
|
|
535
533
|
|
|
536
534
|
def schemas(
|
|
@@ -13,7 +13,7 @@ Three pre-defined tables are exposed under the ``main`` schema:
|
|
|
13
13
|
|
|
14
14
|
* ``items`` — supports INSERT/UPDATE/DELETE with RETURNING.
|
|
15
15
|
* ``items_no_returning`` — supports INSERT/UPDATE/DELETE *without* RETURNING.
|
|
16
|
-
Used to exercise the
|
|
16
|
+
Used to exercise the ``count``-only RETURNING rejection path.
|
|
17
17
|
* ``items_insert_only`` — supports INSERT only (no UPDATE/DELETE/RETURNING).
|
|
18
18
|
|
|
19
19
|
State is held module-global, keyed by ``attach_opaque_data``. Per the
|
|
@@ -60,6 +60,7 @@ from vgi.schema_utils import schema as build_schema
|
|
|
60
60
|
from vgi.table_function import BindParams, InitParams, ProcessParams, TableFunctionGenerator
|
|
61
61
|
from vgi.table_in_out_function import TableInOutGenerator
|
|
62
62
|
from vgi.worker import Worker
|
|
63
|
+
from vgi.write_results import WriteResultMode, coerce_write_result_mode, write_changes_batch, write_result_schema
|
|
63
64
|
|
|
64
65
|
if TYPE_CHECKING:
|
|
65
66
|
from vgi.catalog.catalog_interface import (
|
|
@@ -82,7 +83,7 @@ CATALOG_NAME = "simple_writable"
|
|
|
82
83
|
_ROWID_FIELD = pa.field("rowid", pa.int64(), metadata={b"is_row_id": b""})
|
|
83
84
|
|
|
84
85
|
# Output schema for write functions returning affected row counts.
|
|
85
|
-
_COUNT_SCHEMA =
|
|
86
|
+
_COUNT_SCHEMA = pa.schema([pa.field("count", pa.int64(), nullable=False)])
|
|
86
87
|
|
|
87
88
|
|
|
88
89
|
# ============================================================================
|
|
@@ -116,7 +117,7 @@ def _table_specs() -> dict[str, pa.Schema]:
|
|
|
116
117
|
"items": build_schema(id=pa.int64(), name=pa.string(), qty=pa.int64()),
|
|
117
118
|
"items_no_returning": build_schema(id=pa.int64(), name=pa.string(), qty=pa.int64()),
|
|
118
119
|
"items_insert_only": build_schema(id=pa.int64(), name=pa.string()),
|
|
119
|
-
# Lies: catalog advertises
|
|
120
|
+
# Lies: catalog advertises rows support but the insert
|
|
120
121
|
# function always emits a (count BIGINT) batch. Used by tests to verify
|
|
121
122
|
# the C++ extension rejects the mismatched batch with a clean IOException
|
|
122
123
|
# instead of crashing inside ArrowToDuckDB.
|
|
@@ -124,8 +125,12 @@ def _table_specs() -> dict[str, pa.Schema]:
|
|
|
124
125
|
}
|
|
125
126
|
|
|
126
127
|
|
|
127
|
-
def
|
|
128
|
-
|
|
128
|
+
def _table_result_mode(name: str) -> str:
|
|
129
|
+
if name == "items_no_returning":
|
|
130
|
+
return "count"
|
|
131
|
+
if name == "items_broken_returning":
|
|
132
|
+
return "rows"
|
|
133
|
+
return "changes"
|
|
129
134
|
|
|
130
135
|
|
|
131
136
|
def _table_supports_update_delete(name: str) -> bool:
|
|
@@ -224,20 +229,25 @@ def _attach_opaque_data_from_process(params: ProcessParams[None]) -> bytes:
|
|
|
224
229
|
return bytes(aid)
|
|
225
230
|
|
|
226
231
|
|
|
227
|
-
def _parse_write_options(params: BindParams[None]) -> dict[str, Any]:
|
|
232
|
+
def _parse_write_options(params: BindParams[None] | ProcessParams[None]) -> dict[str, Any]:
|
|
228
233
|
"""Decode the write_options batch passed in named arguments."""
|
|
229
|
-
defaults: dict[str, Any] = {"
|
|
230
|
-
if
|
|
234
|
+
defaults: dict[str, Any] = {"result_mode": "count", "on_conflict": "throw", "on_conflict_columns": []}
|
|
235
|
+
if isinstance(params, BindParams):
|
|
236
|
+
bind_call = params.bind_call
|
|
237
|
+
else:
|
|
238
|
+
assert params.init_call is not None
|
|
239
|
+
bind_call = params.init_call.bind_call
|
|
240
|
+
if not (bind_call.arguments and bind_call.arguments.named):
|
|
231
241
|
return defaults
|
|
232
|
-
val =
|
|
242
|
+
val = bind_call.arguments.named.get("write_options")
|
|
233
243
|
if val is None:
|
|
234
244
|
return defaults
|
|
235
245
|
from vgi_rpc.utils import deserialize_record_batch
|
|
236
246
|
|
|
237
247
|
batch, _ = deserialize_record_batch(val.as_py())
|
|
238
248
|
out = dict(defaults)
|
|
239
|
-
if "
|
|
240
|
-
out["
|
|
249
|
+
if "result_mode" in batch.schema.names:
|
|
250
|
+
out["result_mode"] = coerce_write_result_mode(batch.column("result_mode")[0].as_py())
|
|
241
251
|
if "on_conflict" in batch.schema.names:
|
|
242
252
|
out["on_conflict"] = batch.column("on_conflict")[0].as_py()
|
|
243
253
|
if "on_conflict_columns" in batch.schema.names:
|
|
@@ -245,6 +255,26 @@ def _parse_write_options(params: BindParams[None]) -> dict[str, Any]:
|
|
|
245
255
|
return out
|
|
246
256
|
|
|
247
257
|
|
|
258
|
+
def _emit_changes(
|
|
259
|
+
out: OutputCollector,
|
|
260
|
+
table_schema: pa.Schema,
|
|
261
|
+
old_rows: Sequence[tuple[Any, ...] | None],
|
|
262
|
+
new_rows: Sequence[tuple[Any, ...] | None],
|
|
263
|
+
) -> None:
|
|
264
|
+
names = table_schema.names
|
|
265
|
+
|
|
266
|
+
def as_struct(row: tuple[Any, ...] | None) -> dict[str, Any] | None:
|
|
267
|
+
return None if row is None else dict(zip(names, row, strict=True))
|
|
268
|
+
|
|
269
|
+
out.emit(
|
|
270
|
+
write_changes_batch(
|
|
271
|
+
table_schema,
|
|
272
|
+
[as_struct(row) for row in old_rows],
|
|
273
|
+
[as_struct(row) for row in new_rows],
|
|
274
|
+
)
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
|
|
248
278
|
def _user_schema_from_bind(params: BindParams[None]) -> pa.Schema:
|
|
249
279
|
qualified = _qualified_from_bind(params)
|
|
250
280
|
return _get_user_schema(qualified)
|
|
@@ -329,9 +359,8 @@ class SimpleInsert(TableInOutGenerator[None, None]):
|
|
|
329
359
|
@classmethod
|
|
330
360
|
def on_bind(cls, params: BindParams[None]) -> BindResponse:
|
|
331
361
|
opts = _parse_write_options(params)
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
return BindResponse(output_schema=_COUNT_SCHEMA)
|
|
362
|
+
mode = opts["result_mode"]
|
|
363
|
+
return BindResponse(output_schema=write_result_schema(mode, _user_schema_from_bind(params)))
|
|
335
364
|
|
|
336
365
|
@classmethod
|
|
337
366
|
def process(
|
|
@@ -345,7 +374,7 @@ class SimpleInsert(TableInOutGenerator[None, None]):
|
|
|
345
374
|
attach_opaque_data = _attach_opaque_data_from_process(params)
|
|
346
375
|
bare = _bare_name(qualified)
|
|
347
376
|
user_schema = _get_user_schema(qualified)
|
|
348
|
-
|
|
377
|
+
result_mode: WriteResultMode = _parse_write_options(params)["result_mode"]
|
|
349
378
|
|
|
350
379
|
col_names = [f.name for f in user_schema]
|
|
351
380
|
cols_sql = ", ".join(f'"{c}"' for c in col_names)
|
|
@@ -362,12 +391,14 @@ class SimpleInsert(TableInOutGenerator[None, None]):
|
|
|
362
391
|
)
|
|
363
392
|
conn.execute("COMMIT")
|
|
364
393
|
|
|
365
|
-
if
|
|
394
|
+
if result_mode == "rows":
|
|
366
395
|
out_cols: dict[str, list[Any]] = {c: [] for c in col_names}
|
|
367
396
|
for row in rows_to_insert:
|
|
368
397
|
for c, v in zip(col_names, row, strict=True):
|
|
369
398
|
out_cols[c].append(v)
|
|
370
399
|
out.emit(pa.RecordBatch.from_pydict(out_cols, schema=user_schema))
|
|
400
|
+
elif result_mode == "changes":
|
|
401
|
+
_emit_changes(out, user_schema, [None] * len(rows_to_insert), rows_to_insert)
|
|
371
402
|
else:
|
|
372
403
|
out.emit(pa.RecordBatch.from_pydict({"count": [batch.num_rows]}, schema=_COUNT_SCHEMA))
|
|
373
404
|
|
|
@@ -381,9 +412,8 @@ class SimpleUpdate(TableInOutGenerator[None, None]):
|
|
|
381
412
|
@classmethod
|
|
382
413
|
def on_bind(cls, params: BindParams[None]) -> BindResponse:
|
|
383
414
|
opts = _parse_write_options(params)
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
return BindResponse(output_schema=_COUNT_SCHEMA)
|
|
415
|
+
mode = opts["result_mode"]
|
|
416
|
+
return BindResponse(output_schema=write_result_schema(mode, _user_schema_from_bind(params)))
|
|
387
417
|
|
|
388
418
|
@classmethod
|
|
389
419
|
def process(
|
|
@@ -397,7 +427,7 @@ class SimpleUpdate(TableInOutGenerator[None, None]):
|
|
|
397
427
|
attach_opaque_data = _attach_opaque_data_from_process(params)
|
|
398
428
|
bare = _bare_name(qualified)
|
|
399
429
|
user_schema = _get_user_schema(qualified)
|
|
400
|
-
|
|
430
|
+
result_mode: WriteResultMode = _parse_write_options(params)["result_mode"]
|
|
401
431
|
|
|
402
432
|
update_cols = [n for n in batch.schema.names if n != "rowid"]
|
|
403
433
|
set_clause = ", ".join(f'"{c}"=?' for c in update_cols)
|
|
@@ -405,23 +435,31 @@ class SimpleUpdate(TableInOutGenerator[None, None]):
|
|
|
405
435
|
select_list = ", ".join(f'"{c}"' for c in user_col_names)
|
|
406
436
|
|
|
407
437
|
rowid_col = batch.column("rowid")
|
|
438
|
+
old_rows: list[tuple[Any, ...]] = []
|
|
408
439
|
updated: list[tuple[Any, ...]] = []
|
|
409
440
|
with _connect(attach_opaque_data) as conn:
|
|
410
441
|
conn.execute("BEGIN")
|
|
411
442
|
for i in range(batch.num_rows):
|
|
412
443
|
rowid = rowid_col[i].as_py()
|
|
444
|
+
old_row = conn.execute(f'SELECT {select_list} FROM "{bare}" WHERE rowid=?', (rowid,)).fetchone()
|
|
445
|
+
if old_row is None:
|
|
446
|
+
conn.execute("ROLLBACK")
|
|
447
|
+
raise ValueError(f"Update target rowid {rowid} not in table {qualified}")
|
|
413
448
|
values = tuple(batch.column(c)[i].as_py() for c in update_cols)
|
|
414
449
|
cur = conn.execute(f'UPDATE "{bare}" SET {set_clause} WHERE rowid=?', (*values, rowid))
|
|
415
450
|
if cur.rowcount == 0:
|
|
416
451
|
conn.execute("ROLLBACK")
|
|
417
452
|
raise ValueError(f"Update target rowid {rowid} not in table {qualified}")
|
|
418
453
|
row = conn.execute(f'SELECT {select_list} FROM "{bare}" WHERE rowid=?', (rowid,)).fetchone()
|
|
454
|
+
old_rows.append(old_row)
|
|
419
455
|
updated.append(row)
|
|
420
456
|
conn.execute("COMMIT")
|
|
421
457
|
|
|
422
|
-
if
|
|
458
|
+
if result_mode == "rows":
|
|
423
459
|
cols = {c: [row[i] for row in updated] for i, c in enumerate(user_col_names)}
|
|
424
460
|
out.emit(pa.RecordBatch.from_pydict(cols, schema=user_schema))
|
|
461
|
+
elif result_mode == "changes":
|
|
462
|
+
_emit_changes(out, user_schema, old_rows, updated)
|
|
425
463
|
else:
|
|
426
464
|
out.emit(pa.RecordBatch.from_pydict({"count": [batch.num_rows]}, schema=_COUNT_SCHEMA))
|
|
427
465
|
|
|
@@ -435,9 +473,8 @@ class SimpleDelete(TableInOutGenerator[None, None]):
|
|
|
435
473
|
@classmethod
|
|
436
474
|
def on_bind(cls, params: BindParams[None]) -> BindResponse:
|
|
437
475
|
opts = _parse_write_options(params)
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
return BindResponse(output_schema=_COUNT_SCHEMA)
|
|
476
|
+
mode = opts["result_mode"]
|
|
477
|
+
return BindResponse(output_schema=write_result_schema(mode, _user_schema_from_bind(params)))
|
|
441
478
|
|
|
442
479
|
@classmethod
|
|
443
480
|
def process(
|
|
@@ -451,7 +488,7 @@ class SimpleDelete(TableInOutGenerator[None, None]):
|
|
|
451
488
|
attach_opaque_data = _attach_opaque_data_from_process(params)
|
|
452
489
|
bare = _bare_name(qualified)
|
|
453
490
|
user_schema = _get_user_schema(qualified)
|
|
454
|
-
|
|
491
|
+
result_mode: WriteResultMode = _parse_write_options(params)["result_mode"]
|
|
455
492
|
|
|
456
493
|
user_col_names = [f.name for f in user_schema]
|
|
457
494
|
select_list = ", ".join(f'"{c}"' for c in user_col_names)
|
|
@@ -470,9 +507,11 @@ class SimpleDelete(TableInOutGenerator[None, None]):
|
|
|
470
507
|
deleted.append(row)
|
|
471
508
|
conn.execute("COMMIT")
|
|
472
509
|
|
|
473
|
-
if
|
|
510
|
+
if result_mode == "rows":
|
|
474
511
|
cols = {c: [row[i] for row in deleted] for i, c in enumerate(user_col_names)}
|
|
475
512
|
out.emit(pa.RecordBatch.from_pydict(cols, schema=user_schema))
|
|
513
|
+
elif result_mode == "changes":
|
|
514
|
+
_emit_changes(out, user_schema, deleted, [None] * len(deleted))
|
|
476
515
|
else:
|
|
477
516
|
out.emit(pa.RecordBatch.from_pydict({"count": [batch.num_rows]}, schema=_COUNT_SCHEMA))
|
|
478
517
|
|
|
@@ -491,7 +530,7 @@ class BrokenReturningInsert(TableInOutGenerator[None, None]):
|
|
|
491
530
|
|
|
492
531
|
@classmethod
|
|
493
532
|
def on_bind(cls, params: BindParams[None]) -> BindResponse:
|
|
494
|
-
# Always advertise the count surface, even when
|
|
533
|
+
# Always advertise the count surface, even when result_mode=rows.
|
|
495
534
|
# The C++ side will see this at bind via the worker's output schema and
|
|
496
535
|
# tries to route the responses through ArrowToDuckDB on the table-row
|
|
497
536
|
# schema — that mismatch is what we want to catch at runtime.
|
|
@@ -524,7 +563,7 @@ class BrokenReturningInsert(TableInOutGenerator[None, None]):
|
|
|
524
563
|
rows_to_insert,
|
|
525
564
|
)
|
|
526
565
|
conn.execute("COMMIT")
|
|
527
|
-
# Always emit count, regardless of
|
|
566
|
+
# Always emit count, regardless of result_mode — that's the bug.
|
|
528
567
|
out.emit(pa.RecordBatch.from_pydict({"count": [batch.num_rows]}, schema=_COUNT_SCHEMA))
|
|
529
568
|
|
|
530
569
|
|
|
@@ -619,10 +658,11 @@ class SimpleWritableCatalog(ReadOnlyCatalogInterface):
|
|
|
619
658
|
check_constraints=[],
|
|
620
659
|
primary_key_constraints=[],
|
|
621
660
|
foreign_key_constraints=[],
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
661
|
+
write_result_modes={
|
|
662
|
+
operation: _table_result_mode(name)
|
|
663
|
+
for operation, supported in (("insert", True), ("update", ud), ("delete", ud))
|
|
664
|
+
if supported
|
|
665
|
+
},
|
|
626
666
|
)
|
|
627
667
|
|
|
628
668
|
def table_get(
|
|
@@ -27,12 +27,14 @@ from vgi._test_fixtures.writable.table import (
|
|
|
27
27
|
_get_attach_opaque_data,
|
|
28
28
|
_get_pushdown_filters,
|
|
29
29
|
_get_tx_id,
|
|
30
|
-
|
|
30
|
+
_parse_write_options,
|
|
31
|
+
_result_mode,
|
|
31
32
|
transactor_proxy,
|
|
32
33
|
)
|
|
33
34
|
from vgi.invocation import BindResponse, GlobalInitResponse
|
|
34
35
|
from vgi.table_function import BindParams, InitParams, ProcessParams, TableFunctionGenerator
|
|
35
36
|
from vgi.table_in_out_function import TableInOutGenerator
|
|
37
|
+
from vgi.write_results import WriteResultMode, coerce_write_result_mode, write_result_schema
|
|
36
38
|
|
|
37
39
|
__all__ = [
|
|
38
40
|
"GenericTableDelete",
|
|
@@ -141,15 +143,16 @@ class _GenericWriteBase(TableInOutGenerator[None, None]):
|
|
|
141
143
|
|
|
142
144
|
@classmethod
|
|
143
145
|
def on_bind(cls, params: BindParams[None]) -> BindResponse:
|
|
144
|
-
"""Bind
|
|
146
|
+
"""Bind using the exact result schema requested by the client."""
|
|
145
147
|
table_name = _get_table_name_from_bind(params)
|
|
146
|
-
|
|
148
|
+
result_mode = _result_mode(params)
|
|
149
|
+
if result_mode != "count":
|
|
147
150
|
attach_opaque_data = params.attach_opaque_data # unwrapped plaintext
|
|
148
151
|
tx_id = params.bind_call.transaction_opaque_data
|
|
149
152
|
assert attach_opaque_data is not None and tx_id is not None
|
|
150
153
|
table_schema = _get_table_schema_from_transactor(table_name, attach_opaque_data, tx_id)
|
|
151
154
|
user_fields = [f for f in table_schema if f.name not in ("rowid", "row_id")]
|
|
152
|
-
return BindResponse(output_schema=pa.schema(user_fields))
|
|
155
|
+
return BindResponse(output_schema=write_result_schema(result_mode, pa.schema(user_fields)))
|
|
153
156
|
return BindResponse(output_schema=_COUNT_SCHEMA)
|
|
154
157
|
|
|
155
158
|
@classmethod
|
|
@@ -159,7 +162,7 @@ class _GenericWriteBase(TableInOutGenerator[None, None]):
|
|
|
159
162
|
attach_opaque_data: bytes,
|
|
160
163
|
tx_id: bytes,
|
|
161
164
|
table_name: str,
|
|
162
|
-
|
|
165
|
+
result_mode: WriteResultMode,
|
|
163
166
|
batch: pa.RecordBatch,
|
|
164
167
|
) -> Any:
|
|
165
168
|
"""Open a write stream. Override for operations needing extra args."""
|
|
@@ -167,7 +170,7 @@ class _GenericWriteBase(TableInOutGenerator[None, None]):
|
|
|
167
170
|
attach_opaque_data=attach_opaque_data,
|
|
168
171
|
tx_id=tx_id,
|
|
169
172
|
table_name=table_name,
|
|
170
|
-
|
|
173
|
+
result_mode=result_mode,
|
|
171
174
|
)
|
|
172
175
|
|
|
173
176
|
@classmethod
|
|
@@ -176,9 +179,10 @@ class _GenericWriteBase(TableInOutGenerator[None, None]):
|
|
|
176
179
|
table_name = _get_table_name_from_process(params)
|
|
177
180
|
attach_opaque_data = _get_attach_opaque_data(params)
|
|
178
181
|
tx_id = _get_tx_id(params)
|
|
179
|
-
|
|
182
|
+
assert params.init_call is not None
|
|
183
|
+
result_mode = coerce_write_result_mode(str(_parse_write_options(params.init_call.bind_call)["result_mode"]))
|
|
180
184
|
proxy = transactor_proxy._get_proxy()
|
|
181
|
-
with cls._open_stream(proxy, attach_opaque_data, tx_id, table_name,
|
|
185
|
+
with cls._open_stream(proxy, attach_opaque_data, tx_id, table_name, result_mode, batch) as stream:
|
|
182
186
|
response = stream.exchange(AnnotatedBatch(batch=batch))
|
|
183
187
|
out.emit(response.batch)
|
|
184
188
|
|
|
@@ -211,7 +215,7 @@ class GenericTableUpdate(_GenericWriteBase):
|
|
|
211
215
|
attach_opaque_data: bytes,
|
|
212
216
|
tx_id: bytes,
|
|
213
217
|
table_name: str,
|
|
214
|
-
|
|
218
|
+
result_mode: WriteResultMode,
|
|
215
219
|
batch: pa.RecordBatch,
|
|
216
220
|
) -> Any:
|
|
217
221
|
"""Open an update stream with column list derived from the batch."""
|
|
@@ -221,7 +225,7 @@ class GenericTableUpdate(_GenericWriteBase):
|
|
|
221
225
|
tx_id=tx_id,
|
|
222
226
|
table_name=table_name,
|
|
223
227
|
columns=update_cols,
|
|
224
|
-
|
|
228
|
+
result_mode=result_mode,
|
|
225
229
|
)
|
|
226
230
|
|
|
227
231
|
|
|
@@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, Annotated
|
|
|
16
16
|
import pyarrow as pa
|
|
17
17
|
from vgi_rpc import AnnotatedBatch, ArrowSerializableDataclass, Transient
|
|
18
18
|
|
|
19
|
-
from vgi.schema_utils import schema
|
|
20
19
|
from vgi.table_function import BindParams, ProcessParams
|
|
21
20
|
|
|
22
21
|
if TYPE_CHECKING:
|
|
@@ -24,6 +23,7 @@ if TYPE_CHECKING:
|
|
|
24
23
|
|
|
25
24
|
from vgi.transactor.client import TransactorClient
|
|
26
25
|
from vgi.transactor.protocol import TransactorProtocol
|
|
26
|
+
from vgi.write_results import WriteResultMode, coerce_write_result_mode
|
|
27
27
|
|
|
28
28
|
__all__ = [
|
|
29
29
|
"TransactorProxy",
|
|
@@ -32,17 +32,17 @@ __all__ = [
|
|
|
32
32
|
]
|
|
33
33
|
|
|
34
34
|
# Output schema for write functions returning affected row counts.
|
|
35
|
-
_COUNT_SCHEMA = schema(count
|
|
35
|
+
_COUNT_SCHEMA = pa.schema([pa.field("count", pa.int64(), nullable=False)])
|
|
36
36
|
|
|
37
37
|
# DuckDB's native rowid pseudocolumn, marked with is_row_id metadata so the
|
|
38
38
|
# C++ extension knows which column carries the physical row identifier.
|
|
39
39
|
_ROWID_FIELD = pa.field("rowid", pa.int64(), metadata={b"is_row_id": b""})
|
|
40
40
|
|
|
41
41
|
|
|
42
|
-
def _parse_write_options(bind_call: BindRequest) -> dict[str,
|
|
42
|
+
def _parse_write_options(bind_call: BindRequest) -> dict[str, str | list[str]]:
|
|
43
43
|
"""Parse the write_options RecordBatch from the bind call's named arguments."""
|
|
44
|
-
defaults: dict[str,
|
|
45
|
-
"
|
|
44
|
+
defaults: dict[str, str | list[str]] = {
|
|
45
|
+
"result_mode": "count",
|
|
46
46
|
"on_conflict": "throw",
|
|
47
47
|
"on_conflict_columns": [],
|
|
48
48
|
}
|
|
@@ -56,8 +56,8 @@ def _parse_write_options(bind_call: BindRequest) -> dict[str, bool | str | list[
|
|
|
56
56
|
options_bytes = val.as_py()
|
|
57
57
|
batch, _ = deserialize_record_batch(options_bytes)
|
|
58
58
|
result = dict(defaults)
|
|
59
|
-
if "
|
|
60
|
-
result["
|
|
59
|
+
if "result_mode" in batch.schema.names:
|
|
60
|
+
result["result_mode"] = coerce_write_result_mode(batch.column("result_mode")[0].as_py())
|
|
61
61
|
if "on_conflict" in batch.schema.names:
|
|
62
62
|
result["on_conflict"] = batch.column("on_conflict")[0].as_py()
|
|
63
63
|
if "on_conflict_columns" in batch.schema.names:
|
|
@@ -65,10 +65,10 @@ def _parse_write_options(bind_call: BindRequest) -> dict[str, bool | str | list[
|
|
|
65
65
|
return result
|
|
66
66
|
|
|
67
67
|
|
|
68
|
-
def
|
|
69
|
-
"""
|
|
68
|
+
def _result_mode(params: BindParams[None]) -> WriteResultMode:
|
|
69
|
+
"""Return the exact writable result mode requested by the client."""
|
|
70
70
|
opts = _parse_write_options(params.bind_call)
|
|
71
|
-
return
|
|
71
|
+
return coerce_write_result_mode(str(opts["result_mode"]))
|
|
72
72
|
|
|
73
73
|
|
|
74
74
|
def _get_tx_id(params: ProcessParams[None]) -> bytes:
|
|
@@ -907,10 +907,7 @@ class WritableCatalog(ReadOnlyCatalogInterface):
|
|
|
907
907
|
check_constraints=check_constraints,
|
|
908
908
|
primary_key_constraints=primary_key_constraints,
|
|
909
909
|
foreign_key_constraints=foreign_key_constraints,
|
|
910
|
-
|
|
911
|
-
supports_update=True,
|
|
912
|
-
supports_delete=True,
|
|
913
|
-
supports_returning=True,
|
|
910
|
+
write_result_modes={"insert": "changes", "update": "changes", "delete": "changes"},
|
|
914
911
|
comment=table_comment,
|
|
915
912
|
tags={},
|
|
916
913
|
)
|
vgi/catalog/__init__.py
CHANGED
|
@@ -44,6 +44,7 @@ from vgi.catalog.descriptors import Catalog, ForeignKeyDef, Index, Macro, Schema
|
|
|
44
44
|
from vgi.catalog.secret_type import SecretTypeSpec
|
|
45
45
|
from vgi.catalog.setting import Setting, SettingSpec
|
|
46
46
|
from vgi.catalog.storage import CatalogStorage, CatalogStorageSqlite
|
|
47
|
+
from vgi.write_results import WriteOperation, WriteResultMode, write_changes_batch, write_result_schema
|
|
47
48
|
|
|
48
49
|
__all__ = [
|
|
49
50
|
# Type aliases
|
|
@@ -89,6 +90,10 @@ __all__ = [
|
|
|
89
90
|
"Sql",
|
|
90
91
|
"Table",
|
|
91
92
|
"View",
|
|
93
|
+
"WriteOperation",
|
|
94
|
+
"WriteResultMode",
|
|
95
|
+
"write_result_schema",
|
|
96
|
+
"write_changes_batch",
|
|
92
97
|
# Interfaces
|
|
93
98
|
"CatalogInterface",
|
|
94
99
|
"ReadOnlyCatalogInterface",
|
vgi/catalog/catalog_interface.py
CHANGED
|
@@ -422,13 +422,10 @@ class TableInfo(CatalogSchemaObject, ArrowSerializableDataclass):
|
|
|
422
422
|
check_constraints: SQL CHECK constraint expressions.
|
|
423
423
|
primary_key_constraints: Column-index groups forming the primary key.
|
|
424
424
|
foreign_key_constraints: Serialized foreign-key constraint specs.
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
INSERT/UPDATE/DELETE ... RETURNING at plan time with a
|
|
430
|
-
BinderException. Workers that can emit the affected rows from their
|
|
431
|
-
write functions must opt in by setting this to True.
|
|
425
|
+
write_result_modes: Maximum result mode supported for each writable
|
|
426
|
+
operation. Keys are ``insert``, ``update``, and ``delete``; a missing
|
|
427
|
+
key means the operation is unsupported. Values are ordered
|
|
428
|
+
``count < rows < changes`` and promise every lower mode.
|
|
432
429
|
supports_column_statistics: Statistics capability flag — indicates this
|
|
433
430
|
table can provide column statistics.
|
|
434
431
|
scan_function: Optional inlined function-discovery result. When
|
|
@@ -503,10 +500,9 @@ class TableInfo(CatalogSchemaObject, ArrowSerializableDataclass):
|
|
|
503
500
|
)
|
|
504
501
|
foreign_key_constraints: Annotated[list[bytes], ArrowType(pa.list_(pa.binary()))] = field(default_factory=list)
|
|
505
502
|
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
supports_returning: bool = False
|
|
503
|
+
write_result_modes: Annotated[dict[str, str], ArrowType(pa.map_(pa.string(), pa.string()))] = field(
|
|
504
|
+
default_factory=dict
|
|
505
|
+
)
|
|
510
506
|
|
|
511
507
|
supports_column_statistics: bool = False
|
|
512
508
|
|
vgi/catalog/descriptors.py
CHANGED
|
@@ -41,6 +41,7 @@ from vgi.catalog.catalog_interface import (
|
|
|
41
41
|
from vgi.invocation import BindResponse, FunctionType
|
|
42
42
|
from vgi.metadata import CatalogFunctionType
|
|
43
43
|
from vgi.schema_path import SchemaPath, schema_path_display, schema_path_key
|
|
44
|
+
from vgi.write_results import validate_write_result_modes
|
|
44
45
|
|
|
45
46
|
if TYPE_CHECKING:
|
|
46
47
|
from vgi.function import Function
|
|
@@ -268,6 +269,9 @@ class Table:
|
|
|
268
269
|
delete_function: [`TableInOutGenerator`][] class backing DELETE. Requires
|
|
269
270
|
a scan ``function`` to provide row IDs. ``None`` means DELETE is
|
|
270
271
|
unsupported.
|
|
272
|
+
write_result_modes: Maximum result mode for each configured write
|
|
273
|
+
function. Omitted operations default to ``count``. Modes are ordered
|
|
274
|
+
``count < rows < changes``.
|
|
271
275
|
not_null: Tuple of column names with NOT NULL constraints.
|
|
272
276
|
unique: Tuple of column name tuples for UNIQUE constraints.
|
|
273
277
|
check: Tuple of SQL expressions for CHECK constraints.
|
|
@@ -332,6 +336,7 @@ class Table:
|
|
|
332
336
|
insert_function: type[TableInOutGenerator[Any, Any]] | None = None
|
|
333
337
|
update_function: type[TableInOutGenerator[Any, Any]] | None = None
|
|
334
338
|
delete_function: type[TableInOutGenerator[Any, Any]] | None = None
|
|
339
|
+
write_result_modes: dict[str, str] = field(default_factory=dict)
|
|
335
340
|
not_null: tuple[str, ...] = ()
|
|
336
341
|
unique: tuple[tuple[str, ...], ...] = ()
|
|
337
342
|
check: tuple[str, ...] = ()
|
|
@@ -388,6 +393,19 @@ class Table:
|
|
|
388
393
|
f"override or leave inline_bind=False."
|
|
389
394
|
)
|
|
390
395
|
|
|
396
|
+
modes = validate_write_result_modes(self.write_result_modes)
|
|
397
|
+
functions = {
|
|
398
|
+
"insert": self.insert_function,
|
|
399
|
+
"update": self.update_function,
|
|
400
|
+
"delete": self.delete_function,
|
|
401
|
+
}
|
|
402
|
+
for operation in modes:
|
|
403
|
+
if functions[operation] is None:
|
|
404
|
+
raise ValueError(
|
|
405
|
+
f"Table '{self.name}': write_result_modes declares {operation!r} "
|
|
406
|
+
f"without a corresponding {operation}_function"
|
|
407
|
+
)
|
|
408
|
+
|
|
391
409
|
# Resolve columns to validate constraints
|
|
392
410
|
resolved = self._get_resolved_columns()
|
|
393
411
|
column_names = {f.name for f in resolved}
|
|
@@ -650,9 +668,15 @@ class Table:
|
|
|
650
668
|
check_constraints=list(self.check),
|
|
651
669
|
primary_key_constraints=self._resolve_primary_key_indices(),
|
|
652
670
|
foreign_key_constraints=self._serialize_foreign_keys(schema_path),
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
671
|
+
write_result_modes={
|
|
672
|
+
operation: self.write_result_modes.get(operation, "count")
|
|
673
|
+
for operation, function in (
|
|
674
|
+
("insert", self.insert_function),
|
|
675
|
+
("update", self.update_function),
|
|
676
|
+
("delete", self.delete_function),
|
|
677
|
+
)
|
|
678
|
+
if function is not None
|
|
679
|
+
},
|
|
656
680
|
supports_column_statistics=bool(self.statistics),
|
|
657
681
|
comment=self.comment,
|
|
658
682
|
tags=dict(self.tags),
|