vgi-python 0.47.0__py3-none-any.whl → 0.48.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vgi/__init__.py +6 -0
- vgi/_test_fixtures/accumulate/worker.py +3 -0
- vgi/_test_fixtures/attach_options.py +2 -0
- vgi/_test_fixtures/bad_protocol.py +1 -1
- vgi/_test_fixtures/catalog.py +79 -28
- vgi/_test_fixtures/catalog_contents.py +3 -0
- vgi/_test_fixtures/constraint_facts.py +45 -0
- vgi/_test_fixtures/http_server.py +10 -11
- vgi/_test_fixtures/narrow_bind/worker.py +1 -3
- vgi/_test_fixtures/projection_repro/worker.py +12 -3
- vgi/_test_fixtures/schema_reconcile/worker.py +19 -3
- vgi/_test_fixtures/simple_writable.py +4 -5
- vgi/_test_fixtures/table/catalog_scans.py +3 -0
- vgi/_test_fixtures/table/splits.py +6 -0
- vgi/_test_fixtures/ticket_probe.py +3 -0
- vgi/_test_fixtures/versioned.py +2 -0
- vgi/_test_fixtures/versioned_tables.py +3 -3
- vgi/_test_fixtures/worker.py +199 -132
- vgi/_test_fixtures/writable/generic.py +2 -2
- vgi/_test_fixtures/writable/table.py +2 -3
- vgi/_test_fixtures/writable/worker.py +89 -123
- vgi/arguments.py +94 -39
- vgi/attach_capabilities.py +104 -0
- vgi/attach_ticket.py +1 -1
- vgi/catalog/__init__.py +67 -0
- vgi/catalog/capabilities.py +184 -0
- vgi/catalog/catalog_interface.py +352 -137
- vgi/catalog/column_metadata.py +109 -0
- vgi/catalog/constraints.py +457 -0
- vgi/catalog/contents_codec.py +2 -1
- vgi/catalog/dependencies.py +504 -0
- vgi/catalog/descriptors.py +86 -104
- vgi/catalog/evaluation_context.py +105 -0
- vgi/catalog/indexes.py +92 -0
- vgi/catalog/layout.py +124 -0
- vgi/catalog/macros.py +352 -0
- vgi/catalog/metadata_changes.py +88 -0
- vgi/catalog/options.py +219 -0
- vgi/catalog/query.py +383 -0
- vgi/catalog/snapshot.py +3 -1
- vgi/catalog/sql.py +165 -0
- vgi/client/catalog_mixin.py +372 -55
- vgi/client/cli_table.py +61 -19
- vgi/client/cli_utils.py +36 -4
- vgi/client/client.py +71 -3
- vgi/column_path.py +244 -0
- vgi/exceptions.py +12 -0
- vgi/http/vgi-client.js +8 -262
- vgi/invocation.py +62 -2
- vgi/logical_types.py +267 -0
- vgi/meta_worker.py +43 -0
- vgi/protocol.py +308 -81
- vgi/protocol_version.txt +1 -1
- vgi/query_execution.py +462 -0
- vgi/query_parameters.py +173 -0
- vgi/query_provider.py +127 -0
- vgi/rpc_server.py +3 -3
- vgi/schema_path.py +9 -0
- vgi/secret_protocol.py +1 -1
- vgi/table_function.py +36 -2
- vgi/transactor/protocol.py +6 -4
- vgi/typed_values.py +135 -0
- vgi/worker.py +1053 -81
- {vgi_python-0.47.0.dist-info → vgi_python-0.48.0.dist-info}/METADATA +2 -2
- {vgi_python-0.47.0.dist-info → vgi_python-0.48.0.dist-info}/RECORD +68 -48
- {vgi_python-0.47.0.dist-info → vgi_python-0.48.0.dist-info}/WHEEL +1 -1
- {vgi_python-0.47.0.dist-info → vgi_python-0.48.0.dist-info}/entry_points.txt +0 -0
- {vgi_python-0.47.0.dist-info → vgi_python-0.48.0.dist-info}/licenses/LICENSE +0 -0
vgi/__init__.py
CHANGED
|
@@ -79,6 +79,7 @@ from vgi.filter_v2_builder import (
|
|
|
79
79
|
type_payload,
|
|
80
80
|
value_payload,
|
|
81
81
|
)
|
|
82
|
+
from vgi.invocation import BindRepeatability
|
|
82
83
|
from vgi.metadata import (
|
|
83
84
|
ArgumentMonotonicity,
|
|
84
85
|
CatalogFunctionType,
|
|
@@ -99,6 +100,8 @@ from vgi.metadata import (
|
|
|
99
100
|
from vgi.protocol import (
|
|
100
101
|
BindRequest,
|
|
101
102
|
InitRequest,
|
|
103
|
+
TableFunctionProjectionRequest,
|
|
104
|
+
TableFunctionProjectionResponse,
|
|
102
105
|
VgiOutputCollector,
|
|
103
106
|
)
|
|
104
107
|
from vgi.scalar_function import (
|
|
@@ -132,6 +135,8 @@ from vgi.worker import Worker
|
|
|
132
135
|
from vgi.write_results import WriteOperation, WriteResultMode, write_changes_batch, write_result_schema
|
|
133
136
|
|
|
134
137
|
__all__ = [
|
|
138
|
+
"TableFunctionProjectionRequest",
|
|
139
|
+
"TableFunctionProjectionResponse",
|
|
135
140
|
"AggregateBindParams",
|
|
136
141
|
"AggregateFunction",
|
|
137
142
|
"WriteOperation",
|
|
@@ -151,6 +156,7 @@ __all__ = [
|
|
|
151
156
|
"argument_specs_to_schema",
|
|
152
157
|
"parameter_default_values_from_specs",
|
|
153
158
|
"BindRequest",
|
|
159
|
+
"BindRepeatability",
|
|
154
160
|
"ColumnBounds",
|
|
155
161
|
"ColumnRefNode",
|
|
156
162
|
"ComparisonNode",
|
|
@@ -69,6 +69,7 @@ from vgi.catalog.catalog_interface import AttachOpaqueData, CatalogAttachResult,
|
|
|
69
69
|
from vgi.function_storage import BoundStorage, FunctionStorage
|
|
70
70
|
from vgi.invocation import BindResponse
|
|
71
71
|
from vgi.metadata import FunctionExample
|
|
72
|
+
from vgi.protocol import ClientCapabilities
|
|
72
73
|
from vgi.table_buffering_function import TableBufferingFunction, TableBufferingParams
|
|
73
74
|
from vgi.table_function import (
|
|
74
75
|
BindParams,
|
|
@@ -734,6 +735,7 @@ class AccumulateCatalog(ReadOnlyCatalogInterface):
|
|
|
734
735
|
options: dict[str, Any],
|
|
735
736
|
data_version_spec: str | None,
|
|
736
737
|
implementation_version: str | None,
|
|
738
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
737
739
|
ctx: CallContext | None = None,
|
|
738
740
|
) -> CatalogAttachResult:
|
|
739
741
|
"""Attach, minting a random per-ATTACH storage scope id."""
|
|
@@ -742,6 +744,7 @@ class AccumulateCatalog(ReadOnlyCatalogInterface):
|
|
|
742
744
|
options=options,
|
|
743
745
|
data_version_spec=data_version_spec,
|
|
744
746
|
implementation_version=implementation_version,
|
|
747
|
+
client_capabilities=client_capabilities,
|
|
745
748
|
ctx=ctx,
|
|
746
749
|
)
|
|
747
750
|
return dataclasses.replace(
|
|
@@ -46,6 +46,7 @@ from vgi.catalog.catalog_interface import (
|
|
|
46
46
|
)
|
|
47
47
|
from vgi.catalog.descriptors import Catalog, Schema
|
|
48
48
|
from vgi.invocation import BindResponse
|
|
49
|
+
from vgi.protocol import ClientCapabilities
|
|
49
50
|
from vgi.schema_utils import schema
|
|
50
51
|
from vgi.table_function import BindParams, ProcessParams, TableFunctionGenerator, init_single_worker
|
|
51
52
|
from vgi.worker import Worker
|
|
@@ -260,6 +261,7 @@ class AttachOptionsCatalog(ReadOnlyCatalogInterface):
|
|
|
260
261
|
options: dict[str, Any],
|
|
261
262
|
data_version_spec: str | None,
|
|
262
263
|
implementation_version: str | None,
|
|
264
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
263
265
|
ctx: CallContext | None = None,
|
|
264
266
|
) -> CatalogAttachResult:
|
|
265
267
|
del data_version_spec, implementation_version, ctx
|
|
@@ -51,7 +51,7 @@ class BadProtocol(VgiProtocol):
|
|
|
51
51
|
# finds no such protocol hosted, and the caller gets
|
|
52
52
|
# ``ProtocolNotSupportedError`` before the version gate it came to test
|
|
53
53
|
# ever runs.
|
|
54
|
-
protocol_name: ClassVar[str] = "vgi.
|
|
54
|
+
protocol_name: ClassVar[str] = "vgi.v3"
|
|
55
55
|
protocol_version: ClassVar[str] = BAD_PROTOCOL_VERSION
|
|
56
56
|
|
|
57
57
|
|
vgi/_test_fixtures/catalog.py
CHANGED
|
@@ -8,13 +8,13 @@ be used for testing and as a reference implementation.
|
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
import dataclasses
|
|
11
12
|
import uuid
|
|
12
13
|
from collections.abc import Sequence
|
|
13
14
|
from dataclasses import dataclass, field
|
|
14
15
|
from typing import TYPE_CHECKING, Any, Literal, overload
|
|
15
16
|
|
|
16
17
|
if TYPE_CHECKING:
|
|
17
|
-
import pyarrow as pa
|
|
18
18
|
from vgi_rpc.rpc import CallContext
|
|
19
19
|
|
|
20
20
|
from vgi._redact import short_hash
|
|
@@ -35,7 +35,11 @@ from vgi.catalog import (
|
|
|
35
35
|
TransactionOpaqueData,
|
|
36
36
|
ViewInfo,
|
|
37
37
|
)
|
|
38
|
+
from vgi.catalog.layout import TableLayout
|
|
39
|
+
from vgi.catalog.macros import MacroOverload
|
|
40
|
+
from vgi.protocol import ClientCapabilities
|
|
38
41
|
from vgi.schema_path import SchemaKey, schema_path_key
|
|
42
|
+
from vgi.typed_values import TypedValueBatch
|
|
39
43
|
from vgi.worker import Worker
|
|
40
44
|
|
|
41
45
|
|
|
@@ -154,6 +158,7 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
154
158
|
options: dict[str, Any],
|
|
155
159
|
data_version_spec: str | None,
|
|
156
160
|
implementation_version: str | None,
|
|
161
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
157
162
|
ctx: CallContext | None = None,
|
|
158
163
|
) -> CatalogAttachResult:
|
|
159
164
|
"""Attach to a catalog with the given name.
|
|
@@ -354,19 +359,27 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
354
359
|
on_conflict: OnConflict = OnConflict.ERROR,
|
|
355
360
|
comment: str | None,
|
|
356
361
|
tags: dict[str, str],
|
|
362
|
+
options: TypedValueBatch | None = None,
|
|
357
363
|
) -> None:
|
|
358
364
|
"""Create a new schema."""
|
|
365
|
+
if options is not None:
|
|
366
|
+
raise NotImplementedError("Schema options are not supported by this fixture")
|
|
359
367
|
catalog = self._get_catalog(attach_opaque_data)
|
|
360
368
|
path_key = schema_path_key(path)
|
|
361
369
|
if path_key in catalog.schemas:
|
|
362
|
-
|
|
363
|
-
|
|
370
|
+
if on_conflict == OnConflict.IGNORE:
|
|
371
|
+
return
|
|
372
|
+
if on_conflict != OnConflict.REPLACE:
|
|
373
|
+
msg = f"Schema {path!r} already exists"
|
|
374
|
+
raise ValueError(msg)
|
|
375
|
+
if len(path) > 1 and schema_path_key(path[:-1]) not in catalog.schemas:
|
|
376
|
+
raise ValueError("Parent schema does not exist")
|
|
364
377
|
catalog.schemas[path_key] = SchemaData(
|
|
365
378
|
info=SchemaInfo(
|
|
366
379
|
attach_opaque_data=attach_opaque_data,
|
|
367
380
|
path=list(path),
|
|
368
381
|
comment=comment,
|
|
369
|
-
tags=tags,
|
|
382
|
+
tags=dict(tags),
|
|
370
383
|
)
|
|
371
384
|
)
|
|
372
385
|
self._increment_version(attach_opaque_data)
|
|
@@ -507,13 +520,18 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
507
520
|
name: str,
|
|
508
521
|
columns: SerializedSchema,
|
|
509
522
|
on_conflict: OnConflict,
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
523
|
+
constraint_definitions: list[bytes],
|
|
524
|
+
comment: str | None = None,
|
|
525
|
+
tags: dict[str, str] | None = None,
|
|
526
|
+
options: TypedValueBatch | None = None,
|
|
527
|
+
layout: TableLayout | None = None,
|
|
515
528
|
) -> None:
|
|
516
529
|
"""Create a new table."""
|
|
530
|
+
if constraint_definitions:
|
|
531
|
+
raise NotImplementedError("this metadata-only catalog does not enforce canonical constraints")
|
|
532
|
+
if any(value is not None for value in (comment, tags, options, layout)):
|
|
533
|
+
raise NotImplementedError("table creation metadata/options/layout profiles are not supported")
|
|
534
|
+
|
|
517
535
|
schema_data = self._get_schema(attach_opaque_data, schema_path)
|
|
518
536
|
if name in schema_data.tables:
|
|
519
537
|
if on_conflict == OnConflict.ERROR:
|
|
@@ -528,9 +546,7 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
528
546
|
name=name,
|
|
529
547
|
schema_path=schema_path,
|
|
530
548
|
columns=columns,
|
|
531
|
-
|
|
532
|
-
unique_constraints=unique_constraints,
|
|
533
|
-
check_constraints=check_constraints,
|
|
549
|
+
constraints=[],
|
|
534
550
|
comment=None,
|
|
535
551
|
tags={},
|
|
536
552
|
)
|
|
@@ -582,9 +598,8 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
582
598
|
name=old_info.name,
|
|
583
599
|
schema_path=old_info.schema_path,
|
|
584
600
|
columns=old_info.columns,
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
check_constraints=old_info.check_constraints,
|
|
601
|
+
constraints=old_info.constraints,
|
|
602
|
+
definition_revision=old_info.definition_revision,
|
|
588
603
|
comment=comment,
|
|
589
604
|
tags=old_info.tags,
|
|
590
605
|
)
|
|
@@ -619,9 +634,8 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
619
634
|
name=new_name,
|
|
620
635
|
schema_path=old_info.schema_path,
|
|
621
636
|
columns=old_info.columns,
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
check_constraints=old_info.check_constraints,
|
|
637
|
+
constraints=old_info.constraints,
|
|
638
|
+
definition_revision=old_info.definition_revision,
|
|
625
639
|
comment=old_info.comment,
|
|
626
640
|
tags=old_info.tags,
|
|
627
641
|
)
|
|
@@ -637,8 +651,15 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
637
651
|
name: str,
|
|
638
652
|
definition: str,
|
|
639
653
|
on_conflict: OnConflict,
|
|
654
|
+
aliases: list[str] | None = None,
|
|
655
|
+
security: str = "regular",
|
|
656
|
+
comment: str | None = None,
|
|
657
|
+
tags: dict[str, str] | None = None,
|
|
658
|
+
column_comments: dict[str, str] | None = None,
|
|
640
659
|
) -> None:
|
|
641
660
|
"""Create a new view."""
|
|
661
|
+
if security != "regular":
|
|
662
|
+
raise NotImplementedError("view security mode is unsupported by this provider")
|
|
642
663
|
schema_data = self._get_schema(attach_opaque_data, schema_path)
|
|
643
664
|
if name in schema_data.views:
|
|
644
665
|
if on_conflict == OnConflict.ERROR:
|
|
@@ -653,8 +674,10 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
653
674
|
name=name,
|
|
654
675
|
schema_path=schema_path,
|
|
655
676
|
definition=definition,
|
|
656
|
-
|
|
657
|
-
|
|
677
|
+
aliases=list(aliases or []),
|
|
678
|
+
comment=comment,
|
|
679
|
+
tags=dict(tags or {}),
|
|
680
|
+
column_comments=dict(column_comments or {}),
|
|
658
681
|
)
|
|
659
682
|
)
|
|
660
683
|
self._increment_version(attach_opaque_data)
|
|
@@ -707,6 +730,9 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
707
730
|
name=new_name,
|
|
708
731
|
schema_path=old_info.schema_path,
|
|
709
732
|
definition=old_info.definition,
|
|
733
|
+
aliases=list(old_info.aliases),
|
|
734
|
+
security=old_info.security,
|
|
735
|
+
column_comments=dict(old_info.column_comments),
|
|
710
736
|
comment=old_info.comment,
|
|
711
737
|
tags=old_info.tags,
|
|
712
738
|
)
|
|
@@ -738,12 +764,43 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
738
764
|
name=old_info.name,
|
|
739
765
|
schema_path=old_info.schema_path,
|
|
740
766
|
definition=old_info.definition,
|
|
767
|
+
aliases=list(old_info.aliases),
|
|
768
|
+
security=old_info.security,
|
|
769
|
+
column_comments=dict(old_info.column_comments),
|
|
741
770
|
comment=comment,
|
|
742
771
|
tags=old_info.tags,
|
|
743
772
|
)
|
|
744
773
|
)
|
|
745
774
|
self._increment_version(attach_opaque_data)
|
|
746
775
|
|
|
776
|
+
def view_column_comment_set(
|
|
777
|
+
self,
|
|
778
|
+
*,
|
|
779
|
+
attach_opaque_data: AttachOpaqueData,
|
|
780
|
+
transaction_opaque_data: TransactionOpaqueData | None,
|
|
781
|
+
schema_path: list[str],
|
|
782
|
+
name: str,
|
|
783
|
+
column_path: list[str],
|
|
784
|
+
comment: str | None,
|
|
785
|
+
ignore_not_found: bool,
|
|
786
|
+
) -> None:
|
|
787
|
+
"""Set or clear the comment keyed by a raw view output name."""
|
|
788
|
+
if len(column_path) != 1 or not column_path[0] or "\0" in column_path[0]:
|
|
789
|
+
raise ValueError("view column comments require one raw output identifier")
|
|
790
|
+
schema_data = self._get_schema(attach_opaque_data, schema_path)
|
|
791
|
+
view = schema_data.views.get(name)
|
|
792
|
+
if view is None:
|
|
793
|
+
if ignore_not_found:
|
|
794
|
+
return
|
|
795
|
+
raise ValueError(f"View {name!r} not found in schema {schema_path!r}")
|
|
796
|
+
comments = dict(view.info.column_comments)
|
|
797
|
+
if comment is None:
|
|
798
|
+
comments.pop(column_path[0], None)
|
|
799
|
+
else:
|
|
800
|
+
comments[column_path[0]] = comment
|
|
801
|
+
schema_data.views[name] = ViewData(info=dataclasses.replace(view.info, column_comments=comments))
|
|
802
|
+
self._increment_version(attach_opaque_data)
|
|
803
|
+
|
|
747
804
|
def macro_create(
|
|
748
805
|
self,
|
|
749
806
|
*,
|
|
@@ -752,11 +809,8 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
752
809
|
schema_path: list[str],
|
|
753
810
|
name: str,
|
|
754
811
|
macro_type: MacroType,
|
|
755
|
-
|
|
756
|
-
definition: str,
|
|
812
|
+
overloads: list[MacroOverload],
|
|
757
813
|
on_conflict: OnConflict,
|
|
758
|
-
parameter_default_values: pa.RecordBatch | None = None,
|
|
759
|
-
arguments_schema: pa.Schema | None = None,
|
|
760
814
|
) -> None:
|
|
761
815
|
"""Create a new macro."""
|
|
762
816
|
schema_data = self._get_schema(attach_opaque_data, schema_path)
|
|
@@ -773,12 +827,9 @@ class InMemoryCatalog(CatalogInterface):
|
|
|
773
827
|
name=name,
|
|
774
828
|
schema_path=schema_path,
|
|
775
829
|
macro_type=macro_type,
|
|
776
|
-
|
|
777
|
-
parameter_default_values=parameter_default_values,
|
|
778
|
-
definition=definition,
|
|
830
|
+
overloads=overloads,
|
|
779
831
|
comment=None,
|
|
780
832
|
tags={},
|
|
781
|
-
arguments_schema=arguments_schema,
|
|
782
833
|
)
|
|
783
834
|
)
|
|
784
835
|
self._increment_version(attach_opaque_data)
|
|
@@ -78,6 +78,7 @@ from vgi.catalog import (
|
|
|
78
78
|
Table,
|
|
79
79
|
View,
|
|
80
80
|
)
|
|
81
|
+
from vgi.protocol import ClientCapabilities
|
|
81
82
|
from vgi.worker import Worker
|
|
82
83
|
|
|
83
84
|
CATALOG_PROBE = "contents_probe"
|
|
@@ -232,6 +233,7 @@ class _PrivateMemoryCatalog(InMemoryCatalog):
|
|
|
232
233
|
options: dict[str, Any],
|
|
233
234
|
data_version_spec: str | None,
|
|
234
235
|
implementation_version: str | None,
|
|
236
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
235
237
|
ctx: Any = None,
|
|
236
238
|
) -> CatalogAttachResult:
|
|
237
239
|
"""Attach a fresh private catalog, advertising ``catalog_contents``."""
|
|
@@ -248,6 +250,7 @@ class _PrivateMemoryCatalog(InMemoryCatalog):
|
|
|
248
250
|
options=options,
|
|
249
251
|
data_version_spec=data_version_spec,
|
|
250
252
|
implementation_version=implementation_version,
|
|
253
|
+
client_capabilities=client_capabilities,
|
|
251
254
|
ctx=ctx,
|
|
252
255
|
)
|
|
253
256
|
return replace(
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright 2025, 2026 Query Farm LLC - https://query.farm
|
|
2
|
+
|
|
3
|
+
"""Explicit facts for fixed fixture tables; never an inference from Arrow flags.
|
|
4
|
+
|
|
5
|
+
Callers own the proof: immutable rows are checked before publication, or every
|
|
6
|
+
fixture storage write validates values. Keys are fixed provider-owned identifiers
|
|
7
|
+
for tables whose definitions cannot be dropped/recreated through their API.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from vgi.catalog.constraints import (
|
|
13
|
+
ConstraintDefinition,
|
|
14
|
+
ConstraintEnforcement,
|
|
15
|
+
ConstraintInfo,
|
|
16
|
+
ConstraintKind,
|
|
17
|
+
ConstraintValidation,
|
|
18
|
+
)
|
|
19
|
+
from vgi.catalog.dependencies import CatalogObjectRef
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def fixed_fact(
|
|
23
|
+
table: str, path: list[str], key: str, definition: ConstraintDefinition, **kwargs: Any
|
|
24
|
+
) -> ConstraintInfo:
|
|
25
|
+
"""Attach an explicit fixture definition to its stable provider-owned key."""
|
|
26
|
+
return ConstraintInfo(
|
|
27
|
+
ref=CatalogObjectRef(kind="constraint", schema_path=path, owner_table=table, name=key),
|
|
28
|
+
definition=definition,
|
|
29
|
+
enforcement=ConstraintEnforcement.ENFORCED,
|
|
30
|
+
validation=ConstraintValidation.ALL_ROWS,
|
|
31
|
+
**kwargs,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def fixed_not_null(table: str, path: list[str], columns: tuple[tuple[str, ...], ...]) -> tuple[ConstraintInfo, ...]:
|
|
36
|
+
"""Encode explicit NOT_NULL facts that the fixed fixture actually enforces."""
|
|
37
|
+
return tuple(
|
|
38
|
+
fixed_fact(
|
|
39
|
+
table,
|
|
40
|
+
path,
|
|
41
|
+
f"not-null-{index}",
|
|
42
|
+
ConstraintDefinition(kind=ConstraintKind.NOT_NULL, column_paths=[list(column)]),
|
|
43
|
+
)
|
|
44
|
+
for index, column in enumerate(columns)
|
|
45
|
+
)
|
|
@@ -310,16 +310,6 @@ def main() -> None:
|
|
|
310
310
|
)
|
|
311
311
|
sys.stderr.flush()
|
|
312
312
|
|
|
313
|
-
import socket
|
|
314
|
-
|
|
315
|
-
if port == 0:
|
|
316
|
-
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
317
|
-
s.bind((host, 0))
|
|
318
|
-
port = int(s.getsockname()[1])
|
|
319
|
-
|
|
320
|
-
if demo_storage:
|
|
321
|
-
demo_blob_storage.set_base_url(f"http://{host}:{port}")
|
|
322
|
-
|
|
323
313
|
from vgi.serve import _resolve_authenticate, _resolve_oauth_resource_metadata
|
|
324
314
|
|
|
325
315
|
authenticate = _resolve_authenticate()
|
|
@@ -487,6 +477,15 @@ def main() -> None:
|
|
|
487
477
|
# we serve the Falcon app directly.
|
|
488
478
|
serving_app: Any = wsgi_app
|
|
489
479
|
|
|
480
|
+
# Keep the selected socket bound while publishing its port. Selecting
|
|
481
|
+
# and releasing a free port first lets another test worker take it.
|
|
482
|
+
# Auto-selection has always used IPv4; retain one listener for port 0.
|
|
483
|
+
http_server = waitress.create_server(serving_app, host=host, port=port, ipv6=port != 0)
|
|
484
|
+
if port == 0:
|
|
485
|
+
port = int(http_server.effective_port)
|
|
486
|
+
if demo_storage:
|
|
487
|
+
demo_blob_storage.set_base_url(f"http://{host}:{port}")
|
|
488
|
+
|
|
490
489
|
if port_file is not None:
|
|
491
490
|
# Atomic side-channel publication so test harnesses can watch
|
|
492
491
|
# for the file without racing a partial write. Same helper the
|
|
@@ -498,7 +497,7 @@ def main() -> None:
|
|
|
498
497
|
print(f"PORT:{port}", flush=True)
|
|
499
498
|
sys.stderr.write(f"Serving ExampleWorker on http://{host}:{port}{prefix}\n")
|
|
500
499
|
sys.stderr.flush()
|
|
501
|
-
|
|
500
|
+
http_server.run()
|
|
502
501
|
|
|
503
502
|
app()
|
|
504
503
|
|
|
@@ -156,9 +156,7 @@ class NarrowBindCatalog(ReadOnlyCatalogInterface):
|
|
|
156
156
|
name=table_name,
|
|
157
157
|
schema_path=["main"],
|
|
158
158
|
columns=SerializedSchema(_serialize_schema(_TABLE_SCHEMA)),
|
|
159
|
-
|
|
160
|
-
unique_constraints=[],
|
|
161
|
-
check_constraints=[],
|
|
159
|
+
constraints=[],
|
|
162
160
|
)
|
|
163
161
|
|
|
164
162
|
def schemas(
|
|
@@ -42,8 +42,10 @@ from vgi_rpc import ArrowSerializableDataclass
|
|
|
42
42
|
from vgi_rpc.rpc import OutputCollector
|
|
43
43
|
|
|
44
44
|
from vgi import Worker
|
|
45
|
+
from vgi._test_fixtures.constraint_facts import fixed_not_null
|
|
45
46
|
from vgi.arguments import Arg
|
|
46
47
|
from vgi.catalog import Catalog, Schema
|
|
48
|
+
from vgi.catalog.capabilities import CONSTRAINT_RECORDS_READ, CatalogCapabilities
|
|
47
49
|
from vgi.catalog.catalog_interface import (
|
|
48
50
|
AttachOpaqueData,
|
|
49
51
|
ReadOnlyCatalogInterface,
|
|
@@ -357,6 +359,10 @@ class ProjReproCatalog(ReadOnlyCatalogInterface):
|
|
|
357
359
|
catalog = _CATALOG
|
|
358
360
|
catalog_name = CATALOG_NAME
|
|
359
361
|
|
|
362
|
+
def catalog_capabilities(self, *, attach_opaque_data: AttachOpaqueData) -> CatalogCapabilities:
|
|
363
|
+
"""The generator always supplies these four fields and exposes no writes."""
|
|
364
|
+
return CatalogCapabilities(read=[CONSTRAINT_RECORDS_READ])
|
|
365
|
+
|
|
360
366
|
def _info(self, table_name: str) -> TableInfo:
|
|
361
367
|
return TableInfo(
|
|
362
368
|
comment=f"reproducer table -> {_TABLE_NAMES[table_name]}",
|
|
@@ -364,9 +370,12 @@ class ProjReproCatalog(ReadOnlyCatalogInterface):
|
|
|
364
370
|
name=table_name,
|
|
365
371
|
schema_path=["main"],
|
|
366
372
|
columns=SerializedSchema(_serialize_schema(WIDE_SCHEMA)),
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
373
|
+
constraints=[
|
|
374
|
+
fact.serialize_to_bytes()
|
|
375
|
+
for fact in fixed_not_null(
|
|
376
|
+
table_name, ["main"], (("topic",), ("partition",), ("offset",), ("headers",))
|
|
377
|
+
)
|
|
378
|
+
],
|
|
370
379
|
)
|
|
371
380
|
|
|
372
381
|
def schemas(
|
|
@@ -37,7 +37,9 @@ import pyarrow as pa
|
|
|
37
37
|
from vgi_rpc.rpc import OutputCollector
|
|
38
38
|
|
|
39
39
|
from vgi import Worker
|
|
40
|
+
from vgi._test_fixtures.constraint_facts import fixed_not_null
|
|
40
41
|
from vgi.catalog import Catalog, Schema
|
|
42
|
+
from vgi.catalog.capabilities import CONSTRAINT_ENFORCE, CONSTRAINT_RECORDS_READ, CatalogCapabilities
|
|
41
43
|
from vgi.catalog.catalog_interface import (
|
|
42
44
|
AttachOpaqueData,
|
|
43
45
|
ReadOnlyCatalogInterface,
|
|
@@ -56,6 +58,7 @@ from vgi.table_function import (
|
|
|
56
58
|
TableFunctionGenerator,
|
|
57
59
|
)
|
|
58
60
|
from vgi.table_in_out_function import TableInOutGenerator
|
|
61
|
+
from vgi.typed_values import TypedValueBatch
|
|
59
62
|
|
|
60
63
|
CATALOG_NAME = "schema_reconcile"
|
|
61
64
|
_SCHEMA_NAME = "main"
|
|
@@ -217,6 +220,7 @@ def _all_rows(spec: TableSpec) -> list[tuple[Any, dict[str, Any]]]:
|
|
|
217
220
|
|
|
218
221
|
|
|
219
222
|
def _insert_row(spec: TableSpec, rid: Any, payload: dict[str, Any]) -> None:
|
|
223
|
+
TypedValueBatch(pa.RecordBatch.from_pylist([{**payload, "rowid": rid}], schema=spec.table_schema))
|
|
220
224
|
with _lock, _connect() as conn:
|
|
221
225
|
conn.execute(
|
|
222
226
|
f"INSERT OR REPLACE INTO {spec.storage_table} (rid_blob, payload) VALUES (?, ?)",
|
|
@@ -234,6 +238,7 @@ def _update_row(spec: TableSpec, rid: Any, updates: dict[str, Any]) -> bool:
|
|
|
234
238
|
return False
|
|
235
239
|
payload = pickle.loads(row[0])
|
|
236
240
|
payload.update(updates)
|
|
241
|
+
TypedValueBatch(pa.RecordBatch.from_pylist([{**payload, "rowid": rid}], schema=spec.table_schema))
|
|
237
242
|
conn.execute(
|
|
238
243
|
f"UPDATE {spec.storage_table} SET payload = ? WHERE rid_blob = ?",
|
|
239
244
|
(pickle.dumps(payload), _rid_key(rid)),
|
|
@@ -518,16 +523,27 @@ class SchemaReconcileCatalog(ReadOnlyCatalogInterface):
|
|
|
518
523
|
catalog = _CATALOG
|
|
519
524
|
catalog_name = CATALOG_NAME
|
|
520
525
|
|
|
526
|
+
def catalog_capabilities(self, *, attach_opaque_data: AttachOpaqueData) -> CatalogCapabilities:
|
|
527
|
+
"""Every fixture insert/update checks required values before publication."""
|
|
528
|
+
return CatalogCapabilities(read=[CONSTRAINT_RECORDS_READ], write=[CONSTRAINT_ENFORCE])
|
|
529
|
+
|
|
521
530
|
def _table_info(self, spec: TableSpec) -> TableInfo:
|
|
531
|
+
# Qualify ALL_ROWS for storage created by a previous fixture process,
|
|
532
|
+
# as well as the writes validated by the current implementation.
|
|
533
|
+
for rid, payload in _all_rows(spec):
|
|
534
|
+
TypedValueBatch(pa.RecordBatch.from_pylist([{**payload, "rowid": rid}], schema=spec.table_schema))
|
|
522
535
|
return TableInfo(
|
|
523
536
|
comment=f"Schema-reconcile {spec.name} (rowid type {spec.rowid_field.type})",
|
|
524
537
|
tags={},
|
|
525
538
|
name=spec.name,
|
|
526
539
|
schema_path=[_SCHEMA_NAME],
|
|
527
540
|
columns=SerializedSchema(_serialize_schema(spec.table_schema)),
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
541
|
+
constraints=[
|
|
542
|
+
fact.serialize_to_bytes()
|
|
543
|
+
for fact in fixed_not_null(
|
|
544
|
+
spec.name, [_SCHEMA_NAME], (("id",), ("ts",), ("nested",), ("nested", "a"), ("tags",))
|
|
545
|
+
)
|
|
546
|
+
],
|
|
531
547
|
write_result_modes={"insert": "changes", "update": "changes", "delete": "changes"},
|
|
532
548
|
)
|
|
533
549
|
|
|
@@ -56,6 +56,7 @@ from vgi.catalog import (
|
|
|
56
56
|
)
|
|
57
57
|
from vgi.catalog.descriptors import Catalog, Schema
|
|
58
58
|
from vgi.invocation import BindResponse, GlobalInitResponse
|
|
59
|
+
from vgi.protocol import ClientCapabilities
|
|
59
60
|
from vgi.schema_utils import schema as build_schema
|
|
60
61
|
from vgi.table_function import BindParams, InitParams, ProcessParams, TableFunctionGenerator
|
|
61
62
|
from vgi.table_in_out_function import TableInOutGenerator
|
|
@@ -336,6 +337,7 @@ class SimpleScan(TableFunctionGenerator[None, "_ScanState"]):
|
|
|
336
337
|
state.cursor = len(state.rows)
|
|
337
338
|
arrow_arrays = [pa.array(col, type=state.schema.field(i).type) for i, col in enumerate(col_arrays)]
|
|
338
339
|
out.emit(pa.RecordBatch.from_arrays(arrow_arrays, schema=state.schema))
|
|
340
|
+
out.finish()
|
|
339
341
|
|
|
340
342
|
|
|
341
343
|
@dataclass(kw_only=True)
|
|
@@ -599,6 +601,7 @@ class SimpleWritableCatalog(ReadOnlyCatalogInterface):
|
|
|
599
601
|
options: dict[str, Any],
|
|
600
602
|
data_version_spec: str | None,
|
|
601
603
|
implementation_version: str | None,
|
|
604
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
602
605
|
ctx: Any | None = None,
|
|
603
606
|
) -> CatalogAttachResult:
|
|
604
607
|
del options, data_version_spec, implementation_version, ctx
|
|
@@ -653,11 +656,7 @@ class SimpleWritableCatalog(ReadOnlyCatalogInterface):
|
|
|
653
656
|
name=name,
|
|
654
657
|
schema_path=schema_path,
|
|
655
658
|
columns=SerializedSchema(full.serialize().to_pybytes()),
|
|
656
|
-
|
|
657
|
-
unique_constraints=[],
|
|
658
|
-
check_constraints=[],
|
|
659
|
-
primary_key_constraints=[],
|
|
660
|
-
foreign_key_constraints=[],
|
|
659
|
+
constraints=[],
|
|
661
660
|
write_result_modes={
|
|
662
661
|
operation: _table_result_mode(name)
|
|
663
662
|
for operation, supported in (("insert", True), ("update", ud), ("delete", ud))
|
|
@@ -18,6 +18,8 @@ from vgi.table_function import (
|
|
|
18
18
|
init_single_worker,
|
|
19
19
|
)
|
|
20
20
|
|
|
21
|
+
STATIC_CATALOG_ROWS: dict[str, pa.Table] = {}
|
|
22
|
+
|
|
21
23
|
|
|
22
24
|
def _static_scan_function(
|
|
23
25
|
func_name: str,
|
|
@@ -31,6 +33,7 @@ def _static_scan_function(
|
|
|
31
33
|
return a fixed dataset. Each generated class is decorated with
|
|
32
34
|
``@init_single_worker`` and has a unique ``Meta.name``.
|
|
33
35
|
"""
|
|
36
|
+
STATIC_CATALOG_ROWS[func_name] = pa.Table.from_pydict(data, schema=output_schema)
|
|
34
37
|
|
|
35
38
|
@init_single_worker
|
|
36
39
|
class StaticScanFunction(TableFunctionGenerator[_EmptyArgs, _OneShotState]):
|
|
@@ -1113,6 +1113,12 @@ def _render_filters_canonical(filters: PushdownFilters | None) -> str:
|
|
|
1113
1113
|
for c in columns:
|
|
1114
1114
|
bounds = filters.get_column_bounds(c)
|
|
1115
1115
|
if bounds is None:
|
|
1116
|
+
# DuckDB 2.0 can send an exact IN set without redundant range
|
|
1117
|
+
# predicates. Render its integer envelope in the same canonical form.
|
|
1118
|
+
values = filters.get_column_values(c)
|
|
1119
|
+
discrete = [] if values is None else [_bound_as_int(value) for value in values if value.is_valid]
|
|
1120
|
+
if discrete:
|
|
1121
|
+
parts.extend((f"{c}>={min(discrete)}", f"{c}<={max(discrete)}"))
|
|
1116
1122
|
continue
|
|
1117
1123
|
# Normalized to INCLUSIVE integer bounds, because that is the only form
|
|
1118
1124
|
# every SDK can produce: Rust's ColumnBounds is integer-coerced and
|
|
@@ -39,6 +39,7 @@ from vgi_rpc.rpc import CallContext, OutputCollector
|
|
|
39
39
|
from vgi.catalog import AttachOpaqueData, Catalog, CatalogAttachResult, ReadOnlyCatalogInterface, Schema, Table
|
|
40
40
|
from vgi.catalog.attach_option import AttachOption
|
|
41
41
|
from vgi.invocation import BindResponse
|
|
42
|
+
from vgi.protocol import ClientCapabilities
|
|
42
43
|
from vgi.table_function import BindParams, ProcessParams, TableFunctionGenerator, init_single_worker
|
|
43
44
|
from vgi.worker import Worker
|
|
44
45
|
|
|
@@ -139,6 +140,7 @@ class TicketProbeCatalog(ReadOnlyCatalogInterface):
|
|
|
139
140
|
options: dict[str, Any],
|
|
140
141
|
data_version_spec: str | None,
|
|
141
142
|
implementation_version: str | None,
|
|
143
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
142
144
|
ctx: CallContext | None = None,
|
|
143
145
|
) -> CatalogAttachResult:
|
|
144
146
|
"""Attach, recording ``region`` and ``sha256(api_key)[:12]`` (never the key)."""
|
|
@@ -147,6 +149,7 @@ class TicketProbeCatalog(ReadOnlyCatalogInterface):
|
|
|
147
149
|
options=options,
|
|
148
150
|
data_version_spec=data_version_spec,
|
|
149
151
|
implementation_version=implementation_version,
|
|
152
|
+
client_capabilities=client_capabilities,
|
|
150
153
|
ctx=ctx,
|
|
151
154
|
)
|
|
152
155
|
lowered = {key.lower(): value for key, value in options.items()}
|
vgi/_test_fixtures/versioned.py
CHANGED
|
@@ -34,6 +34,7 @@ from vgi.catalog import (
|
|
|
34
34
|
TransactionOpaqueData,
|
|
35
35
|
)
|
|
36
36
|
from vgi.catalog.descriptors import Catalog, Schema
|
|
37
|
+
from vgi.protocol import ClientCapabilities
|
|
37
38
|
from vgi.worker import Worker
|
|
38
39
|
|
|
39
40
|
if TYPE_CHECKING:
|
|
@@ -78,6 +79,7 @@ class VersionedCatalog(ReadOnlyCatalogInterface):
|
|
|
78
79
|
options: dict[str, Any],
|
|
79
80
|
data_version_spec: str | None,
|
|
80
81
|
implementation_version: str | None,
|
|
82
|
+
client_capabilities: ClientCapabilities | None = None,
|
|
81
83
|
ctx: CallContext | None = None,
|
|
82
84
|
) -> CatalogAttachResult:
|
|
83
85
|
"""Validate requested versions and pin an HTTP session."""
|