vgi-python 0.41.0__py3-none-any.whl → 0.42.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vgi/_duckdb.py +3 -3
- vgi/_test_fixtures/http_server.py +2 -1
- vgi/_test_fixtures/identity_fixture.py +7 -1
- vgi/client/client.py +80 -37
- vgi/filter_v2.py +29 -1
- vgi/rpc_server.py +56 -3
- vgi/serve.py +73 -2
- {vgi_python-0.41.0.dist-info → vgi_python-0.42.1.dist-info}/METADATA +2 -2
- {vgi_python-0.41.0.dist-info → vgi_python-0.42.1.dist-info}/RECORD +12 -12
- {vgi_python-0.41.0.dist-info → vgi_python-0.42.1.dist-info}/WHEEL +0 -0
- {vgi_python-0.41.0.dist-info → vgi_python-0.42.1.dist-info}/entry_points.txt +0 -0
- {vgi_python-0.41.0.dist-info → vgi_python-0.42.1.dist-info}/licenses/LICENSE +0 -0
vgi/_duckdb.py
CHANGED
|
@@ -7,8 +7,8 @@ engine (expression-filter evaluation, the transactor, the statistics
|
|
|
7
7
|
fixtures) resolves it through this module, which prefers Query Farm's
|
|
8
8
|
``haybarn`` distribution when installed and falls back to stock ``duckdb``.
|
|
9
9
|
|
|
10
|
-
Install one via the extras: ``pip install vgi[haybarn]`` (preferred) or
|
|
11
|
-
``pip install vgi[duckdb]``.
|
|
10
|
+
Install one via the extras: ``pip install 'vgi-python[haybarn]'`` (preferred) or
|
|
11
|
+
``pip install 'vgi-python[duckdb]'``.
|
|
12
12
|
"""
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
@@ -50,7 +50,7 @@ def engine_module() -> ModuleType:
|
|
|
50
50
|
else:
|
|
51
51
|
raise ImportError(
|
|
52
52
|
"No DuckDB-compatible engine is installed. Install 'haybarn' (preferred) "
|
|
53
|
-
"or 'duckdb', e.g. `pip install vgi[haybarn]` or `pip install vgi[duckdb]`."
|
|
53
|
+
"or 'duckdb', e.g. `pip install 'vgi-python[haybarn]'` or `pip install 'vgi-python[duckdb]'`."
|
|
54
54
|
)
|
|
55
55
|
return _engine
|
|
56
56
|
|
|
@@ -372,7 +372,7 @@ def main() -> None:
|
|
|
372
372
|
from vgi._test_fixtures.catalog_contents import CONTENTS_WORKERS
|
|
373
373
|
from vgi._test_fixtures.narrow_bind.worker import NarrowBindWorker
|
|
374
374
|
from vgi._test_fixtures.twin_catalogs import TwinAWorker, TwinBWorker
|
|
375
|
-
from vgi.rpc_server import build_rpc_server
|
|
375
|
+
from vgi.rpc_server import build_rpc_server, resolve_grant_keys
|
|
376
376
|
|
|
377
377
|
worker_classes: list[type] = [
|
|
378
378
|
ExampleWorker,
|
|
@@ -408,6 +408,7 @@ def main() -> None:
|
|
|
408
408
|
describe=describe,
|
|
409
409
|
external_location=external_location,
|
|
410
410
|
introspect_principals=introspect_principals,
|
|
411
|
+
grant_keys=resolve_grant_keys(),
|
|
411
412
|
)
|
|
412
413
|
wsgi_app = make_wsgi_app(
|
|
413
414
|
server,
|
|
@@ -61,12 +61,18 @@ class IdentityExampleWorker(ExampleWorker):
|
|
|
61
61
|
def conformance_authenticate(req: Any) -> AuthContext:
|
|
62
62
|
"""Derive the caller from ``X-Conformance-Principal`` / ``X-Conformance-Auth-Time``.
|
|
63
63
|
|
|
64
|
-
An absent principal header means *unauthenticated
|
|
64
|
+
An absent principal header means *unauthenticated*, unless an
|
|
65
|
+
``Authorization`` header is present: that request falls through to the
|
|
66
|
+
sealed-grant and ``resolve_token`` bearers. The auth-time header is
|
|
65
67
|
placed in the claims verbatim and unparsed, because the guard does the
|
|
66
68
|
parsing and the fixture only carries the value.
|
|
67
69
|
"""
|
|
68
70
|
principal = req.get_header(PRINCIPAL_HEADER)
|
|
69
71
|
if not principal:
|
|
72
|
+
if req.get_header("Authorization"):
|
|
73
|
+
# A bearer is not ours: fall through to the identity bearer
|
|
74
|
+
# authenticators vgi-rpc appends (sealed grants, resolve_token).
|
|
75
|
+
raise ValueError("no conformance principal header")
|
|
70
76
|
return AuthContext(domain=None, authenticated=False, principal=None)
|
|
71
77
|
claims: dict[str, object] = {}
|
|
72
78
|
auth_time = req.get_header(AUTH_TIME_HEADER)
|
vgi/client/client.py
CHANGED
|
@@ -2757,7 +2757,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
|
|
|
2757
2757
|
arguments: Arguments | None = None,
|
|
2758
2758
|
function_type: FunctionType = FunctionType.TABLE,
|
|
2759
2759
|
settings: dict[str, Any] | None = None,
|
|
2760
|
+
secrets: dict[str, Any] | None = None,
|
|
2760
2761
|
transaction_opaque_data: bytes | None = None,
|
|
2762
|
+
at_unit: str | None = None,
|
|
2763
|
+
at_value: str | None = None,
|
|
2761
2764
|
) -> BindResponse:
|
|
2762
2765
|
"""Resolve a function's bind response without running init()/process().
|
|
2763
2766
|
|
|
@@ -2789,8 +2792,16 @@ class Client(CatalogClientMixin, AggregateClientMixin):
|
|
|
2789
2792
|
settings: Optional dictionary of settings/pragmas — some
|
|
2790
2793
|
functions' output schema depends on setting values (see
|
|
2791
2794
|
`Meta.required_settings`).
|
|
2795
|
+
secrets: Optional dictionary of secret name to value pairs, as for
|
|
2796
|
+
`table_function` — a function whose bind resolves secrets
|
|
2797
|
+
(e.g. a secret-backed source) needs them here too.
|
|
2792
2798
|
transaction_opaque_data: Optional unique identifier for the
|
|
2793
2799
|
DuckDB transaction.
|
|
2800
|
+
at_unit: Optional time travel unit (e.g. 'timestamp', 'version'):
|
|
2801
|
+
bind the function as of a past point, as `table_function`'s
|
|
2802
|
+
`at_unit` does, so a versioned table's historical schema can be
|
|
2803
|
+
discovered. `None` binds live.
|
|
2804
|
+
at_value: Optional time travel value, paired with `at_unit`.
|
|
2794
2805
|
|
|
2795
2806
|
Returns:
|
|
2796
2807
|
[`BindResponse`][] with `output_schema` and any opaque bind data —
|
|
@@ -2814,8 +2825,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
|
|
|
2814
2825
|
arguments=arguments,
|
|
2815
2826
|
function_type=function_type,
|
|
2816
2827
|
settings=settings,
|
|
2817
|
-
secrets=
|
|
2828
|
+
secrets=secrets,
|
|
2818
2829
|
transaction_opaque_data=transaction_opaque_data,
|
|
2830
|
+
at_unit=at_unit,
|
|
2831
|
+
at_value=at_value,
|
|
2819
2832
|
)
|
|
2820
2833
|
return self._do_bind(self._primary.proxy, bind_request, None)
|
|
2821
2834
|
except ClientError as e:
|
|
@@ -3273,15 +3286,23 @@ class Client(CatalogClientMixin, AggregateClientMixin):
|
|
|
3273
3286
|
# AnnotatedBatch (not just .batch) so custom_metadata survives the
|
|
3274
3287
|
# cross-thread handoff for batch_metadata_callback below.
|
|
3275
3288
|
output_queue: Queue[AnnotatedBatch | BaseException | None] = Queue()
|
|
3289
|
+
# Set when the consumer stops before every worker finished (the
|
|
3290
|
+
# generator was closed early, or a worker failed). Each reader thread is
|
|
3291
|
+
# its stream's only reader, so it is also the one that ends the stream:
|
|
3292
|
+
# a producer stream advances one tick per batch, so a reader notices the
|
|
3293
|
+
# flag within one batch and cancels from its own thread. Ending it from
|
|
3294
|
+
# the consumer's thread instead would put two readers on one pipe.
|
|
3295
|
+
stop = threading.Event()
|
|
3276
3296
|
|
|
3277
3297
|
def read_worker_output(worker: WorkerConnection) -> None:
|
|
3278
3298
|
"""Thread function that reads all output from a single worker."""
|
|
3279
3299
|
try:
|
|
3280
|
-
|
|
3300
|
+
stream = worker.stream
|
|
3301
|
+
if stream is None:
|
|
3281
3302
|
output_queue.put(None)
|
|
3282
3303
|
return
|
|
3283
3304
|
|
|
3284
|
-
for output in
|
|
3305
|
+
for output in stream:
|
|
3285
3306
|
_logger.debug(
|
|
3286
3307
|
"received_output_from_worker worker_index=%s num_rows=%s",
|
|
3287
3308
|
worker.worker_index,
|
|
@@ -3289,6 +3310,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
|
|
|
3289
3310
|
)
|
|
3290
3311
|
if output.batch.num_rows > 0:
|
|
3291
3312
|
output_queue.put(output)
|
|
3313
|
+
if stop.is_set():
|
|
3314
|
+
_logger.debug("table_function_worker_cancelled worker_index=%s", worker.worker_index)
|
|
3315
|
+
stream.cancel()
|
|
3316
|
+
break
|
|
3292
3317
|
|
|
3293
3318
|
output_queue.put(None) # Signal completion
|
|
3294
3319
|
except StopIteration:
|
|
@@ -3308,41 +3333,59 @@ class Client(CatalogClientMixin, AggregateClientMixin):
|
|
|
3308
3333
|
thread.start()
|
|
3309
3334
|
threads.append(thread)
|
|
3310
3335
|
|
|
3311
|
-
# Collect outputs from all workers until all are done
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
self._join_threads(threads)
|
|
3337
|
-
_logger.debug("all_table_function_workers_complete")
|
|
3336
|
+
# Collect outputs from all workers until all are done. The finally
|
|
3337
|
+
# block also runs when the consumer closes the generator early
|
|
3338
|
+
# (GeneratorExit at the yield) or a worker fails: the reader threads are
|
|
3339
|
+
# told to stop and joined before any stream is touched here.
|
|
3340
|
+
completed = False
|
|
3341
|
+
try:
|
|
3342
|
+
workers_finished = 0
|
|
3343
|
+
while workers_finished < num_workers:
|
|
3344
|
+
result = output_queue.get()
|
|
3345
|
+
|
|
3346
|
+
# Check for exceptions from worker threads
|
|
3347
|
+
if isinstance(result, BaseException):
|
|
3348
|
+
if isinstance(result, RpcError):
|
|
3349
|
+
raise ClientError.from_rpc_error(result) from result
|
|
3350
|
+
raise ClientError(f"Worker thread failed: {result}") from result
|
|
3351
|
+
|
|
3352
|
+
# None signals a worker finished
|
|
3353
|
+
if result is None:
|
|
3354
|
+
workers_finished += 1
|
|
3355
|
+
_logger.debug(
|
|
3356
|
+
"worker_finished workers_finished=%s total_workers=%s",
|
|
3357
|
+
workers_finished,
|
|
3358
|
+
num_workers,
|
|
3359
|
+
)
|
|
3360
|
+
continue
|
|
3338
3361
|
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3362
|
+
if batch_metadata_callback is not None:
|
|
3363
|
+
batch_metadata_callback(result.custom_metadata)
|
|
3364
|
+
yield result.batch
|
|
3365
|
+
completed = True
|
|
3366
|
+
finally:
|
|
3367
|
+
if not completed:
|
|
3368
|
+
stop.set()
|
|
3369
|
+
self._join_threads(threads)
|
|
3370
|
+
_logger.debug("all_table_function_workers_complete completed=%s", completed)
|
|
3371
|
+
|
|
3372
|
+
# Close streams and secondary workers. A stream a reader cancelled is
|
|
3373
|
+
# already closed, so close() is a no-op for it.
|
|
3374
|
+
for worker, thread in zip(all_workers, threads, strict=True):
|
|
3375
|
+
if thread.is_alive():
|
|
3376
|
+
# Still reading after the join timeout: the stream's state is
|
|
3377
|
+
# unknown and its reader still owns it. Kill the worker so
|
|
3378
|
+
# neither that reader nor a later stop() blocks on the pipe.
|
|
3379
|
+
_logger.warning("table_function_worker_killed worker_index=%s", worker.worker_index)
|
|
3380
|
+
if worker.proc is not None:
|
|
3381
|
+
worker.proc.kill()
|
|
3382
|
+
worker.stream = None
|
|
3383
|
+
continue
|
|
3384
|
+
if worker.stream is not None:
|
|
3385
|
+
worker.stream.close()
|
|
3386
|
+
worker.stream = None
|
|
3387
|
+
self._close_secondary_workers()
|
|
3388
|
+
_logger.debug("parallel_table_function_complete")
|
|
3346
3389
|
|
|
3347
3390
|
# ==================================================================
|
|
3348
3391
|
# Custom COPY formats
|
vgi/filter_v2.py
CHANGED
|
@@ -117,6 +117,34 @@ class FilterV2Error(ValueError):
|
|
|
117
117
|
"""A Filter Encoding v2 document is malformed or cannot be evaluated."""
|
|
118
118
|
|
|
119
119
|
|
|
120
|
+
def _payload_nbytes(batch: pa.RecordBatch) -> int:
|
|
121
|
+
"""Size of a filter payload batch, for the payload limit.
|
|
122
|
+
|
|
123
|
+
``RecordBatch.nbytes`` is not implemented for view types (``string_view``,
|
|
124
|
+
``binary_view``, ``list_view``) before pyarrow 19 and raises a raw
|
|
125
|
+
``ArrowTypeError``. Fall back to the total buffer size there, which walks
|
|
126
|
+
every buffer generically and can only overcount (shared buffers), never
|
|
127
|
+
undercount, so the limit still holds.
|
|
128
|
+
|
|
129
|
+
Args:
|
|
130
|
+
batch: The filter payload batch.
|
|
131
|
+
|
|
132
|
+
Returns:
|
|
133
|
+
Its size in bytes, for comparison with ``MAX_PAYLOAD_BYTES``.
|
|
134
|
+
|
|
135
|
+
Raises:
|
|
136
|
+
FilterV2Error: If neither measurement is available for the batch.
|
|
137
|
+
"""
|
|
138
|
+
try:
|
|
139
|
+
return int(batch.nbytes)
|
|
140
|
+
except (pa.ArrowTypeError, pa.ArrowNotImplementedError):
|
|
141
|
+
pass
|
|
142
|
+
try:
|
|
143
|
+
return int(batch.get_total_buffer_size())
|
|
144
|
+
except (pa.ArrowTypeError, pa.ArrowNotImplementedError) as exc:
|
|
145
|
+
raise FilterV2Error(f"cannot measure filter payload size: {exc}") from exc
|
|
146
|
+
|
|
147
|
+
|
|
120
148
|
class PredicateMode(StrEnum):
|
|
121
149
|
"""Whether a predicate must be applied exactly or is only a pruning hint."""
|
|
122
150
|
|
|
@@ -592,7 +620,7 @@ class _Parser:
|
|
|
592
620
|
raise FilterV2Error(f"noncanonical payload field name {name!r}")
|
|
593
621
|
if name.startswith("type_") and self.batch.column(i)[0].is_valid:
|
|
594
622
|
raise FilterV2Error(f"{name} must contain a NULL value")
|
|
595
|
-
if self.batch
|
|
623
|
+
if _payload_nbytes(self.batch) > MAX_PAYLOAD_BYTES:
|
|
596
624
|
raise FilterV2Error("filter payload exceeds 16 MiB")
|
|
597
625
|
return parse_evaluation_context(self.batch.schema)
|
|
598
626
|
|
vgi/rpc_server.py
CHANGED
|
@@ -34,13 +34,14 @@ from typing import TYPE_CHECKING, Any, Literal
|
|
|
34
34
|
|
|
35
35
|
if TYPE_CHECKING:
|
|
36
36
|
from vgi_rpc.external import ExternalLocationConfig
|
|
37
|
+
from vgi_rpc.grants import GrantKeys
|
|
37
38
|
from vgi_rpc.rpc import RpcServer
|
|
38
39
|
from vgi_rpc.rpc._token_identity import IdentityImpl
|
|
39
40
|
|
|
40
41
|
from vgi.meta_worker import MetaWorker
|
|
41
42
|
from vgi.worker import Worker
|
|
42
43
|
|
|
43
|
-
__all__ = ["Transport", "build_rpc_server"]
|
|
44
|
+
__all__ = ["Transport", "build_rpc_server", "resolve_grant_keys"]
|
|
44
45
|
|
|
45
46
|
Transport = Literal["pipe", "unix", "tcp", "iroh", "http"]
|
|
46
47
|
"""The transport a server is being built for.
|
|
@@ -65,6 +66,7 @@ def build_rpc_server(
|
|
|
65
66
|
transport: Transport,
|
|
66
67
|
describe: bool = True,
|
|
67
68
|
introspect_principals: Iterable[str] | None = None,
|
|
69
|
+
grant_keys: GrantKeys | None = None,
|
|
68
70
|
external_location: ExternalLocationConfig | None = None,
|
|
69
71
|
otel_config: Any = None,
|
|
70
72
|
) -> RpcServer:
|
|
@@ -83,6 +85,11 @@ def build_rpc_server(
|
|
|
83
85
|
introspect_principals: HTTP only. Principals permitted to call
|
|
84
86
|
``introspect_token``; ``None`` reads ``VGI_INTROSPECT_PRINCIPALS``.
|
|
85
87
|
Only consulted when the worker overrides ``resolve_token``.
|
|
88
|
+
grant_keys: HTTP only. Sealed-grant configuration (see
|
|
89
|
+
[`resolve_grant_keys`][vgi.rpc_server.resolve_grant_keys]). When
|
|
90
|
+
given, Identity is hosted even without hooks: the framework mints
|
|
91
|
+
sealed grants unless the worker overrides ``mint_grant``, and the
|
|
92
|
+
HTTP server accepts them back as bearer credentials.
|
|
86
93
|
external_location: Optional external-storage configuration for large
|
|
87
94
|
batches (HTTP).
|
|
88
95
|
otel_config: Optional ``OtelConfig``. When given, the server is
|
|
@@ -116,13 +123,18 @@ def build_rpc_server(
|
|
|
116
123
|
if transport in _IDENTITY_TRANSPORTS:
|
|
117
124
|
# A Worker's hooks are classmethods; a MetaWorker answers for the one
|
|
118
125
|
# child that implements each hook.
|
|
119
|
-
identity = _build_identity(
|
|
126
|
+
identity = _build_identity(
|
|
127
|
+
type(worker) if isinstance(worker, Worker) else worker, introspect_principals, grant_keys
|
|
128
|
+
)
|
|
120
129
|
|
|
121
130
|
server = RpcServer(
|
|
122
131
|
primary,
|
|
123
132
|
worker,
|
|
124
133
|
extra_protocols=extra,
|
|
125
134
|
identity=identity,
|
|
135
|
+
# Explicit, never vgi-rpc's "env" default: grants are an HTTP-only
|
|
136
|
+
# identity feature here, and the caller resolved them already.
|
|
137
|
+
grant_keys=grant_keys if transport in _IDENTITY_TRANSPORTS else None,
|
|
126
138
|
external_location=external_location,
|
|
127
139
|
enable_describe=describe,
|
|
128
140
|
server_version=_get_vgi_version(),
|
|
@@ -191,6 +203,7 @@ def _validated_hosted_protocols(worker: Worker | MetaWorker, primary: type) -> t
|
|
|
191
203
|
def _build_identity(
|
|
192
204
|
worker_cls: type[Worker] | MetaWorker,
|
|
193
205
|
introspect_principals: Iterable[str] | None,
|
|
206
|
+
grant_keys: GrantKeys | None = None,
|
|
194
207
|
) -> IdentityImpl | None:
|
|
195
208
|
"""Build the ``vgi_rpc.Identity.v1`` implementation, or ``None``.
|
|
196
209
|
|
|
@@ -217,6 +230,9 @@ def _build_identity(
|
|
|
217
230
|
the one child class that overrides each.
|
|
218
231
|
introspect_principals: Principals permitted to introspect, or ``None``
|
|
219
232
|
to read the environment.
|
|
233
|
+
grant_keys: Sealed-grant configuration. When given, Identity is built
|
|
234
|
+
even with no hook, and mints sealed grants unless the worker
|
|
235
|
+
overrides ``mint_grant``.
|
|
220
236
|
|
|
221
237
|
Returns:
|
|
222
238
|
The implementation, or ``None`` when this worker does not resolve
|
|
@@ -225,7 +241,7 @@ def _build_identity(
|
|
|
225
241
|
"""
|
|
226
242
|
resolver = worker_cls._introspect_resolver()
|
|
227
243
|
minter = worker_cls._grant_minter()
|
|
228
|
-
if resolver is None and minter is None:
|
|
244
|
+
if resolver is None and minter is None and grant_keys is None:
|
|
229
245
|
return None
|
|
230
246
|
|
|
231
247
|
# Not re-exported by ``vgi_rpc.rpc``, so the private module is the only
|
|
@@ -239,6 +255,9 @@ def _build_identity(
|
|
|
239
255
|
# validates it only when a resolver is supplied — a worker that mints
|
|
240
256
|
# grants but resolves nothing is not an oracle and needs no allowlist.
|
|
241
257
|
introspect_principals=(_resolve_introspect_principals(introspect_principals) if resolver is not None else None),
|
|
258
|
+
# With keys and no mint_grant override, IdentityImpl mints sealed
|
|
259
|
+
# grants itself; the HTTP server then accepts them as bearers.
|
|
260
|
+
grant_keys=grant_keys,
|
|
242
261
|
)
|
|
243
262
|
|
|
244
263
|
|
|
@@ -285,3 +304,37 @@ def _resolve_introspect_principals(explicit: Iterable[str] | None) -> list[str]:
|
|
|
285
304
|
)
|
|
286
305
|
sys.exit(1)
|
|
287
306
|
return principals
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def resolve_grant_keys(explicit: Sequence[str] | None = None) -> GrantKeys | None:
|
|
310
|
+
"""Resolve the sealed-grant configuration, or exit with an actionable message.
|
|
311
|
+
|
|
312
|
+
Grants are opt-in: with no key configured this returns ``None`` and
|
|
313
|
+
nothing changes. Keys come from *explicit* (``--grant-key``, repeatable)
|
|
314
|
+
or else ``VGI_RPC_GRANT_KEYS`` (comma-separated). Either way
|
|
315
|
+
``VGI_RPC_GRANT_AUDIENCE`` and ``VGI_RPC_GRANT_MAX_TTL_SECONDS`` (default
|
|
316
|
+
7 days) apply. Each key is standard base64 of exactly 32 bytes; the first
|
|
317
|
+
mints and every key verifies, which is how keys rotate.
|
|
318
|
+
|
|
319
|
+
Args:
|
|
320
|
+
explicit: Keys given on the command line, or ``None`` to read the
|
|
321
|
+
environment.
|
|
322
|
+
|
|
323
|
+
Returns:
|
|
324
|
+
The configuration, or ``None`` when no key is configured.
|
|
325
|
+
|
|
326
|
+
Raises:
|
|
327
|
+
SystemExit: A key or lifetime is malformed. A worker refuses to start
|
|
328
|
+
rather than run with a key it misread.
|
|
329
|
+
|
|
330
|
+
"""
|
|
331
|
+
from vgi_rpc.grants import GRANT_KEYS_ENV, GrantKeys
|
|
332
|
+
|
|
333
|
+
env: dict[str, str] = dict(os.environ)
|
|
334
|
+
if explicit:
|
|
335
|
+
env[GRANT_KEYS_ENV] = ",".join(explicit)
|
|
336
|
+
try:
|
|
337
|
+
return GrantKeys.from_env(env)
|
|
338
|
+
except ValueError as exc:
|
|
339
|
+
sys.stderr.write(f"Error: invalid sealed-grant configuration ({GRANT_KEYS_ENV} / --grant-key): {exc}\n")
|
|
340
|
+
sys.exit(1)
|
vgi/serve.py
CHANGED
|
@@ -41,6 +41,7 @@ from vgi.profiling import maybe_start_profile
|
|
|
41
41
|
|
|
42
42
|
if TYPE_CHECKING:
|
|
43
43
|
import falcon
|
|
44
|
+
from vgi_rpc.grants import GrantKeys
|
|
44
45
|
from vgi_rpc.otel import OtelConfig
|
|
45
46
|
from vgi_rpc.rpc import AuthContext, PeerAuthenticationPolicy, PeerIdentityProvider
|
|
46
47
|
|
|
@@ -216,6 +217,7 @@ def create_app(
|
|
|
216
217
|
max_stream_response_bytes: int | None = None,
|
|
217
218
|
max_externalized_response_bytes: int | None = None,
|
|
218
219
|
introspect_principals: Iterable[str] | None = None,
|
|
220
|
+
grant_keys: GrantKeys | None = None,
|
|
219
221
|
iroh_bridge_issuer: str | None = None,
|
|
220
222
|
iroh_trusted_proxy_addresses: Iterable[str] = (),
|
|
221
223
|
iroh_authenticate: bool = True,
|
|
@@ -277,6 +279,16 @@ def create_app(
|
|
|
277
279
|
``vgi_rpc.Identity.v1``'s ``introspect_token``. Only consulted
|
|
278
280
|
when the worker class overrides ``resolve_token``. ``None`` reads
|
|
279
281
|
``VGI_INTROSPECT_PRINCIPALS``.
|
|
282
|
+
grant_keys: Sealed-grant configuration. ``None`` reads
|
|
283
|
+
``VGI_RPC_GRANT_KEYS`` (see
|
|
284
|
+
[`resolve_grant_keys`][vgi.rpc_server.resolve_grant_keys]); no key
|
|
285
|
+
configured means grants are off. With keys, ``issue_grant`` mints
|
|
286
|
+
sealed grants (unless the worker overrides ``mint_grant``) and
|
|
287
|
+
requests may present them as ``Authorization: Bearer <grant>``.
|
|
288
|
+
Independently, a worker that overrides ``resolve_token`` has it
|
|
289
|
+
consulted for bearer credentials. Both come after *authenticate*
|
|
290
|
+
in the chain, so the deployment's own authenticator answers first
|
|
291
|
+
and must raise ``ValueError`` for a credential it does not know.
|
|
280
292
|
iroh_bridge_issuer: Operator-controlled identity namespace for an
|
|
281
293
|
``iroh-http/2`` bridge. ``None`` disables Iroh header trust.
|
|
282
294
|
iroh_trusted_proxy_addresses: Exact immediate bridge addresses.
|
|
@@ -297,7 +309,7 @@ def create_app(
|
|
|
297
309
|
sys.exit(1)
|
|
298
310
|
|
|
299
311
|
from vgi.otel import VgiTracer
|
|
300
|
-
from vgi.rpc_server import build_rpc_server
|
|
312
|
+
from vgi.rpc_server import build_rpc_server, resolve_grant_keys
|
|
301
313
|
|
|
302
314
|
# Resolve the signing key once, here, so the worker (which seals catalog
|
|
303
315
|
# opaque-data envelopes) and the HTTP state-token machinery share the same
|
|
@@ -319,7 +331,9 @@ def create_app(
|
|
|
319
331
|
transport="http",
|
|
320
332
|
describe=describe,
|
|
321
333
|
introspect_principals=introspect_principals,
|
|
334
|
+
grant_keys=grant_keys if grant_keys is not None else resolve_grant_keys(),
|
|
322
335
|
)
|
|
336
|
+
authenticate, identity_bearer = _compose_identity_bearer(server, authenticate)
|
|
323
337
|
|
|
324
338
|
effective_peer_identity_providers = tuple(peer_identity_providers)
|
|
325
339
|
effective_peer_authentication_policy = peer_authentication_policy
|
|
@@ -356,6 +370,7 @@ def create_app(
|
|
|
356
370
|
max_stream_response_bytes=max_stream_response_bytes,
|
|
357
371
|
max_externalized_response_bytes=max_externalized_response_bytes,
|
|
358
372
|
enable_landing_page=False,
|
|
373
|
+
identity_bearer=identity_bearer,
|
|
359
374
|
)
|
|
360
375
|
|
|
361
376
|
# Frontend: either redirect to external CDN or serve pre-rendered worker page
|
|
@@ -490,6 +505,16 @@ def main() -> None:
|
|
|
490
505
|
"that case, with no permissive default. Env: VGI_INTROSPECT_PRINCIPALS."
|
|
491
506
|
),
|
|
492
507
|
),
|
|
508
|
+
grant_key: list[str] | None = typer.Option( # noqa: B008
|
|
509
|
+
None,
|
|
510
|
+
"--grant-key",
|
|
511
|
+
help=(
|
|
512
|
+
"Sealed-grant key: standard base64 of 32 bytes; repeat to rotate (the first "
|
|
513
|
+
"mints, all verify). Enables issue_grant without a mint_grant override and "
|
|
514
|
+
"accepts the grants as HTTP bearer credentials. Env: VGI_RPC_GRANT_KEYS "
|
|
515
|
+
"(comma-separated), VGI_RPC_GRANT_AUDIENCE, VGI_RPC_GRANT_MAX_TTL_SECONDS."
|
|
516
|
+
),
|
|
517
|
+
),
|
|
493
518
|
access_log_sample: float | None = typer.Option( # noqa: B008
|
|
494
519
|
None,
|
|
495
520
|
"--access-log-sample",
|
|
@@ -573,6 +598,7 @@ def main() -> None:
|
|
|
573
598
|
max_stream_response_bytes=max_stream_response_bytes,
|
|
574
599
|
max_externalized_response_bytes=max_externalized_response_bytes,
|
|
575
600
|
introspect_principals=introspect_principals,
|
|
601
|
+
grant_keys=grant_key,
|
|
576
602
|
server=server,
|
|
577
603
|
worker_ref=worker_ref,
|
|
578
604
|
http_workers=http_workers,
|
|
@@ -770,7 +796,44 @@ def _resolve_authenticate() -> Callable[..., Any] | None:
|
|
|
770
796
|
# this worker", with user identity handled upstream.
|
|
771
797
|
from vgi_rpc.http import require_all
|
|
772
798
|
|
|
773
|
-
|
|
799
|
+
gated = require_all(gate, inner)
|
|
800
|
+
# Kept so create_app can put the identity bearers (sealed grants,
|
|
801
|
+
# resolve_token) inside the gate rather than beside it.
|
|
802
|
+
setattr(gated, _PROXY_GATE_PARTS_ATTR, (gate, inner))
|
|
803
|
+
return gated
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
#: Attribute on a proxy-gated authenticator holding its ``(gate, inner)``.
|
|
807
|
+
_PROXY_GATE_PARTS_ATTR = "_vgi_proxy_gate_parts"
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _compose_identity_bearer(
|
|
811
|
+
server: Any,
|
|
812
|
+
authenticate: Callable[..., Any] | None,
|
|
813
|
+
) -> tuple[Callable[..., Any] | None, bool]:
|
|
814
|
+
"""Return ``(authenticate, identity_bearer)`` for ``make_wsgi_app``.
|
|
815
|
+
|
|
816
|
+
vgi-rpc appends the identity bearers (sealed grants, then the worker's
|
|
817
|
+
``resolve_token``) after *authenticate* itself. It refuses when
|
|
818
|
+
*authenticate* depends on proxy-injected evidence, because an OR beside a
|
|
819
|
+
proxy-proof gate would let a grant bypass it. VGI's own gate
|
|
820
|
+
(``VGI_PROXY_PROOF_MODE``) is built by :func:`_resolve_authenticate`, so
|
|
821
|
+
here the bearers go *inside* it: ``require_all(gate, inner, then grants,
|
|
822
|
+
then resolve_token)``. Any other authenticator is left to vgi-rpc.
|
|
823
|
+
"""
|
|
824
|
+
identity = getattr(server, "identity", None)
|
|
825
|
+
parts = getattr(authenticate, _PROXY_GATE_PARTS_ATTR, None)
|
|
826
|
+
if identity is None or parts is None:
|
|
827
|
+
return authenticate, True
|
|
828
|
+
if identity.grant_keys is None and identity.resolve_token_hook is None:
|
|
829
|
+
return authenticate, True
|
|
830
|
+
from vgi_rpc.http import compose_identity_authenticate, require_all
|
|
831
|
+
|
|
832
|
+
gate, inner = parts
|
|
833
|
+
composed = compose_identity_authenticate(
|
|
834
|
+
inner, grant_keys=identity.grant_keys, resolve_token=identity.resolve_token_hook
|
|
835
|
+
)
|
|
836
|
+
return require_all(gate, composed), False
|
|
774
837
|
|
|
775
838
|
|
|
776
839
|
def _resolve_proxy_proof_gate() -> Any | None:
|
|
@@ -1352,6 +1415,7 @@ def _serve_http(
|
|
|
1352
1415
|
max_stream_response_bytes: int | None = None,
|
|
1353
1416
|
max_externalized_response_bytes: int | None = None,
|
|
1354
1417
|
introspect_principals: str | None = None,
|
|
1418
|
+
grant_keys: Sequence[str] | None = None,
|
|
1355
1419
|
server: str = "waitress",
|
|
1356
1420
|
worker_ref: str | None = None,
|
|
1357
1421
|
http_workers: int | None = None,
|
|
@@ -1369,6 +1433,13 @@ def _serve_http(
|
|
|
1369
1433
|
# export_serve_config for why the environment is the channel.
|
|
1370
1434
|
if introspect_principals is not None:
|
|
1371
1435
|
os.environ["VGI_INTROSPECT_PRINCIPALS"] = introspect_principals
|
|
1436
|
+
if grant_keys:
|
|
1437
|
+
from vgi_rpc.grants import GRANT_KEYS_ENV
|
|
1438
|
+
|
|
1439
|
+
from vgi.rpc_server import resolve_grant_keys
|
|
1440
|
+
|
|
1441
|
+
resolve_grant_keys(grant_keys) # a malformed key stops startup here
|
|
1442
|
+
os.environ[GRANT_KEYS_ENV] = ",".join(grant_keys)
|
|
1372
1443
|
|
|
1373
1444
|
if server == "granian":
|
|
1374
1445
|
_serve_http_granian(
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: vgi-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.42.1
|
|
4
4
|
Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
|
|
5
5
|
Project-URL: Homepage, https://query.farm
|
|
6
6
|
Project-URL: Repository, https://github.com/Query-farm/vgi-python
|
|
@@ -162,7 +162,7 @@ Requires-Dist: httpx2>=2.12.0
|
|
|
162
162
|
Requires-Dist: platformdirs
|
|
163
163
|
Requires-Dist: pyarrow
|
|
164
164
|
Requires-Dist: typer>=0.9
|
|
165
|
-
Requires-Dist: vgi-rpc>=0.
|
|
165
|
+
Requires-Dist: vgi-rpc>=0.49.0
|
|
166
166
|
Provides-Extra: azure
|
|
167
167
|
Requires-Dist: azure-identity>=1.16.0; extra == 'azure'
|
|
168
168
|
Requires-Dist: pymssql>=2.3.0; extra == 'azure'
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
vgi/__init__.py,sha256=8uLMNQcNzWjp8VzYtj2u98_Wp8WpB8c_dKlMv67TH0g,5384
|
|
2
|
-
vgi/_duckdb.py,sha256=
|
|
2
|
+
vgi/_duckdb.py,sha256=Tu8Yb4vABRv4yOMo7qul8LxG_wH5ZqZuOCSSjyiMgbY,2198
|
|
3
3
|
vgi/_storage_profile.py,sha256=VkTsXojuE0tHEzurmteQSAiL1vI3CZSgYkL6D_h8GvE,5061
|
|
4
4
|
vgi/aggregate_function.py,sha256=NYxudbvQ9ZLkP2w5hRdd4YgsGUZ_0cVfqqXnlmJPfls,27389
|
|
5
5
|
vgi/argument_spec.py,sha256=SADH97VlFu3asI8CWZ_I7XTuSxwqFciFgdupdAf-tkY,34231
|
|
@@ -10,7 +10,7 @@ vgi/cache_control.py,sha256=IyBp9gZTCc4Y-4V7Rs05_o5lSEjbR2jt6cddXPU0dKw,7812
|
|
|
10
10
|
vgi/copy_from_function.py,sha256=iSkR48b3W9IVQ-HY2I8-5W1QjTtWF7k1GkogHNE7ZMM,7114
|
|
11
11
|
vgi/copy_to_function.py,sha256=8l0v_DyEF1EnFwinStrZDQ-yNFfGJsv3Mr5o2jG30f0,9279
|
|
12
12
|
vgi/exceptions.py,sha256=oX_sZc9xGWi7Xf8cJQf89fX19i3ocDEj_V_76GINgBQ,7294
|
|
13
|
-
vgi/filter_v2.py,sha256=
|
|
13
|
+
vgi/filter_v2.py,sha256=ezQ1rDweMJJkow-5mQPG5BYVnN93426ObYPD2KzUNSs,58889
|
|
14
14
|
vgi/filter_v2_builder.py,sha256=8OmyzDI6GDdH4pYm2sJFMidzKusi0I8IGNipolPpjX0,5535
|
|
15
15
|
vgi/function.py,sha256=KIfvKk1nzBvRDPXViHNX89tv6gvIMiDj4e8fl6ANqQg,11626
|
|
16
16
|
vgi/function_storage.py,sha256=FjVwxc1BIjWo-82j8YE5qcU8w66sGjwyGsTmCUhn-Kw,67576
|
|
@@ -25,13 +25,13 @@ vgi/profiling.py,sha256=rK7vij3oHhXC3JHI2BsMySsXh7_hrXi0aXpCW8iFSIc,3671
|
|
|
25
25
|
vgi/protocol.py,sha256=J9vaxstGvxe4Vmlf_Aij9HkabGbdFx_nEsQI_m-V7ak,175438
|
|
26
26
|
vgi/protocol_version.txt,sha256=yJuHwmLp1f_SDnOqx0oXcq3Mn1ohaCzDOtz1CG5f71w,6
|
|
27
27
|
vgi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
28
|
-
vgi/rpc_server.py,sha256=
|
|
28
|
+
vgi/rpc_server.py,sha256=Qdqi6rHkX78SYXUZqG5D-a9NR0ZtGoxW22he0J9xS-0,14914
|
|
29
29
|
vgi/scalar_function.py,sha256=YBy5s4eEBYTMjACGPE-8-82BObyJoM9qt9cnm24b_RQ,53485
|
|
30
30
|
vgi/schema_path.py,sha256=GVlkZlCUj7GbVhgiocSMC6Ff5iTJhKqiYE62hMIEKiI,2403
|
|
31
31
|
vgi/schema_utils.py,sha256=zQCliUgIpO2u77Mhtgg3OFvOEDNgJopu6kj2fw8YP1I,7945
|
|
32
32
|
vgi/secret_protocol.py,sha256=iCDl7nF7Nb4cJV4vpfbldunTDqDJWzCzS7uCmOACf7U,7151
|
|
33
33
|
vgi/secret_service.py,sha256=r7qCzfnOMirf9wSCUe-mgFe3z6v9_cDRzixuSAZG_bs,8973
|
|
34
|
-
vgi/serve.py,sha256=
|
|
34
|
+
vgi/serve.py,sha256=V56NcO9VzWaMlkUCXumLBsfMEZ8oZeckT_EJ9sumsQc,63442
|
|
35
35
|
vgi/split_token.py,sha256=tutcksj-hgWrLY7UW3sSqkyoaD-cMqNTOeff40MGl_g,10850
|
|
36
36
|
vgi/table_buffering_function.py,sha256=SI6yLAcs8WBHqS6TeaZ92s-Q6GXn0ReROyQWDujw2iA,19554
|
|
37
37
|
vgi/table_filter_pushdown.py,sha256=4Qo4OAOLjTYQ5pinNTOjLCDnb96OHqpcxJlbSJo3cJU,67144
|
|
@@ -50,8 +50,8 @@ vgi/_test_fixtures/companion.py,sha256=Ya_rg2J4tXOYC7zBFOjTTh-4AYuRhJN5BT4tVZULQ
|
|
|
50
50
|
vgi/_test_fixtures/copy_from.py,sha256=YAWametIhrKE_K_kRwx37QpKMY0yBWAordE5-cNVn4M,6134
|
|
51
51
|
vgi/_test_fixtures/copy_to.py,sha256=fFHxPYVd0wfAfMEvpeBRijqUasBeoAGREY3kHmm_oK8,9667
|
|
52
52
|
vgi/_test_fixtures/global_functions.py,sha256=G1h0w5o6bHMULDFLZuNjrZVrBbO6Cre6kTqTnQwTFmE,11251
|
|
53
|
-
vgi/_test_fixtures/http_server.py,sha256=
|
|
54
|
-
vgi/_test_fixtures/identity_fixture.py,sha256=
|
|
53
|
+
vgi/_test_fixtures/http_server.py,sha256=hS6yKEE6I0UzwfIk_q7ORKee4jn4vD2xSCYItj8VkWE,20552
|
|
54
|
+
vgi/_test_fixtures/identity_fixture.py,sha256=wNTON3wpOGwH7HS9oCPb4Iav7GMYW0NlhtlZF1TL7aY,3318
|
|
55
55
|
vgi/_test_fixtures/nest_tensor.py,sha256=UjjW9UwipYG3ehQpZOm4hdzsafMyvVc0ZKJCrqiZTPU,24471
|
|
56
56
|
vgi/_test_fixtures/orchard_catalog.py,sha256=8Uv-mKTIgl-OYU9JLgG23owlHGmgniyF2chPoKXGtuI,1643
|
|
57
57
|
vgi/_test_fixtures/secret_cache.py,sha256=su4a90I-YHTfsmVPoJGyycTkEzT7GItreYZEfbXMp2o,9552
|
|
@@ -143,7 +143,7 @@ vgi/client/cli_table.py,sha256=H1xrKZF5yYmSe55YSrLWXGIXPmuGuzQcncEBvqL43zs,26498
|
|
|
143
143
|
vgi/client/cli_transaction.py,sha256=TDW0ZrBy8XM_eYSkYk5ewEsnozfzDJOlopd66Fm7OOE,3142
|
|
144
144
|
vgi/client/cli_utils.py,sha256=ykh7XK-yzcW6WGED2_DhPLIVPSyVG3KUdllV5ztBicw,17458
|
|
145
145
|
vgi/client/cli_view.py,sha256=0qhxoS91-zFMjq8V_Fb0PNW-YDxERu8-zPpxSFEE-G8,8490
|
|
146
|
-
vgi/client/client.py,sha256=
|
|
146
|
+
vgi/client/client.py,sha256=OtrpTHVeM4nbXzJkk77ab_WC5hONuMLoYAD4MlLD5qM,170099
|
|
147
147
|
vgi/client/errors.py,sha256=cSN65cyQMk3crwZCz1IdPErK3mtlEkTNpu-nuQUDOQE,1633
|
|
148
148
|
vgi/http/__init__.py,sha256=hlOwOVcoZqmEKVGk7ganOdr5ryRSpEhLBF9sRD7BkYc,608
|
|
149
149
|
vgi/http/demo_storage.py,sha256=830s-H61thozC3eEuqdnwCpYFfvoJRq9vjyuXa5OURo,8769
|
|
@@ -156,8 +156,8 @@ vgi/transactor/_duckdb_compat.py,sha256=sXVZ9JLKAQyGR1BjWczSwdQEavtr-TcZPoVZZnTr
|
|
|
156
156
|
vgi/transactor/client.py,sha256=7DTeMksogsw6ANjQjGOPpKYrV76rg4_kGjktMJf54jg,4486
|
|
157
157
|
vgi/transactor/protocol.py,sha256=v59IvrKnuvwOXvn_HEcBAbCzDVcx0akgKN0R1mChXg0,5034
|
|
158
158
|
vgi/transactor/server.py,sha256=nzsZQxJZdgappdYX8okrFIrjFtUbdM_Hhskl1fZ2nDY,32553
|
|
159
|
-
vgi_python-0.
|
|
160
|
-
vgi_python-0.
|
|
161
|
-
vgi_python-0.
|
|
162
|
-
vgi_python-0.
|
|
163
|
-
vgi_python-0.
|
|
159
|
+
vgi_python-0.42.1.dist-info/METADATA,sha256=8JvivAw5wSBb73Xpal2NdQIafuA4enfCQnEQAqs5RL4,25953
|
|
160
|
+
vgi_python-0.42.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
161
|
+
vgi_python-0.42.1.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
|
|
162
|
+
vgi_python-0.42.1.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
|
|
163
|
+
vgi_python-0.42.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|