sunstone-data-platform 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunstone_data/__init__.py +36 -0
- sunstone_data/_version.py +24 -0
- sunstone_data/auth/__init__.py +43 -0
- sunstone_data/auth/client_creds.py +239 -0
- sunstone_data/auth/pkce.py +307 -0
- sunstone_data/auth/store.py +97 -0
- sunstone_data/catalog.py +175 -0
- sunstone_data/cli/__init__.py +5 -0
- sunstone_data/cli/_app.py +26 -0
- sunstone_data/cli/_errors.py +31 -0
- sunstone_data/cli/commands/__init__.py +22 -0
- sunstone_data/cli/commands/data_auth.py +120 -0
- sunstone_data/cli/commands/data_import.py +157 -0
- sunstone_data/cli/commands/data_info.py +166 -0
- sunstone_data/cli/commands/data_publish.py +69 -0
- sunstone_data/cli/commands/ls.py +49 -0
- sunstone_data/cli/commands/prov.py +58 -0
- sunstone_data/cli/commands/refs.py +142 -0
- sunstone_data/cli/commands/show.py +139 -0
- sunstone_data/cli/commands/snapshots.py +77 -0
- sunstone_data/cli/commands/sparql.py +68 -0
- sunstone_data/cli/main.py +30 -0
- sunstone_data/config.py +250 -0
- sunstone_data/discovery.py +317 -0
- sunstone_data/env_commands.py +97 -0
- sunstone_data/env_section.py +86 -0
- sunstone_data/graph/__init__.py +36 -0
- sunstone_data/graph/client.py +328 -0
- sunstone_data/graph/dataset.py +438 -0
- sunstone_data/graph/terms.py +34 -0
- sunstone_data/identifiers.py +155 -0
- sunstone_data/import_/__init__.py +28 -0
- sunstone_data/import_/fetcher.py +259 -0
- sunstone_data/import_/owner.py +130 -0
- sunstone_data/import_/parser.py +492 -0
- sunstone_data/import_/pipeline.py +275 -0
- sunstone_data/import_/storage.py +135 -0
- sunstone_data/metadata/__init__.py +41 -0
- sunstone_data/metadata/builder.py +276 -0
- sunstone_data/metadata/models.py +87 -0
- sunstone_data/metadata/provenance.py +171 -0
- sunstone_data/metadata/provenance_rdf.py +102 -0
- sunstone_data/plugin.py +62 -0
- sunstone_data/publish.py +134 -0
- sunstone_data/query.py +624 -0
- sunstone_data/resolve.py +178 -0
- sunstone_data/url.py +78 -0
- sunstone_data/utils/__init__.py +12 -0
- sunstone_data/utils/checksum.py +14 -0
- sunstone_data/utils/encoding.py +16 -0
- sunstone_data/utils/naming.py +38 -0
- sunstone_data/versioning/__init__.py +17 -0
- sunstone_data/versioning/schema_evolution.py +180 -0
- sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
- sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
- sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
- sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
- sunstone_pipeline/__init__.py +1 -0
- sunstone_pipeline/assets/__init__.py +11 -0
- sunstone_pipeline/assets/mirrors.py +123 -0
- sunstone_pipeline/assets/transforms.py +74 -0
- sunstone_pipeline/definitions.py +25 -0
- sunstone_pipeline/io_manager.py +56 -0
- sunstone_pipeline/resources/__init__.py +6 -0
- sunstone_pipeline/resources/iceberg.py +29 -0
- sunstone_pipeline/resources/refs.py +88 -0
sunstone_data/query.py
ADDED
|
@@ -0,0 +1,624 @@
|
|
|
1
|
+
"""DuckDB query interface for the Iceberg REST catalog.
|
|
2
|
+
|
|
3
|
+
This module provides:
|
|
4
|
+
- QuerySession: Context manager for DuckDB queries with version awareness
|
|
5
|
+
- TableRef: Git-like table reference parsing (ns.table@branch, ns.table@v1.0)
|
|
6
|
+
- TablePreview: Pandas-style preview methods (head, tail, sample, describe)
|
|
7
|
+
- get_session()/close_session(): Thread-local singleton for notebook convenience
|
|
8
|
+
|
|
9
|
+
Example (notebook):
|
|
10
|
+
from sunstone_data.query import get_session
|
|
11
|
+
|
|
12
|
+
session = get_session()
|
|
13
|
+
rel = session.table("sources.patents@main")
|
|
14
|
+
df = rel.filter("year >= 2020").limit(100).df()
|
|
15
|
+
|
|
16
|
+
Example (application):
|
|
17
|
+
from sunstone_data.query import QuerySession
|
|
18
|
+
|
|
19
|
+
with QuerySession() as session:
|
|
20
|
+
preview = session.preview("sources.patents@v1.0")
|
|
21
|
+
print(preview.schema())
|
|
22
|
+
df = preview.head(10).df()
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import os
|
|
28
|
+
import threading
|
|
29
|
+
from dataclasses import dataclass, replace
|
|
30
|
+
from typing import Any, Optional
|
|
31
|
+
|
|
32
|
+
import duckdb
|
|
33
|
+
import httpx
|
|
34
|
+
|
|
35
|
+
from sunstone_data.config import Settings, current_environment_name, get_settings
|
|
36
|
+
from sunstone_data.identifiers import (
|
|
37
|
+
quote_identifier,
|
|
38
|
+
validate_ref_name,
|
|
39
|
+
validate_ref_string,
|
|
40
|
+
validate_sql_identifier,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
# The only branch DuckDB can read; see QuerySession._ensure_branch_attached.
|
|
44
|
+
_DUCKDB_BRANCH = "main"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _resolve_token(settings: Settings) -> str | None:
|
|
48
|
+
"""Return a bearer token for iceberg-catalog, or None if none is configured.
|
|
49
|
+
|
|
50
|
+
Mirrors sunstone_data.catalog._resolve_token: iceberg-catalog's auth
|
|
51
|
+
middleware fails closed on authentication regardless of AUTH_ENFORCE, so
|
|
52
|
+
DuckDB's catalog attach needs a real token when one is configured.
|
|
53
|
+
"""
|
|
54
|
+
from sunstone_data.auth.client_creds import resolve_access_token
|
|
55
|
+
|
|
56
|
+
with httpx.Client() as client:
|
|
57
|
+
return resolve_access_token(settings, current_environment_name(), client)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class TableRef:
|
|
62
|
+
"""Parsed table reference with optional version specifier.
|
|
63
|
+
|
|
64
|
+
Supports git-like syntax, with multi-level namespaces (e.g.
|
|
65
|
+
"un.health.cases"):
|
|
66
|
+
- "namespace.table" -> latest on main branch
|
|
67
|
+
- "namespace.table@ref" -> the given ref, carried verbatim
|
|
68
|
+
|
|
69
|
+
The `@ref` part is not classified as a branch, named version, snapshot id,
|
|
70
|
+
or timestamp at parse time. `Resolver` asks graph-server which refs are
|
|
71
|
+
branches, then resolves every other ref against the table. `@ref` is stored
|
|
72
|
+
as `version_ref` unchanged; `branch` stays at its default ("main") unless
|
|
73
|
+
the caller sets it directly.
|
|
74
|
+
"""
|
|
75
|
+
|
|
76
|
+
namespace: str
|
|
77
|
+
table: str
|
|
78
|
+
branch: str = "main"
|
|
79
|
+
version_ref: Optional[str] = None # ref string, carried verbatim
|
|
80
|
+
|
|
81
|
+
@classmethod
|
|
82
|
+
def parse(cls, ref_string: str) -> TableRef:
|
|
83
|
+
"""Parse table reference string.
|
|
84
|
+
|
|
85
|
+
The namespace may have multiple dot-separated levels; the last
|
|
86
|
+
level before `@ref` (or the end of the string) is the table name.
|
|
87
|
+
A ref after `@` is carried verbatim into `version_ref` — it is not
|
|
88
|
+
classified as a branch/tag/snapshot id/timestamp here.
|
|
89
|
+
|
|
90
|
+
Examples:
|
|
91
|
+
"ns.table" -> TableRef(namespace="ns", table="table")
|
|
92
|
+
"ns.sub.table" -> TableRef(namespace="ns.sub", table="table")
|
|
93
|
+
"ns.table@main" -> TableRef(namespace="ns", table="table", version_ref="main")
|
|
94
|
+
"ns.table@v1.0" -> TableRef(namespace="ns", table="table", version_ref="v1.0")
|
|
95
|
+
"ns.table@2026-01-25" -> TableRef(..., version_ref="2026-01-25")
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
ref_string: Table reference string to parse.
|
|
99
|
+
|
|
100
|
+
Returns:
|
|
101
|
+
TableRef with parsed components.
|
|
102
|
+
|
|
103
|
+
Raises:
|
|
104
|
+
ValueError: If table reference format is invalid.
|
|
105
|
+
"""
|
|
106
|
+
# Split on @ for version reference
|
|
107
|
+
if "@" in ref_string:
|
|
108
|
+
table_part, ref_part = ref_string.split("@", 1)
|
|
109
|
+
else:
|
|
110
|
+
table_part = ref_string
|
|
111
|
+
ref_part = None
|
|
112
|
+
|
|
113
|
+
# Parse namespace.table: the last dot separates table from namespace.
|
|
114
|
+
if "." not in table_part:
|
|
115
|
+
raise ValueError(f"Invalid table reference: {ref_string}. Expected format: namespace.table[@version]")
|
|
116
|
+
|
|
117
|
+
namespace, table = table_part.rsplit(".", 1)
|
|
118
|
+
|
|
119
|
+
# Validate namespace and table names. Each namespace level is
|
|
120
|
+
# validated individually since a level cannot itself contain a dot.
|
|
121
|
+
for level in namespace.split("."):
|
|
122
|
+
validate_sql_identifier(level, "namespace")
|
|
123
|
+
validate_sql_identifier(table, "table")
|
|
124
|
+
|
|
125
|
+
# Carry the ref verbatim; Resolver classifies it when the table is read.
|
|
126
|
+
# validate_ref_string (not validate_ref_name) is used here since a
|
|
127
|
+
# ref may be a snapshot id or timestamp starting with a digit, which
|
|
128
|
+
# validate_ref_name's branch/tag naming pattern would reject.
|
|
129
|
+
if ref_part:
|
|
130
|
+
validate_ref_string(ref_part, "ref")
|
|
131
|
+
return cls(namespace=namespace, table=table, version_ref=ref_part)
|
|
132
|
+
|
|
133
|
+
return cls(namespace=namespace, table=table)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass
|
|
137
|
+
class TablePreview:
|
|
138
|
+
"""Preview wrapper for DuckDB relation with pandas-style methods.
|
|
139
|
+
|
|
140
|
+
All methods return DuckDBPyRelation by default for lazy evaluation.
|
|
141
|
+
Call .to_df() or .fetchall() to materialize results.
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
_relation: duckdb.DuckDBPyRelation
|
|
145
|
+
_table_ref: TableRef
|
|
146
|
+
_session: "QuerySession"
|
|
147
|
+
|
|
148
|
+
def schema(self) -> dict[str, Any]:
|
|
149
|
+
"""Get detailed schema information including Iceberg metadata.
|
|
150
|
+
|
|
151
|
+
Returns:
|
|
152
|
+
Dict with 'columns' list (name, type, nullable, field_id, iceberg_type)
|
|
153
|
+
and optional 'row_count' estimate.
|
|
154
|
+
"""
|
|
155
|
+
# Get DuckDB schema from relation
|
|
156
|
+
columns = []
|
|
157
|
+
for name, dtype in zip(self._relation.columns, self._relation.dtypes):
|
|
158
|
+
columns.append(
|
|
159
|
+
{
|
|
160
|
+
"name": name,
|
|
161
|
+
"type": str(dtype),
|
|
162
|
+
"nullable": True, # DuckDB doesn't expose directly
|
|
163
|
+
}
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
# Enhance with Iceberg metadata if available
|
|
167
|
+
try:
|
|
168
|
+
from sunstone_data.catalog import get_catalog
|
|
169
|
+
|
|
170
|
+
catalog = get_catalog(branch=self._table_ref.branch)
|
|
171
|
+
iceberg_table = catalog.load_table(f"{self._table_ref.namespace}.{self._table_ref.table}")
|
|
172
|
+
iceberg_schema = iceberg_table.schema()
|
|
173
|
+
field_map = {f.name: f for f in iceberg_schema.fields}
|
|
174
|
+
|
|
175
|
+
for col in columns:
|
|
176
|
+
if col["name"] in field_map:
|
|
177
|
+
field = field_map[col["name"]]
|
|
178
|
+
col["field_id"] = field.field_id
|
|
179
|
+
col["nullable"] = not field.required
|
|
180
|
+
col["iceberg_type"] = str(field.field_type)
|
|
181
|
+
except Exception:
|
|
182
|
+
pass # Iceberg metadata optional
|
|
183
|
+
|
|
184
|
+
return {"columns": columns}
|
|
185
|
+
|
|
186
|
+
def head(self, n: int = 5) -> duckdb.DuckDBPyRelation:
|
|
187
|
+
"""Get first n rows (lazy). Default n=5 (pandas style)."""
|
|
188
|
+
return self._relation.limit(n)
|
|
189
|
+
|
|
190
|
+
def tail(self, n: int = 5) -> duckdb.DuckDBPyRelation:
|
|
191
|
+
"""Get last n rows (lazy). Note: requires full scan for offset calculation."""
|
|
192
|
+
conn = self._session._ensure_connection()
|
|
193
|
+
view_name = f"_tail_temp_{id(self._relation)}"
|
|
194
|
+
conn.register(view_name, self._relation)
|
|
195
|
+
|
|
196
|
+
count = conn.sql(f"SELECT COUNT(*) FROM {view_name}").fetchone()[0]
|
|
197
|
+
offset = max(0, count - n)
|
|
198
|
+
|
|
199
|
+
return conn.sql(f"SELECT * FROM {view_name} LIMIT {n} OFFSET {offset}")
|
|
200
|
+
|
|
201
|
+
def sample(
|
|
202
|
+
self,
|
|
203
|
+
n: int | None = None,
|
|
204
|
+
frac: float | None = None,
|
|
205
|
+
method: str = "reservoir",
|
|
206
|
+
) -> duckdb.DuckDBPyRelation:
|
|
207
|
+
"""Get random sample of rows (lazy).
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
n: Number of rows to sample (mutually exclusive with frac).
|
|
211
|
+
frac: Fraction of rows to sample, 0.0-1.0 (mutually exclusive with n).
|
|
212
|
+
method: Sampling method - "reservoir" (exact count, default),
|
|
213
|
+
"bernoulli" (row-level probability), "system" (block-level).
|
|
214
|
+
|
|
215
|
+
Returns:
|
|
216
|
+
DuckDBPyRelation for lazy evaluation.
|
|
217
|
+
|
|
218
|
+
Raises:
|
|
219
|
+
ValueError: If neither n nor frac specified, or both specified.
|
|
220
|
+
"""
|
|
221
|
+
if (n is None) == (frac is None):
|
|
222
|
+
raise ValueError("Specify exactly one of n or frac")
|
|
223
|
+
|
|
224
|
+
conn = self._session._ensure_connection()
|
|
225
|
+
view_name = f"_sample_temp_{id(self._relation)}"
|
|
226
|
+
conn.register(view_name, self._relation)
|
|
227
|
+
|
|
228
|
+
if n is not None:
|
|
229
|
+
return conn.sql(f"SELECT * FROM {view_name} USING SAMPLE {n} ROWS ({method})")
|
|
230
|
+
else:
|
|
231
|
+
pct = frac * 100
|
|
232
|
+
return conn.sql(f"SELECT * FROM {view_name} USING SAMPLE {pct}% ({method})")
|
|
233
|
+
|
|
234
|
+
def describe(self) -> duckdb.DuckDBPyRelation:
|
|
235
|
+
"""Get summary statistics for all columns (lazy).
|
|
236
|
+
|
|
237
|
+
Uses DuckDB's SUMMARIZE for comprehensive stats:
|
|
238
|
+
column_name, column_type, min, max, approx_unique, avg, std, q25, q50, q75,
|
|
239
|
+
count, null_percentage
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
DuckDBPyRelation with statistics.
|
|
243
|
+
"""
|
|
244
|
+
conn = self._session._ensure_connection()
|
|
245
|
+
view_name = f"_describe_temp_{id(self._relation)}"
|
|
246
|
+
conn.register(view_name, self._relation)
|
|
247
|
+
|
|
248
|
+
return conn.sql(f"SUMMARIZE SELECT * FROM {view_name}")
|
|
249
|
+
|
|
250
|
+
def to_df(self):
|
|
251
|
+
"""Convert to pandas DataFrame (materializes data).
|
|
252
|
+
|
|
253
|
+
Warning: Large results will consume significant memory.
|
|
254
|
+
Consider using .limit() before .to_df() for large tables.
|
|
255
|
+
"""
|
|
256
|
+
return self._relation.df()
|
|
257
|
+
|
|
258
|
+
def fetchall(self) -> list[tuple]:
|
|
259
|
+
"""Fetch all rows as list of tuples (materializes data)."""
|
|
260
|
+
return self._relation.fetchall()
|
|
261
|
+
|
|
262
|
+
def __repr__(self) -> str:
|
|
263
|
+
return f"TablePreview({self._table_ref.namespace}.{self._table_ref.table})"
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
class QuerySession:
|
|
267
|
+
"""DuckDB query session with version-aware table registration.
|
|
268
|
+
|
|
269
|
+
Usage as context manager (applications):
|
|
270
|
+
with QuerySession() as session:
|
|
271
|
+
rel = session.table("ns.table@v1.0")
|
|
272
|
+
results = rel.filter("x > 10").limit(100).fetchall()
|
|
273
|
+
|
|
274
|
+
Usage as singleton (notebooks):
|
|
275
|
+
from sunstone_data.query import get_session
|
|
276
|
+
session = get_session()
|
|
277
|
+
rel = session.table("ns.table@main")
|
|
278
|
+
"""
|
|
279
|
+
|
|
280
|
+
def __init__(self, branch: str = "main"):
|
|
281
|
+
"""Initialize query session.
|
|
282
|
+
|
|
283
|
+
Args:
|
|
284
|
+
branch: Default branch for table references without explicit branch.
|
|
285
|
+
"""
|
|
286
|
+
self._default_branch = branch
|
|
287
|
+
self._conn: Optional[duckdb.DuckDBPyConnection] = None
|
|
288
|
+
self._attached_branches: set[str] = set()
|
|
289
|
+
self._registered_aliases: dict[str, str] = {} # alias -> full table path
|
|
290
|
+
|
|
291
|
+
def _ensure_connection(self) -> duckdb.DuckDBPyConnection:
|
|
292
|
+
"""Lazily create DuckDB connection with Iceberg extension."""
|
|
293
|
+
if self._conn is None:
|
|
294
|
+
self._conn = duckdb.connect()
|
|
295
|
+
self._conn.execute("INSTALL iceberg; LOAD iceberg;")
|
|
296
|
+
self._configure_storage(self._conn)
|
|
297
|
+
return self._conn
|
|
298
|
+
|
|
299
|
+
@staticmethod
|
|
300
|
+
def _configure_storage(conn: duckdb.DuckDBPyConnection) -> None:
|
|
301
|
+
"""Configure DuckDB storage secrets for data file access.
|
|
302
|
+
|
|
303
|
+
DuckDB's Iceberg extension reads data/metadata files directly
|
|
304
|
+
from object storage. The Iceberg REST catalog does not vend
|
|
305
|
+
storage credentials to DuckDB, so explicit secrets are needed.
|
|
306
|
+
"""
|
|
307
|
+
settings = get_settings()
|
|
308
|
+
if settings.storage_backend == "gcs" and settings.gcs_endpoint:
|
|
309
|
+
QuerySession._register_fake_gcs_filesystem(conn)
|
|
310
|
+
elif settings.storage_backend == "gcs":
|
|
311
|
+
conn.execute("INSTALL httpfs; LOAD httpfs;")
|
|
312
|
+
conn.execute("""
|
|
313
|
+
CREATE OR REPLACE SECRET gcs_secret (
|
|
314
|
+
TYPE gcs,
|
|
315
|
+
PROVIDER credential_chain
|
|
316
|
+
)
|
|
317
|
+
""")
|
|
318
|
+
elif settings.storage_backend == "s3" and settings.s3_access_key:
|
|
319
|
+
endpoint = settings.s3_endpoint.replace("http://", "").replace("https://", "")
|
|
320
|
+
use_ssl = settings.s3_endpoint.startswith("https://")
|
|
321
|
+
conn.execute("INSTALL httpfs; LOAD httpfs;")
|
|
322
|
+
conn.execute(f"SET s3_endpoint = '{endpoint}'")
|
|
323
|
+
conn.execute(f"SET s3_access_key_id = '{settings.s3_access_key}'")
|
|
324
|
+
conn.execute(f"SET s3_secret_access_key = '{settings.s3_secret_key}'")
|
|
325
|
+
conn.execute("SET s3_region = 'us-east-1'")
|
|
326
|
+
conn.execute("SET s3_url_style = 'path'")
|
|
327
|
+
conn.execute(f"SET s3_use_ssl = {str(use_ssl).lower()}")
|
|
328
|
+
|
|
329
|
+
@staticmethod
|
|
330
|
+
def _register_fake_gcs_filesystem(conn: duckdb.DuckDBPyConnection) -> None:
|
|
331
|
+
"""Register fake-gcs-server through gcsfs for DuckDB scans.
|
|
332
|
+
|
|
333
|
+
DuckDB's native GCS secret talks to the S3-compatible GCS XML API.
|
|
334
|
+
fake-gcs-server emulates the Google JSON API instead, which gcsfs
|
|
335
|
+
supports through STORAGE_EMULATOR_HOST/endpoint_url.
|
|
336
|
+
"""
|
|
337
|
+
import gcsfs
|
|
338
|
+
|
|
339
|
+
settings = get_settings()
|
|
340
|
+
kwargs: dict[str, str] = {
|
|
341
|
+
"endpoint_url": settings.gcs_endpoint,
|
|
342
|
+
"token": "anon",
|
|
343
|
+
}
|
|
344
|
+
if settings.gcs_project_id:
|
|
345
|
+
kwargs["project"] = settings.gcs_project_id
|
|
346
|
+
|
|
347
|
+
os.environ["STORAGE_EMULATOR_HOST"] = settings.gcs_endpoint
|
|
348
|
+
conn.register_filesystem(gcsfs.GCSFileSystem(**kwargs))
|
|
349
|
+
|
|
350
|
+
def _ensure_branch_attached(self, branch: str) -> str:
|
|
351
|
+
"""Ensure branch is attached as catalog, return catalog alias.
|
|
352
|
+
|
|
353
|
+
Args:
|
|
354
|
+
branch: Branch name. Only "main" is reachable from DuckDB.
|
|
355
|
+
|
|
356
|
+
Returns:
|
|
357
|
+
Catalog alias (always iceberg_main).
|
|
358
|
+
|
|
359
|
+
Raises:
|
|
360
|
+
ValueError: branch is not "main".
|
|
361
|
+
|
|
362
|
+
Note:
|
|
363
|
+
iceberg-catalog selects a branch through the REST prefix: a client
|
|
364
|
+
configured at /iceberg/{branch}/ gets defaults.prefix={branch} and
|
|
365
|
+
overrides.uri=/iceberg/ back from the config call, then re-routes
|
|
366
|
+
itself. PyIceberg does that, so catalog.py reads any branch.
|
|
367
|
+
DuckDB's Iceberg extension ignores the uri override and appends
|
|
368
|
+
v1/{prefix}/ to the ENDPOINT it was given, which yields
|
|
369
|
+
/iceberg/{branch}/v1/{branch}/... and 404s. ENDPOINT is its only
|
|
370
|
+
lever — it rejects PREFIX and WAREHOUSE options — so DuckDB can
|
|
371
|
+
only reach the catalog's default branch. Raising here stops a
|
|
372
|
+
non-main ref from silently reading main's rows.
|
|
373
|
+
"""
|
|
374
|
+
if branch != _DUCKDB_BRANCH:
|
|
375
|
+
raise ValueError(
|
|
376
|
+
f"cannot query branch {branch!r} through DuckDB: the Iceberg extension "
|
|
377
|
+
f"only reaches the catalog's default branch ({_DUCKDB_BRANCH!r})"
|
|
378
|
+
)
|
|
379
|
+
if branch in self._attached_branches:
|
|
380
|
+
return f"iceberg_{branch}"
|
|
381
|
+
|
|
382
|
+
settings = get_settings()
|
|
383
|
+
validate_ref_name(branch, "branch")
|
|
384
|
+
|
|
385
|
+
endpoint = f"{settings.catalog_uri}/iceberg/"
|
|
386
|
+
catalog_alias = f"iceberg_{branch}"
|
|
387
|
+
quoted_alias = quote_identifier(catalog_alias)
|
|
388
|
+
token = _resolve_token(settings)
|
|
389
|
+
|
|
390
|
+
conn = self._ensure_connection()
|
|
391
|
+
if token is not None:
|
|
392
|
+
conn.execute(
|
|
393
|
+
f"""
|
|
394
|
+
ATTACH IF NOT EXISTS 'warehouse' AS {quoted_alias} (
|
|
395
|
+
TYPE iceberg,
|
|
396
|
+
ENDPOINT ?,
|
|
397
|
+
AUTHORIZATION_TYPE 'oauth2',
|
|
398
|
+
TOKEN ?
|
|
399
|
+
)
|
|
400
|
+
""",
|
|
401
|
+
[endpoint, token],
|
|
402
|
+
)
|
|
403
|
+
else:
|
|
404
|
+
conn.execute(
|
|
405
|
+
f"""
|
|
406
|
+
ATTACH IF NOT EXISTS 'warehouse' AS {quoted_alias} (
|
|
407
|
+
TYPE iceberg,
|
|
408
|
+
ENDPOINT ?,
|
|
409
|
+
AUTHORIZATION_TYPE 'none'
|
|
410
|
+
)
|
|
411
|
+
""",
|
|
412
|
+
[endpoint],
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
self._attached_branches.add(branch)
|
|
416
|
+
return catalog_alias
|
|
417
|
+
|
|
418
|
+
def table(self, ref: str) -> duckdb.DuckDBPyRelation:
|
|
419
|
+
"""Get DuckDB relation for table reference with time travel support.
|
|
420
|
+
|
|
421
|
+
Args:
|
|
422
|
+
ref: Table reference (e.g., "ns.table", "ns.table@v1.0", "ns.table@branch")
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
DuckDBPyRelation for lazy query construction.
|
|
426
|
+
"""
|
|
427
|
+
relation, _ = self._table(TableRef.parse(ref))
|
|
428
|
+
return relation
|
|
429
|
+
|
|
430
|
+
def _table(self, table_ref: TableRef) -> tuple[duckdb.DuckDBPyRelation, TableRef]:
|
|
431
|
+
"""Build a relation and retain the branch selected for metadata reads."""
|
|
432
|
+
from sunstone_data.resolve import Resolver
|
|
433
|
+
|
|
434
|
+
resolved_ref = Resolver().resolve_ref(table_ref)
|
|
435
|
+
catalog_alias = self._ensure_branch_attached(resolved_ref.branch)
|
|
436
|
+
|
|
437
|
+
# Build quoted full table path
|
|
438
|
+
quoted_catalog = quote_identifier(catalog_alias)
|
|
439
|
+
quoted_ns = quote_identifier(table_ref.namespace)
|
|
440
|
+
quoted_table = quote_identifier(table_ref.table)
|
|
441
|
+
full_path = f"{quoted_catalog}.{quoted_ns}.{quoted_table}"
|
|
442
|
+
|
|
443
|
+
conn = self._ensure_connection()
|
|
444
|
+
|
|
445
|
+
if resolved_ref.snapshot_id is not None:
|
|
446
|
+
sql = f"SELECT * FROM {full_path} AT (VERSION => {resolved_ref.snapshot_id})"
|
|
447
|
+
else:
|
|
448
|
+
sql = f"SELECT * FROM {full_path}"
|
|
449
|
+
|
|
450
|
+
resolved_table_ref = replace(table_ref, branch=resolved_ref.branch)
|
|
451
|
+
return conn.sql(sql), resolved_table_ref
|
|
452
|
+
|
|
453
|
+
def register(self, alias: str, ref: str) -> None:
|
|
454
|
+
"""Register table with alias for SQL queries.
|
|
455
|
+
|
|
456
|
+
Args:
|
|
457
|
+
alias: SQL-friendly name to use in queries.
|
|
458
|
+
ref: Table reference (e.g., "ns.table@v1.0").
|
|
459
|
+
"""
|
|
460
|
+
relation = self.table(ref)
|
|
461
|
+
conn = self._ensure_connection()
|
|
462
|
+
conn.register(alias, relation)
|
|
463
|
+
self._registered_aliases[alias] = ref
|
|
464
|
+
|
|
465
|
+
def sql(self, query: str) -> duckdb.DuckDBPyRelation:
|
|
466
|
+
"""Execute SQL query against registered tables.
|
|
467
|
+
|
|
468
|
+
Args:
|
|
469
|
+
query: SQL query string using registered aliases.
|
|
470
|
+
|
|
471
|
+
Returns:
|
|
472
|
+
DuckDBPyRelation for lazy query construction.
|
|
473
|
+
"""
|
|
474
|
+
conn = self._ensure_connection()
|
|
475
|
+
return conn.sql(query)
|
|
476
|
+
|
|
477
|
+
def list_tables(self, namespace: str | None = None) -> list[dict]:
|
|
478
|
+
"""List available tables in the catalog.
|
|
479
|
+
|
|
480
|
+
Args:
|
|
481
|
+
namespace: Optional namespace filter.
|
|
482
|
+
|
|
483
|
+
Returns:
|
|
484
|
+
List of table info dicts.
|
|
485
|
+
"""
|
|
486
|
+
from sunstone_data.catalog import list_tables_with_metadata
|
|
487
|
+
|
|
488
|
+
return list_tables_with_metadata(namespace)
|
|
489
|
+
|
|
490
|
+
def preview(self, ref: str) -> TablePreview:
|
|
491
|
+
"""Get TablePreview wrapper for inspection and preview operations.
|
|
492
|
+
|
|
493
|
+
Args:
|
|
494
|
+
ref: Table reference (e.g., "ns.table", "ns.table@v1.0").
|
|
495
|
+
|
|
496
|
+
Returns:
|
|
497
|
+
TablePreview with schema(), head(), tail(), sample(), describe() methods.
|
|
498
|
+
|
|
499
|
+
Example:
|
|
500
|
+
>>> session = get_session()
|
|
501
|
+
>>> preview = session.preview("sources.patents")
|
|
502
|
+
>>> print(preview.schema())
|
|
503
|
+
>>> preview.head(10).fetchall()
|
|
504
|
+
>>> preview.sample(n=100).to_df()
|
|
505
|
+
"""
|
|
506
|
+
relation, table_ref = self._table(TableRef.parse(ref))
|
|
507
|
+
return TablePreview(_relation=relation, _table_ref=table_ref, _session=self)
|
|
508
|
+
|
|
509
|
+
def close(self) -> None:
|
|
510
|
+
"""Close DuckDB connection."""
|
|
511
|
+
if self._conn is not None:
|
|
512
|
+
self._conn.close()
|
|
513
|
+
self._conn = None
|
|
514
|
+
self._attached_branches.clear()
|
|
515
|
+
self._registered_aliases.clear()
|
|
516
|
+
|
|
517
|
+
def __enter__(self) -> QuerySession:
|
|
518
|
+
return self
|
|
519
|
+
|
|
520
|
+
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
|
|
521
|
+
self.close()
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def get_connection(branch: str = "main") -> duckdb.DuckDBPyConnection:
|
|
525
|
+
"""Create DuckDB connection attached to the Iceberg REST catalog.
|
|
526
|
+
|
|
527
|
+
Args:
|
|
528
|
+
branch: Branch to connect to. Only "main" is reachable from DuckDB —
|
|
529
|
+
see QuerySession._ensure_branch_attached for why.
|
|
530
|
+
|
|
531
|
+
Returns:
|
|
532
|
+
duckdb.DuckDBPyConnection: DuckDB connection with the Iceberg catalog attached.
|
|
533
|
+
|
|
534
|
+
Raises:
|
|
535
|
+
ValueError: branch is not "main".
|
|
536
|
+
"""
|
|
537
|
+
if branch != _DUCKDB_BRANCH:
|
|
538
|
+
raise ValueError(
|
|
539
|
+
f"cannot connect to branch {branch!r} through DuckDB: the Iceberg extension "
|
|
540
|
+
f"only reaches the catalog's default branch ({_DUCKDB_BRANCH!r})"
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
settings = get_settings()
|
|
544
|
+
|
|
545
|
+
conn = duckdb.connect()
|
|
546
|
+
|
|
547
|
+
# Install and load iceberg extension
|
|
548
|
+
conn.execute("INSTALL iceberg; LOAD iceberg;")
|
|
549
|
+
QuerySession._configure_storage(conn)
|
|
550
|
+
|
|
551
|
+
# Attach the Iceberg REST catalog. An unprefixed /iceberg/ endpoint gets
|
|
552
|
+
# the catalog's default branch as the REST prefix.
|
|
553
|
+
endpoint = f"{settings.catalog_uri}/iceberg/"
|
|
554
|
+
token = _resolve_token(settings)
|
|
555
|
+
if token is not None:
|
|
556
|
+
conn.execute(
|
|
557
|
+
"""
|
|
558
|
+
ATTACH IF NOT EXISTS 'warehouse' AS iceberg (
|
|
559
|
+
TYPE iceberg,
|
|
560
|
+
ENDPOINT ?,
|
|
561
|
+
AUTHORIZATION_TYPE 'oauth2',
|
|
562
|
+
TOKEN ?
|
|
563
|
+
)
|
|
564
|
+
""",
|
|
565
|
+
[endpoint, token],
|
|
566
|
+
)
|
|
567
|
+
else:
|
|
568
|
+
conn.execute(
|
|
569
|
+
"""
|
|
570
|
+
ATTACH IF NOT EXISTS 'warehouse' AS iceberg (
|
|
571
|
+
TYPE iceberg,
|
|
572
|
+
ENDPOINT ?,
|
|
573
|
+
AUTHORIZATION_TYPE 'none'
|
|
574
|
+
)
|
|
575
|
+
""",
|
|
576
|
+
[endpoint],
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
return conn
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def list_tables(conn: duckdb.DuckDBPyConnection) -> list[tuple]:
|
|
583
|
+
"""List all tables in the attached Iceberg catalog.
|
|
584
|
+
|
|
585
|
+
Args:
|
|
586
|
+
conn: DuckDB connection with the Iceberg catalog attached.
|
|
587
|
+
|
|
588
|
+
Returns:
|
|
589
|
+
list[tuple]: List of table information tuples.
|
|
590
|
+
"""
|
|
591
|
+
return conn.execute("SHOW ALL TABLES").fetchall()
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
# Thread-local singleton for notebook convenience
|
|
595
|
+
_thread_local = threading.local()
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
def get_session(branch: str = "main") -> QuerySession:
|
|
599
|
+
"""Get or create thread-local query session (notebook convenience).
|
|
600
|
+
|
|
601
|
+
This provides a singleton QuerySession per thread, which is useful
|
|
602
|
+
for interactive notebook usage where you don't want to manage
|
|
603
|
+
connection lifecycle manually.
|
|
604
|
+
|
|
605
|
+
Args:
|
|
606
|
+
branch: Default branch for new sessions.
|
|
607
|
+
|
|
608
|
+
Returns:
|
|
609
|
+
Thread-local QuerySession instance.
|
|
610
|
+
"""
|
|
611
|
+
if not hasattr(_thread_local, "session") or _thread_local.session is None:
|
|
612
|
+
_thread_local.session = QuerySession(branch=branch)
|
|
613
|
+
return _thread_local.session
|
|
614
|
+
|
|
615
|
+
|
|
616
|
+
def close_session() -> None:
|
|
617
|
+
"""Close thread-local session if exists.
|
|
618
|
+
|
|
619
|
+
Call this to clean up the singleton session, for example when
|
|
620
|
+
restarting notebook kernel or shutting down application.
|
|
621
|
+
"""
|
|
622
|
+
if hasattr(_thread_local, "session") and _thread_local.session is not None:
|
|
623
|
+
_thread_local.session.close()
|
|
624
|
+
_thread_local.session = None
|