sunstone-data-platform 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. sunstone_data/__init__.py +36 -0
  2. sunstone_data/_version.py +24 -0
  3. sunstone_data/auth/__init__.py +43 -0
  4. sunstone_data/auth/client_creds.py +239 -0
  5. sunstone_data/auth/pkce.py +307 -0
  6. sunstone_data/auth/store.py +97 -0
  7. sunstone_data/catalog.py +175 -0
  8. sunstone_data/cli/__init__.py +5 -0
  9. sunstone_data/cli/_app.py +26 -0
  10. sunstone_data/cli/_errors.py +31 -0
  11. sunstone_data/cli/commands/__init__.py +22 -0
  12. sunstone_data/cli/commands/data_auth.py +120 -0
  13. sunstone_data/cli/commands/data_import.py +157 -0
  14. sunstone_data/cli/commands/data_info.py +166 -0
  15. sunstone_data/cli/commands/data_publish.py +69 -0
  16. sunstone_data/cli/commands/ls.py +49 -0
  17. sunstone_data/cli/commands/prov.py +58 -0
  18. sunstone_data/cli/commands/refs.py +142 -0
  19. sunstone_data/cli/commands/show.py +139 -0
  20. sunstone_data/cli/commands/snapshots.py +77 -0
  21. sunstone_data/cli/commands/sparql.py +68 -0
  22. sunstone_data/cli/main.py +30 -0
  23. sunstone_data/config.py +250 -0
  24. sunstone_data/discovery.py +317 -0
  25. sunstone_data/env_commands.py +97 -0
  26. sunstone_data/env_section.py +86 -0
  27. sunstone_data/graph/__init__.py +36 -0
  28. sunstone_data/graph/client.py +328 -0
  29. sunstone_data/graph/dataset.py +438 -0
  30. sunstone_data/graph/terms.py +34 -0
  31. sunstone_data/identifiers.py +155 -0
  32. sunstone_data/import_/__init__.py +28 -0
  33. sunstone_data/import_/fetcher.py +259 -0
  34. sunstone_data/import_/owner.py +130 -0
  35. sunstone_data/import_/parser.py +492 -0
  36. sunstone_data/import_/pipeline.py +275 -0
  37. sunstone_data/import_/storage.py +135 -0
  38. sunstone_data/metadata/__init__.py +41 -0
  39. sunstone_data/metadata/builder.py +276 -0
  40. sunstone_data/metadata/models.py +87 -0
  41. sunstone_data/metadata/provenance.py +171 -0
  42. sunstone_data/metadata/provenance_rdf.py +102 -0
  43. sunstone_data/plugin.py +62 -0
  44. sunstone_data/publish.py +134 -0
  45. sunstone_data/query.py +624 -0
  46. sunstone_data/resolve.py +178 -0
  47. sunstone_data/url.py +78 -0
  48. sunstone_data/utils/__init__.py +12 -0
  49. sunstone_data/utils/checksum.py +14 -0
  50. sunstone_data/utils/encoding.py +16 -0
  51. sunstone_data/utils/naming.py +38 -0
  52. sunstone_data/versioning/__init__.py +17 -0
  53. sunstone_data/versioning/schema_evolution.py +180 -0
  54. sunstone_data_platform-1.0.0.dist-info/METADATA +302 -0
  55. sunstone_data_platform-1.0.0.dist-info/RECORD +66 -0
  56. sunstone_data_platform-1.0.0.dist-info/WHEEL +4 -0
  57. sunstone_data_platform-1.0.0.dist-info/entry_points.txt +2 -0
  58. sunstone_pipeline/__init__.py +1 -0
  59. sunstone_pipeline/assets/__init__.py +11 -0
  60. sunstone_pipeline/assets/mirrors.py +123 -0
  61. sunstone_pipeline/assets/transforms.py +74 -0
  62. sunstone_pipeline/definitions.py +25 -0
  63. sunstone_pipeline/io_manager.py +56 -0
  64. sunstone_pipeline/resources/__init__.py +6 -0
  65. sunstone_pipeline/resources/iceberg.py +29 -0
  66. sunstone_pipeline/resources/refs.py +88 -0
sunstone_data/query.py ADDED
@@ -0,0 +1,624 @@
1
+ """DuckDB query interface for the Iceberg REST catalog.
2
+
3
+ This module provides:
4
+ - QuerySession: Context manager for DuckDB queries with version awareness
5
+ - TableRef: Git-like table reference parsing (ns.table@branch, ns.table@v1.0)
6
+ - TablePreview: Pandas-style preview methods (head, tail, sample, describe)
7
+ - get_session()/close_session(): Thread-local singleton for notebook convenience
8
+
9
+ Example (notebook):
10
+ from sunstone_data.query import get_session
11
+
12
+ session = get_session()
13
+ rel = session.table("sources.patents@main")
14
+ df = rel.filter("year >= 2020").limit(100).df()
15
+
16
+ Example (application):
17
+ from sunstone_data.query import QuerySession
18
+
19
+ with QuerySession() as session:
20
+ preview = session.preview("sources.patents@v1.0")
21
+ print(preview.schema())
22
+ df = preview.head(10).df()
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import os
28
+ import threading
29
+ from dataclasses import dataclass, replace
30
+ from typing import Any, Optional
31
+
32
+ import duckdb
33
+ import httpx
34
+
35
+ from sunstone_data.config import Settings, current_environment_name, get_settings
36
+ from sunstone_data.identifiers import (
37
+ quote_identifier,
38
+ validate_ref_name,
39
+ validate_ref_string,
40
+ validate_sql_identifier,
41
+ )
42
+
43
+ # The only branch DuckDB can read; see QuerySession._ensure_branch_attached.
44
+ _DUCKDB_BRANCH = "main"
45
+
46
+
47
+ def _resolve_token(settings: Settings) -> str | None:
48
+ """Return a bearer token for iceberg-catalog, or None if none is configured.
49
+
50
+ Mirrors sunstone_data.catalog._resolve_token: iceberg-catalog's auth
51
+ middleware fails closed on authentication regardless of AUTH_ENFORCE, so
52
+ DuckDB's catalog attach needs a real token when one is configured.
53
+ """
54
+ from sunstone_data.auth.client_creds import resolve_access_token
55
+
56
+ with httpx.Client() as client:
57
+ return resolve_access_token(settings, current_environment_name(), client)
58
+
59
+
60
+ @dataclass
61
+ class TableRef:
62
+ """Parsed table reference with optional version specifier.
63
+
64
+ Supports git-like syntax, with multi-level namespaces (e.g.
65
+ "un.health.cases"):
66
+ - "namespace.table" -> latest on main branch
67
+ - "namespace.table@ref" -> the given ref, carried verbatim
68
+
69
+ The `@ref` part is not classified as a branch, named version, snapshot id,
70
+ or timestamp at parse time. `Resolver` asks graph-server which refs are
71
+ branches, then resolves every other ref against the table. `@ref` is stored
72
+ as `version_ref` unchanged; `branch` stays at its default ("main") unless
73
+ the caller sets it directly.
74
+ """
75
+
76
+ namespace: str
77
+ table: str
78
+ branch: str = "main"
79
+ version_ref: Optional[str] = None # ref string, carried verbatim
80
+
81
+ @classmethod
82
+ def parse(cls, ref_string: str) -> TableRef:
83
+ """Parse table reference string.
84
+
85
+ The namespace may have multiple dot-separated levels; the last
86
+ level before `@ref` (or the end of the string) is the table name.
87
+ A ref after `@` is carried verbatim into `version_ref` — it is not
88
+ classified as a branch/tag/snapshot id/timestamp here.
89
+
90
+ Examples:
91
+ "ns.table" -> TableRef(namespace="ns", table="table")
92
+ "ns.sub.table" -> TableRef(namespace="ns.sub", table="table")
93
+ "ns.table@main" -> TableRef(namespace="ns", table="table", version_ref="main")
94
+ "ns.table@v1.0" -> TableRef(namespace="ns", table="table", version_ref="v1.0")
95
+ "ns.table@2026-01-25" -> TableRef(..., version_ref="2026-01-25")
96
+
97
+ Args:
98
+ ref_string: Table reference string to parse.
99
+
100
+ Returns:
101
+ TableRef with parsed components.
102
+
103
+ Raises:
104
+ ValueError: If table reference format is invalid.
105
+ """
106
+ # Split on @ for version reference
107
+ if "@" in ref_string:
108
+ table_part, ref_part = ref_string.split("@", 1)
109
+ else:
110
+ table_part = ref_string
111
+ ref_part = None
112
+
113
+ # Parse namespace.table: the last dot separates table from namespace.
114
+ if "." not in table_part:
115
+ raise ValueError(f"Invalid table reference: {ref_string}. Expected format: namespace.table[@version]")
116
+
117
+ namespace, table = table_part.rsplit(".", 1)
118
+
119
+ # Validate namespace and table names. Each namespace level is
120
+ # validated individually since a level cannot itself contain a dot.
121
+ for level in namespace.split("."):
122
+ validate_sql_identifier(level, "namespace")
123
+ validate_sql_identifier(table, "table")
124
+
125
+ # Carry the ref verbatim; Resolver classifies it when the table is read.
126
+ # validate_ref_string (not validate_ref_name) is used here since a
127
+ # ref may be a snapshot id or timestamp starting with a digit, which
128
+ # validate_ref_name's branch/tag naming pattern would reject.
129
+ if ref_part:
130
+ validate_ref_string(ref_part, "ref")
131
+ return cls(namespace=namespace, table=table, version_ref=ref_part)
132
+
133
+ return cls(namespace=namespace, table=table)
134
+
135
+
136
+ @dataclass
137
+ class TablePreview:
138
+ """Preview wrapper for DuckDB relation with pandas-style methods.
139
+
140
+ All methods return DuckDBPyRelation by default for lazy evaluation.
141
+ Call .to_df() or .fetchall() to materialize results.
142
+ """
143
+
144
+ _relation: duckdb.DuckDBPyRelation
145
+ _table_ref: TableRef
146
+ _session: "QuerySession"
147
+
148
+ def schema(self) -> dict[str, Any]:
149
+ """Get detailed schema information including Iceberg metadata.
150
+
151
+ Returns:
152
+ Dict with 'columns' list (name, type, nullable, field_id, iceberg_type)
153
+ and optional 'row_count' estimate.
154
+ """
155
+ # Get DuckDB schema from relation
156
+ columns = []
157
+ for name, dtype in zip(self._relation.columns, self._relation.dtypes):
158
+ columns.append(
159
+ {
160
+ "name": name,
161
+ "type": str(dtype),
162
+ "nullable": True, # DuckDB doesn't expose directly
163
+ }
164
+ )
165
+
166
+ # Enhance with Iceberg metadata if available
167
+ try:
168
+ from sunstone_data.catalog import get_catalog
169
+
170
+ catalog = get_catalog(branch=self._table_ref.branch)
171
+ iceberg_table = catalog.load_table(f"{self._table_ref.namespace}.{self._table_ref.table}")
172
+ iceberg_schema = iceberg_table.schema()
173
+ field_map = {f.name: f for f in iceberg_schema.fields}
174
+
175
+ for col in columns:
176
+ if col["name"] in field_map:
177
+ field = field_map[col["name"]]
178
+ col["field_id"] = field.field_id
179
+ col["nullable"] = not field.required
180
+ col["iceberg_type"] = str(field.field_type)
181
+ except Exception:
182
+ pass # Iceberg metadata optional
183
+
184
+ return {"columns": columns}
185
+
186
+ def head(self, n: int = 5) -> duckdb.DuckDBPyRelation:
187
+ """Get first n rows (lazy). Default n=5 (pandas style)."""
188
+ return self._relation.limit(n)
189
+
190
+ def tail(self, n: int = 5) -> duckdb.DuckDBPyRelation:
191
+ """Get last n rows (lazy). Note: requires full scan for offset calculation."""
192
+ conn = self._session._ensure_connection()
193
+ view_name = f"_tail_temp_{id(self._relation)}"
194
+ conn.register(view_name, self._relation)
195
+
196
+ count = conn.sql(f"SELECT COUNT(*) FROM {view_name}").fetchone()[0]
197
+ offset = max(0, count - n)
198
+
199
+ return conn.sql(f"SELECT * FROM {view_name} LIMIT {n} OFFSET {offset}")
200
+
201
+ def sample(
202
+ self,
203
+ n: int | None = None,
204
+ frac: float | None = None,
205
+ method: str = "reservoir",
206
+ ) -> duckdb.DuckDBPyRelation:
207
+ """Get random sample of rows (lazy).
208
+
209
+ Args:
210
+ n: Number of rows to sample (mutually exclusive with frac).
211
+ frac: Fraction of rows to sample, 0.0-1.0 (mutually exclusive with n).
212
+ method: Sampling method - "reservoir" (exact count, default),
213
+ "bernoulli" (row-level probability), "system" (block-level).
214
+
215
+ Returns:
216
+ DuckDBPyRelation for lazy evaluation.
217
+
218
+ Raises:
219
+ ValueError: If neither n nor frac specified, or both specified.
220
+ """
221
+ if (n is None) == (frac is None):
222
+ raise ValueError("Specify exactly one of n or frac")
223
+
224
+ conn = self._session._ensure_connection()
225
+ view_name = f"_sample_temp_{id(self._relation)}"
226
+ conn.register(view_name, self._relation)
227
+
228
+ if n is not None:
229
+ return conn.sql(f"SELECT * FROM {view_name} USING SAMPLE {n} ROWS ({method})")
230
+ else:
231
+ pct = frac * 100
232
+ return conn.sql(f"SELECT * FROM {view_name} USING SAMPLE {pct}% ({method})")
233
+
234
+ def describe(self) -> duckdb.DuckDBPyRelation:
235
+ """Get summary statistics for all columns (lazy).
236
+
237
+ Uses DuckDB's SUMMARIZE for comprehensive stats:
238
+ column_name, column_type, min, max, approx_unique, avg, std, q25, q50, q75,
239
+ count, null_percentage
240
+
241
+ Returns:
242
+ DuckDBPyRelation with statistics.
243
+ """
244
+ conn = self._session._ensure_connection()
245
+ view_name = f"_describe_temp_{id(self._relation)}"
246
+ conn.register(view_name, self._relation)
247
+
248
+ return conn.sql(f"SUMMARIZE SELECT * FROM {view_name}")
249
+
250
+ def to_df(self):
251
+ """Convert to pandas DataFrame (materializes data).
252
+
253
+ Warning: Large results will consume significant memory.
254
+ Consider using .limit() before .to_df() for large tables.
255
+ """
256
+ return self._relation.df()
257
+
258
+ def fetchall(self) -> list[tuple]:
259
+ """Fetch all rows as list of tuples (materializes data)."""
260
+ return self._relation.fetchall()
261
+
262
+ def __repr__(self) -> str:
263
+ return f"TablePreview({self._table_ref.namespace}.{self._table_ref.table})"
264
+
265
+
266
+ class QuerySession:
267
+ """DuckDB query session with version-aware table registration.
268
+
269
+ Usage as context manager (applications):
270
+ with QuerySession() as session:
271
+ rel = session.table("ns.table@v1.0")
272
+ results = rel.filter("x > 10").limit(100).fetchall()
273
+
274
+ Usage as singleton (notebooks):
275
+ from sunstone_data.query import get_session
276
+ session = get_session()
277
+ rel = session.table("ns.table@main")
278
+ """
279
+
280
+ def __init__(self, branch: str = "main"):
281
+ """Initialize query session.
282
+
283
+ Args:
284
+ branch: Default branch for table references without explicit branch.
285
+ """
286
+ self._default_branch = branch
287
+ self._conn: Optional[duckdb.DuckDBPyConnection] = None
288
+ self._attached_branches: set[str] = set()
289
+ self._registered_aliases: dict[str, str] = {} # alias -> full table path
290
+
291
+ def _ensure_connection(self) -> duckdb.DuckDBPyConnection:
292
+ """Lazily create DuckDB connection with Iceberg extension."""
293
+ if self._conn is None:
294
+ self._conn = duckdb.connect()
295
+ self._conn.execute("INSTALL iceberg; LOAD iceberg;")
296
+ self._configure_storage(self._conn)
297
+ return self._conn
298
+
299
+ @staticmethod
300
+ def _configure_storage(conn: duckdb.DuckDBPyConnection) -> None:
301
+ """Configure DuckDB storage secrets for data file access.
302
+
303
+ DuckDB's Iceberg extension reads data/metadata files directly
304
+ from object storage. The Iceberg REST catalog does not vend
305
+ storage credentials to DuckDB, so explicit secrets are needed.
306
+ """
307
+ settings = get_settings()
308
+ if settings.storage_backend == "gcs" and settings.gcs_endpoint:
309
+ QuerySession._register_fake_gcs_filesystem(conn)
310
+ elif settings.storage_backend == "gcs":
311
+ conn.execute("INSTALL httpfs; LOAD httpfs;")
312
+ conn.execute("""
313
+ CREATE OR REPLACE SECRET gcs_secret (
314
+ TYPE gcs,
315
+ PROVIDER credential_chain
316
+ )
317
+ """)
318
+ elif settings.storage_backend == "s3" and settings.s3_access_key:
319
+ endpoint = settings.s3_endpoint.replace("http://", "").replace("https://", "")
320
+ use_ssl = settings.s3_endpoint.startswith("https://")
321
+ conn.execute("INSTALL httpfs; LOAD httpfs;")
322
+ conn.execute(f"SET s3_endpoint = '{endpoint}'")
323
+ conn.execute(f"SET s3_access_key_id = '{settings.s3_access_key}'")
324
+ conn.execute(f"SET s3_secret_access_key = '{settings.s3_secret_key}'")
325
+ conn.execute("SET s3_region = 'us-east-1'")
326
+ conn.execute("SET s3_url_style = 'path'")
327
+ conn.execute(f"SET s3_use_ssl = {str(use_ssl).lower()}")
328
+
329
+ @staticmethod
330
+ def _register_fake_gcs_filesystem(conn: duckdb.DuckDBPyConnection) -> None:
331
+ """Register fake-gcs-server through gcsfs for DuckDB scans.
332
+
333
+ DuckDB's native GCS secret talks to the S3-compatible GCS XML API.
334
+ fake-gcs-server emulates the Google JSON API instead, which gcsfs
335
+ supports through STORAGE_EMULATOR_HOST/endpoint_url.
336
+ """
337
+ import gcsfs
338
+
339
+ settings = get_settings()
340
+ kwargs: dict[str, str] = {
341
+ "endpoint_url": settings.gcs_endpoint,
342
+ "token": "anon",
343
+ }
344
+ if settings.gcs_project_id:
345
+ kwargs["project"] = settings.gcs_project_id
346
+
347
+ os.environ["STORAGE_EMULATOR_HOST"] = settings.gcs_endpoint
348
+ conn.register_filesystem(gcsfs.GCSFileSystem(**kwargs))
349
+
350
+ def _ensure_branch_attached(self, branch: str) -> str:
351
+ """Ensure branch is attached as catalog, return catalog alias.
352
+
353
+ Args:
354
+ branch: Branch name. Only "main" is reachable from DuckDB.
355
+
356
+ Returns:
357
+ Catalog alias (always iceberg_main).
358
+
359
+ Raises:
360
+ ValueError: branch is not "main".
361
+
362
+ Note:
363
+ iceberg-catalog selects a branch through the REST prefix: a client
364
+ configured at /iceberg/{branch}/ gets defaults.prefix={branch} and
365
+ overrides.uri=/iceberg/ back from the config call, then re-routes
366
+ itself. PyIceberg does that, so catalog.py reads any branch.
367
+ DuckDB's Iceberg extension ignores the uri override and appends
368
+ v1/{prefix}/ to the ENDPOINT it was given, which yields
369
+ /iceberg/{branch}/v1/{branch}/... and 404s. ENDPOINT is its only
370
+ lever — it rejects PREFIX and WAREHOUSE options — so DuckDB can
371
+ only reach the catalog's default branch. Raising here stops a
372
+ non-main ref from silently reading main's rows.
373
+ """
374
+ if branch != _DUCKDB_BRANCH:
375
+ raise ValueError(
376
+ f"cannot query branch {branch!r} through DuckDB: the Iceberg extension "
377
+ f"only reaches the catalog's default branch ({_DUCKDB_BRANCH!r})"
378
+ )
379
+ if branch in self._attached_branches:
380
+ return f"iceberg_{branch}"
381
+
382
+ settings = get_settings()
383
+ validate_ref_name(branch, "branch")
384
+
385
+ endpoint = f"{settings.catalog_uri}/iceberg/"
386
+ catalog_alias = f"iceberg_{branch}"
387
+ quoted_alias = quote_identifier(catalog_alias)
388
+ token = _resolve_token(settings)
389
+
390
+ conn = self._ensure_connection()
391
+ if token is not None:
392
+ conn.execute(
393
+ f"""
394
+ ATTACH IF NOT EXISTS 'warehouse' AS {quoted_alias} (
395
+ TYPE iceberg,
396
+ ENDPOINT ?,
397
+ AUTHORIZATION_TYPE 'oauth2',
398
+ TOKEN ?
399
+ )
400
+ """,
401
+ [endpoint, token],
402
+ )
403
+ else:
404
+ conn.execute(
405
+ f"""
406
+ ATTACH IF NOT EXISTS 'warehouse' AS {quoted_alias} (
407
+ TYPE iceberg,
408
+ ENDPOINT ?,
409
+ AUTHORIZATION_TYPE 'none'
410
+ )
411
+ """,
412
+ [endpoint],
413
+ )
414
+
415
+ self._attached_branches.add(branch)
416
+ return catalog_alias
417
+
418
+ def table(self, ref: str) -> duckdb.DuckDBPyRelation:
419
+ """Get DuckDB relation for table reference with time travel support.
420
+
421
+ Args:
422
+ ref: Table reference (e.g., "ns.table", "ns.table@v1.0", "ns.table@branch")
423
+
424
+ Returns:
425
+ DuckDBPyRelation for lazy query construction.
426
+ """
427
+ relation, _ = self._table(TableRef.parse(ref))
428
+ return relation
429
+
430
+ def _table(self, table_ref: TableRef) -> tuple[duckdb.DuckDBPyRelation, TableRef]:
431
+ """Build a relation and retain the branch selected for metadata reads."""
432
+ from sunstone_data.resolve import Resolver
433
+
434
+ resolved_ref = Resolver().resolve_ref(table_ref)
435
+ catalog_alias = self._ensure_branch_attached(resolved_ref.branch)
436
+
437
+ # Build quoted full table path
438
+ quoted_catalog = quote_identifier(catalog_alias)
439
+ quoted_ns = quote_identifier(table_ref.namespace)
440
+ quoted_table = quote_identifier(table_ref.table)
441
+ full_path = f"{quoted_catalog}.{quoted_ns}.{quoted_table}"
442
+
443
+ conn = self._ensure_connection()
444
+
445
+ if resolved_ref.snapshot_id is not None:
446
+ sql = f"SELECT * FROM {full_path} AT (VERSION => {resolved_ref.snapshot_id})"
447
+ else:
448
+ sql = f"SELECT * FROM {full_path}"
449
+
450
+ resolved_table_ref = replace(table_ref, branch=resolved_ref.branch)
451
+ return conn.sql(sql), resolved_table_ref
452
+
453
+ def register(self, alias: str, ref: str) -> None:
454
+ """Register table with alias for SQL queries.
455
+
456
+ Args:
457
+ alias: SQL-friendly name to use in queries.
458
+ ref: Table reference (e.g., "ns.table@v1.0").
459
+ """
460
+ relation = self.table(ref)
461
+ conn = self._ensure_connection()
462
+ conn.register(alias, relation)
463
+ self._registered_aliases[alias] = ref
464
+
465
+ def sql(self, query: str) -> duckdb.DuckDBPyRelation:
466
+ """Execute SQL query against registered tables.
467
+
468
+ Args:
469
+ query: SQL query string using registered aliases.
470
+
471
+ Returns:
472
+ DuckDBPyRelation for lazy query construction.
473
+ """
474
+ conn = self._ensure_connection()
475
+ return conn.sql(query)
476
+
477
+ def list_tables(self, namespace: str | None = None) -> list[dict]:
478
+ """List available tables in the catalog.
479
+
480
+ Args:
481
+ namespace: Optional namespace filter.
482
+
483
+ Returns:
484
+ List of table info dicts.
485
+ """
486
+ from sunstone_data.catalog import list_tables_with_metadata
487
+
488
+ return list_tables_with_metadata(namespace)
489
+
490
+ def preview(self, ref: str) -> TablePreview:
491
+ """Get TablePreview wrapper for inspection and preview operations.
492
+
493
+ Args:
494
+ ref: Table reference (e.g., "ns.table", "ns.table@v1.0").
495
+
496
+ Returns:
497
+ TablePreview with schema(), head(), tail(), sample(), describe() methods.
498
+
499
+ Example:
500
+ >>> session = get_session()
501
+ >>> preview = session.preview("sources.patents")
502
+ >>> print(preview.schema())
503
+ >>> preview.head(10).fetchall()
504
+ >>> preview.sample(n=100).to_df()
505
+ """
506
+ relation, table_ref = self._table(TableRef.parse(ref))
507
+ return TablePreview(_relation=relation, _table_ref=table_ref, _session=self)
508
+
509
+ def close(self) -> None:
510
+ """Close DuckDB connection."""
511
+ if self._conn is not None:
512
+ self._conn.close()
513
+ self._conn = None
514
+ self._attached_branches.clear()
515
+ self._registered_aliases.clear()
516
+
517
+ def __enter__(self) -> QuerySession:
518
+ return self
519
+
520
+ def __exit__(self, exc_type, exc_val, exc_tb) -> None:
521
+ self.close()
522
+
523
+
524
+ def get_connection(branch: str = "main") -> duckdb.DuckDBPyConnection:
525
+ """Create DuckDB connection attached to the Iceberg REST catalog.
526
+
527
+ Args:
528
+ branch: Branch to connect to. Only "main" is reachable from DuckDB —
529
+ see QuerySession._ensure_branch_attached for why.
530
+
531
+ Returns:
532
+ duckdb.DuckDBPyConnection: DuckDB connection with the Iceberg catalog attached.
533
+
534
+ Raises:
535
+ ValueError: branch is not "main".
536
+ """
537
+ if branch != _DUCKDB_BRANCH:
538
+ raise ValueError(
539
+ f"cannot connect to branch {branch!r} through DuckDB: the Iceberg extension "
540
+ f"only reaches the catalog's default branch ({_DUCKDB_BRANCH!r})"
541
+ )
542
+
543
+ settings = get_settings()
544
+
545
+ conn = duckdb.connect()
546
+
547
+ # Install and load iceberg extension
548
+ conn.execute("INSTALL iceberg; LOAD iceberg;")
549
+ QuerySession._configure_storage(conn)
550
+
551
+ # Attach the Iceberg REST catalog. An unprefixed /iceberg/ endpoint gets
552
+ # the catalog's default branch as the REST prefix.
553
+ endpoint = f"{settings.catalog_uri}/iceberg/"
554
+ token = _resolve_token(settings)
555
+ if token is not None:
556
+ conn.execute(
557
+ """
558
+ ATTACH IF NOT EXISTS 'warehouse' AS iceberg (
559
+ TYPE iceberg,
560
+ ENDPOINT ?,
561
+ AUTHORIZATION_TYPE 'oauth2',
562
+ TOKEN ?
563
+ )
564
+ """,
565
+ [endpoint, token],
566
+ )
567
+ else:
568
+ conn.execute(
569
+ """
570
+ ATTACH IF NOT EXISTS 'warehouse' AS iceberg (
571
+ TYPE iceberg,
572
+ ENDPOINT ?,
573
+ AUTHORIZATION_TYPE 'none'
574
+ )
575
+ """,
576
+ [endpoint],
577
+ )
578
+
579
+ return conn
580
+
581
+
582
+ def list_tables(conn: duckdb.DuckDBPyConnection) -> list[tuple]:
583
+ """List all tables in the attached Iceberg catalog.
584
+
585
+ Args:
586
+ conn: DuckDB connection with the Iceberg catalog attached.
587
+
588
+ Returns:
589
+ list[tuple]: List of table information tuples.
590
+ """
591
+ return conn.execute("SHOW ALL TABLES").fetchall()
592
+
593
+
594
+ # Thread-local singleton for notebook convenience
595
+ _thread_local = threading.local()
596
+
597
+
598
+ def get_session(branch: str = "main") -> QuerySession:
599
+ """Get or create thread-local query session (notebook convenience).
600
+
601
+ This provides a singleton QuerySession per thread, which is useful
602
+ for interactive notebook usage where you don't want to manage
603
+ connection lifecycle manually.
604
+
605
+ Args:
606
+ branch: Default branch for new sessions.
607
+
608
+ Returns:
609
+ Thread-local QuerySession instance.
610
+ """
611
+ if not hasattr(_thread_local, "session") or _thread_local.session is None:
612
+ _thread_local.session = QuerySession(branch=branch)
613
+ return _thread_local.session
614
+
615
+
616
+ def close_session() -> None:
617
+ """Close thread-local session if exists.
618
+
619
+ Call this to clean up the singleton session, for example when
620
+ restarting notebook kernel or shutting down application.
621
+ """
622
+ if hasattr(_thread_local, "session") and _thread_local.session is not None:
623
+ _thread_local.session.close()
624
+ _thread_local.session = None