pipeloom-duckdb 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pipeloom_duckdb/__init__.py +1 -0
- pipeloom_duckdb/catalog.py +125 -0
- pipeloom_duckdb/engine.py +826 -0
- pipeloom_duckdb/extensions.py +44 -0
- pipeloom_duckdb/py.typed +0 -0
- pipeloom_duckdb/spec.py +118 -0
- pipeloom_duckdb/sqltext.py +160 -0
- pipeloom_duckdb-1.1.0.dist-info/METADATA +57 -0
- pipeloom_duckdb-1.1.0.dist-info/RECORD +13 -0
- pipeloom_duckdb-1.1.0.dist-info/WHEEL +4 -0
- pipeloom_duckdb-1.1.0.dist-info/entry_points.txt +5 -0
- pipeloom_duckdb-1.1.0.dist-info/licenses/LICENSE +202 -0
- pipeloom_duckdb-1.1.0.dist-info/licenses/NOTICE +2 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""DuckDB engine plugin for pipeloom. Import `pipeloom_duckdb.spec` to read its capabilities."""
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""The DuckDB catalog: tables and views in a database file, a model catalog in a sibling file.
|
|
2
|
+
|
|
3
|
+
It imports no DuckDB. The engine binds its session with `attach` for the length of a session.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Iterator
|
|
9
|
+
from contextlib import AbstractContextManager, contextmanager
|
|
10
|
+
from pathlib import Path, PurePosixPath, PureWindowsPath
|
|
11
|
+
from typing import TYPE_CHECKING, Any
|
|
12
|
+
|
|
13
|
+
from pipeloom.engine.types import CatalogCapabilities, Namespace, Schema, TargetRef
|
|
14
|
+
from pipeloom_duckdb.spec import MEMORY
|
|
15
|
+
from pipeloom_duckdb.sqltext import ident, schema_of, string
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from pipeloom_duckdb.engine import DuckDBSession
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _name(value: str) -> str:
|
|
22
|
+
if (
|
|
23
|
+
not value
|
|
24
|
+
or value in (".", "..")
|
|
25
|
+
or ":" in value
|
|
26
|
+
or any(ord(c) < 32 for c in value)
|
|
27
|
+
or PurePosixPath(value).name != value
|
|
28
|
+
or PureWindowsPath(value).name != value
|
|
29
|
+
):
|
|
30
|
+
raise ValueError(f"{value!r} is not usable as a DuckDB catalog name")
|
|
31
|
+
return value
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class DuckDBCatalog:
|
|
35
|
+
"""`path` holds the tables without a catalog. A target catalog `<name>` is the database
|
|
36
|
+
`<name>.duckdb` next to it, attached as `<name>` (ADR-0051)."""
|
|
37
|
+
|
|
38
|
+
capabilities = CatalogCapabilities(views=True)
|
|
39
|
+
|
|
40
|
+
def __init__(self, path: str) -> None:
|
|
41
|
+
self.path = path if path == MEMORY else Path(path).resolve().as_posix()
|
|
42
|
+
self._s: DuckDBSession | None = None
|
|
43
|
+
self._main = ""
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def memory(self) -> bool:
|
|
47
|
+
return self.path == MEMORY
|
|
48
|
+
|
|
49
|
+
def attach(self, session: DuckDBSession) -> AbstractContextManager[None]:
|
|
50
|
+
return self._attach(session)
|
|
51
|
+
|
|
52
|
+
@contextmanager
|
|
53
|
+
def _attach(self, session: DuckDBSession) -> Iterator[None]:
|
|
54
|
+
self._s = session
|
|
55
|
+
self._main = str(session.scalar("SELECT current_database()"))
|
|
56
|
+
if not self.memory: # earlier catalogs, so views across them bind
|
|
57
|
+
for sibling in sorted(Path(self.path).parent.glob("*.duckdb")):
|
|
58
|
+
if sibling.as_posix() != self.path:
|
|
59
|
+
self._attach_db(sibling.stem, create=False)
|
|
60
|
+
try:
|
|
61
|
+
yield
|
|
62
|
+
finally:
|
|
63
|
+
self._s = None
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def session(self) -> DuckDBSession:
|
|
67
|
+
if self._s is None:
|
|
68
|
+
raise RuntimeError("the DuckDB catalog is used outside an engine session")
|
|
69
|
+
return self._s
|
|
70
|
+
|
|
71
|
+
def _attached(self) -> set[str]:
|
|
72
|
+
return {r[0] for r in self.session.rows("SELECT database_name FROM duckdb_databases()")}
|
|
73
|
+
|
|
74
|
+
def _attach_db(self, name: str, *, create: bool) -> bool:
|
|
75
|
+
"""Attach catalog `name`; False when its file does not exist and `create` is off."""
|
|
76
|
+
with self.session.lock: # check and attach as one step: units attach at once
|
|
77
|
+
if name in self._attached():
|
|
78
|
+
return True
|
|
79
|
+
if self.memory:
|
|
80
|
+
if not create:
|
|
81
|
+
return False
|
|
82
|
+
location = MEMORY
|
|
83
|
+
else:
|
|
84
|
+
file = Path(self.path).parent / f"{name}.duckdb"
|
|
85
|
+
if not create and not file.exists():
|
|
86
|
+
return False
|
|
87
|
+
location = file.as_posix()
|
|
88
|
+
self.session.execute(f"ATTACH {string(location)} AS {ident(name)}")
|
|
89
|
+
return True
|
|
90
|
+
|
|
91
|
+
def database(self, catalog: str | None, *, create: bool = True) -> str | None:
|
|
92
|
+
"""The attached database of a target catalog; None when it does not exist yet."""
|
|
93
|
+
if catalog is None or catalog == self._main:
|
|
94
|
+
return self._main
|
|
95
|
+
name = _name(catalog)
|
|
96
|
+
return name if self._attach_db(name, create=create) else None
|
|
97
|
+
|
|
98
|
+
def ident(self, target: TargetRef) -> str:
|
|
99
|
+
db = self.database(target.catalog)
|
|
100
|
+
assert db is not None
|
|
101
|
+
return ".".join(ident(p) for p in (db, target.schema, target.name))
|
|
102
|
+
|
|
103
|
+
# -- the Catalog protocol ---------------------------------------------------------------
|
|
104
|
+
|
|
105
|
+
def ensure_namespace(self, ns: Namespace) -> None:
|
|
106
|
+
db = self.database(ns.catalog)
|
|
107
|
+
assert db is not None
|
|
108
|
+
self.session.execute(f"CREATE SCHEMA IF NOT EXISTS {ident(db)}.{ident(ns.schema)}")
|
|
109
|
+
|
|
110
|
+
def exists(self, target: TargetRef) -> bool:
|
|
111
|
+
db = self.database(target.catalog, create=False)
|
|
112
|
+
if db is None:
|
|
113
|
+
return False
|
|
114
|
+
query = (
|
|
115
|
+
"SELECT count(*) FROM (SELECT database_name, schema_name, table_name AS n "
|
|
116
|
+
"FROM duckdb_tables() UNION ALL SELECT database_name, schema_name, view_name "
|
|
117
|
+
"FROM duckdb_views()) WHERE database_name = ? AND schema_name = ? AND n = ?"
|
|
118
|
+
)
|
|
119
|
+
return bool(self.session.scalar(query, [db, target.schema, target.name]))
|
|
120
|
+
|
|
121
|
+
def table_schema(self, target: TargetRef) -> Schema | None:
|
|
122
|
+
if not self.exists(target):
|
|
123
|
+
return None
|
|
124
|
+
rel: Any = self.session.query(f"SELECT * FROM {self.ident(target)}")
|
|
125
|
+
return schema_of(rel.columns, rel.types)
|