sqlspec 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of sqlspec might be problematic. Click here for more details.
- sqlspec/__init__.py +16 -3
- sqlspec/_serialization.py +3 -10
- sqlspec/_sql.py +1147 -0
- sqlspec/_typing.py +343 -41
- sqlspec/adapters/adbc/__init__.py +2 -6
- sqlspec/adapters/adbc/config.py +474 -149
- sqlspec/adapters/adbc/driver.py +330 -644
- sqlspec/adapters/aiosqlite/__init__.py +2 -6
- sqlspec/adapters/aiosqlite/config.py +143 -57
- sqlspec/adapters/aiosqlite/driver.py +269 -462
- sqlspec/adapters/asyncmy/__init__.py +3 -8
- sqlspec/adapters/asyncmy/config.py +247 -202
- sqlspec/adapters/asyncmy/driver.py +217 -451
- sqlspec/adapters/asyncpg/__init__.py +4 -7
- sqlspec/adapters/asyncpg/config.py +329 -176
- sqlspec/adapters/asyncpg/driver.py +418 -498
- sqlspec/adapters/bigquery/__init__.py +2 -2
- sqlspec/adapters/bigquery/config.py +407 -0
- sqlspec/adapters/bigquery/driver.py +592 -634
- sqlspec/adapters/duckdb/__init__.py +4 -1
- sqlspec/adapters/duckdb/config.py +432 -321
- sqlspec/adapters/duckdb/driver.py +393 -436
- sqlspec/adapters/oracledb/__init__.py +3 -8
- sqlspec/adapters/oracledb/config.py +625 -0
- sqlspec/adapters/oracledb/driver.py +549 -942
- sqlspec/adapters/psqlpy/__init__.py +4 -7
- sqlspec/adapters/psqlpy/config.py +372 -203
- sqlspec/adapters/psqlpy/driver.py +197 -550
- sqlspec/adapters/psycopg/__init__.py +3 -8
- sqlspec/adapters/psycopg/config.py +741 -0
- sqlspec/adapters/psycopg/driver.py +732 -733
- sqlspec/adapters/sqlite/__init__.py +2 -6
- sqlspec/adapters/sqlite/config.py +146 -81
- sqlspec/adapters/sqlite/driver.py +243 -426
- sqlspec/base.py +220 -825
- sqlspec/config.py +354 -0
- sqlspec/driver/__init__.py +22 -0
- sqlspec/driver/_async.py +252 -0
- sqlspec/driver/_common.py +338 -0
- sqlspec/driver/_sync.py +261 -0
- sqlspec/driver/mixins/__init__.py +17 -0
- sqlspec/driver/mixins/_pipeline.py +523 -0
- sqlspec/driver/mixins/_result_utils.py +122 -0
- sqlspec/driver/mixins/_sql_translator.py +35 -0
- sqlspec/driver/mixins/_storage.py +993 -0
- sqlspec/driver/mixins/_type_coercion.py +131 -0
- sqlspec/exceptions.py +299 -7
- sqlspec/extensions/aiosql/__init__.py +10 -0
- sqlspec/extensions/aiosql/adapter.py +474 -0
- sqlspec/extensions/litestar/__init__.py +1 -6
- sqlspec/extensions/litestar/_utils.py +1 -5
- sqlspec/extensions/litestar/config.py +5 -6
- sqlspec/extensions/litestar/handlers.py +13 -12
- sqlspec/extensions/litestar/plugin.py +22 -24
- sqlspec/extensions/litestar/providers.py +37 -55
- sqlspec/loader.py +528 -0
- sqlspec/service/__init__.py +3 -0
- sqlspec/service/base.py +24 -0
- sqlspec/service/pagination.py +26 -0
- sqlspec/statement/__init__.py +21 -0
- sqlspec/statement/builder/__init__.py +54 -0
- sqlspec/statement/builder/_ddl_utils.py +119 -0
- sqlspec/statement/builder/_parsing_utils.py +135 -0
- sqlspec/statement/builder/base.py +328 -0
- sqlspec/statement/builder/ddl.py +1379 -0
- sqlspec/statement/builder/delete.py +80 -0
- sqlspec/statement/builder/insert.py +274 -0
- sqlspec/statement/builder/merge.py +95 -0
- sqlspec/statement/builder/mixins/__init__.py +65 -0
- sqlspec/statement/builder/mixins/_aggregate_functions.py +151 -0
- sqlspec/statement/builder/mixins/_case_builder.py +91 -0
- sqlspec/statement/builder/mixins/_common_table_expr.py +91 -0
- sqlspec/statement/builder/mixins/_delete_from.py +34 -0
- sqlspec/statement/builder/mixins/_from.py +61 -0
- sqlspec/statement/builder/mixins/_group_by.py +119 -0
- sqlspec/statement/builder/mixins/_having.py +35 -0
- sqlspec/statement/builder/mixins/_insert_from_select.py +48 -0
- sqlspec/statement/builder/mixins/_insert_into.py +36 -0
- sqlspec/statement/builder/mixins/_insert_values.py +69 -0
- sqlspec/statement/builder/mixins/_join.py +110 -0
- sqlspec/statement/builder/mixins/_limit_offset.py +53 -0
- sqlspec/statement/builder/mixins/_merge_clauses.py +405 -0
- sqlspec/statement/builder/mixins/_order_by.py +46 -0
- sqlspec/statement/builder/mixins/_pivot.py +82 -0
- sqlspec/statement/builder/mixins/_returning.py +37 -0
- sqlspec/statement/builder/mixins/_select_columns.py +60 -0
- sqlspec/statement/builder/mixins/_set_ops.py +122 -0
- sqlspec/statement/builder/mixins/_unpivot.py +80 -0
- sqlspec/statement/builder/mixins/_update_from.py +54 -0
- sqlspec/statement/builder/mixins/_update_set.py +91 -0
- sqlspec/statement/builder/mixins/_update_table.py +29 -0
- sqlspec/statement/builder/mixins/_where.py +374 -0
- sqlspec/statement/builder/mixins/_window_functions.py +86 -0
- sqlspec/statement/builder/protocols.py +20 -0
- sqlspec/statement/builder/select.py +206 -0
- sqlspec/statement/builder/update.py +178 -0
- sqlspec/statement/filters.py +571 -0
- sqlspec/statement/parameters.py +736 -0
- sqlspec/statement/pipelines/__init__.py +67 -0
- sqlspec/statement/pipelines/analyzers/__init__.py +9 -0
- sqlspec/statement/pipelines/analyzers/_analyzer.py +649 -0
- sqlspec/statement/pipelines/base.py +315 -0
- sqlspec/statement/pipelines/context.py +119 -0
- sqlspec/statement/pipelines/result_types.py +41 -0
- sqlspec/statement/pipelines/transformers/__init__.py +8 -0
- sqlspec/statement/pipelines/transformers/_expression_simplifier.py +256 -0
- sqlspec/statement/pipelines/transformers/_literal_parameterizer.py +623 -0
- sqlspec/statement/pipelines/transformers/_remove_comments.py +66 -0
- sqlspec/statement/pipelines/transformers/_remove_hints.py +81 -0
- sqlspec/statement/pipelines/validators/__init__.py +23 -0
- sqlspec/statement/pipelines/validators/_dml_safety.py +275 -0
- sqlspec/statement/pipelines/validators/_parameter_style.py +297 -0
- sqlspec/statement/pipelines/validators/_performance.py +703 -0
- sqlspec/statement/pipelines/validators/_security.py +990 -0
- sqlspec/statement/pipelines/validators/base.py +67 -0
- sqlspec/statement/result.py +527 -0
- sqlspec/statement/splitter.py +701 -0
- sqlspec/statement/sql.py +1198 -0
- sqlspec/storage/__init__.py +15 -0
- sqlspec/storage/backends/__init__.py +0 -0
- sqlspec/storage/backends/base.py +166 -0
- sqlspec/storage/backends/fsspec.py +315 -0
- sqlspec/storage/backends/obstore.py +464 -0
- sqlspec/storage/protocol.py +170 -0
- sqlspec/storage/registry.py +315 -0
- sqlspec/typing.py +157 -36
- sqlspec/utils/correlation.py +155 -0
- sqlspec/utils/deprecation.py +3 -6
- sqlspec/utils/fixtures.py +6 -11
- sqlspec/utils/logging.py +135 -0
- sqlspec/utils/module_loader.py +45 -43
- sqlspec/utils/serializers.py +4 -0
- sqlspec/utils/singleton.py +6 -8
- sqlspec/utils/sync_tools.py +15 -27
- sqlspec/utils/text.py +58 -26
- {sqlspec-0.11.0.dist-info → sqlspec-0.12.0.dist-info}/METADATA +100 -26
- sqlspec-0.12.0.dist-info/RECORD +145 -0
- sqlspec/adapters/bigquery/config/__init__.py +0 -3
- sqlspec/adapters/bigquery/config/_common.py +0 -40
- sqlspec/adapters/bigquery/config/_sync.py +0 -87
- sqlspec/adapters/oracledb/config/__init__.py +0 -9
- sqlspec/adapters/oracledb/config/_asyncio.py +0 -186
- sqlspec/adapters/oracledb/config/_common.py +0 -131
- sqlspec/adapters/oracledb/config/_sync.py +0 -186
- sqlspec/adapters/psycopg/config/__init__.py +0 -19
- sqlspec/adapters/psycopg/config/_async.py +0 -169
- sqlspec/adapters/psycopg/config/_common.py +0 -56
- sqlspec/adapters/psycopg/config/_sync.py +0 -168
- sqlspec/filters.py +0 -330
- sqlspec/mixins.py +0 -306
- sqlspec/statement.py +0 -378
- sqlspec-0.11.0.dist-info/RECORD +0 -69
- {sqlspec-0.11.0.dist-info → sqlspec-0.12.0.dist-info}/WHEEL +0 -0
- {sqlspec-0.11.0.dist-info → sqlspec-0.12.0.dist-info}/licenses/LICENSE +0 -0
- {sqlspec-0.11.0.dist-info → sqlspec-0.12.0.dist-info}/licenses/NOTICE +0 -0
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
"""Unified Storage Registry for ObjectStore backends.
|
|
2
|
+
|
|
3
|
+
This module provides a flexible, lazy-loading storage registry that supports:
|
|
4
|
+
- URI-first access pattern with automatic backend detection
|
|
5
|
+
- ObStore preferred, FSSpec fallback architecture
|
|
6
|
+
- Intelligent scheme-based routing with dependency detection
|
|
7
|
+
- Named aliases for commonly used configurations (secondary feature)
|
|
8
|
+
- Automatic instrumentation integration
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
# TODO: TRY300 - Review try-except patterns for else block opportunities
|
|
12
|
+
import logging
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Optional, TypeVar, Union, cast
|
|
15
|
+
|
|
16
|
+
from sqlspec.exceptions import ImproperConfigurationError, MissingDependencyError
|
|
17
|
+
from sqlspec.storage.protocol import ObjectStoreProtocol
|
|
18
|
+
from sqlspec.typing import FSSPEC_INSTALLED, OBSTORE_INSTALLED
|
|
19
|
+
|
|
20
|
+
__all__ = ("StorageRegistry", "storage_registry")
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
BackendT = TypeVar("BackendT", bound=ObjectStoreProtocol)
|
|
25
|
+
|
|
26
|
+
FSSPEC_ONLY_SCHEMES = {"http", "https", "ftp", "sftp", "ssh"}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class StorageRegistry:
|
|
30
|
+
"""Unified storage registry with URI-first access and intelligent backend selection.
|
|
31
|
+
|
|
32
|
+
This registry implements Phase 3 of the unified storage redesign:
|
|
33
|
+
- URI-first access pattern - pass URIs directly to get()
|
|
34
|
+
- Automatic ObStore preference when available
|
|
35
|
+
- Intelligent FSSpec fallback for unsupported schemes or when ObStore unavailable
|
|
36
|
+
- Named aliases as secondary feature for commonly used configurations
|
|
37
|
+
- Dependency-aware backend selection with clear error messages
|
|
38
|
+
|
|
39
|
+
Examples:
|
|
40
|
+
# Primary usage: Direct URI access (no registration needed)
|
|
41
|
+
backend = registry.get("s3://my-bucket/file.parquet") # ObStore preferred
|
|
42
|
+
backend = registry.get("file:///tmp/data.csv") # Obstore for local files
|
|
43
|
+
backend = registry.get("gs://bucket/data.json") # ObStore for GCS
|
|
44
|
+
|
|
45
|
+
# Secondary usage: Named aliases for complex configurations
|
|
46
|
+
registry.register_alias(
|
|
47
|
+
"production-s3",
|
|
48
|
+
uri="s3://prod-bucket/data",
|
|
49
|
+
base_path="sqlspec",
|
|
50
|
+
aws_access_key_id="...",
|
|
51
|
+
aws_secret_access_key="..."
|
|
52
|
+
)
|
|
53
|
+
backend = registry.get("production-s3") # Uses alias
|
|
54
|
+
|
|
55
|
+
# Automatic fallback when ObStore unavailable
|
|
56
|
+
# If obstore not installed: s3:// → FSSpec automatically
|
|
57
|
+
# Clear error if neither backend supports the scheme
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
def __init__(self) -> None:
|
|
61
|
+
# Named aliases (secondary feature) - internal storage
|
|
62
|
+
self._alias_configs: dict[str, tuple[type[ObjectStoreProtocol], str, dict[str, Any]]] = {}
|
|
63
|
+
# Expose configs for testing compatibility
|
|
64
|
+
self._aliases: dict[str, dict[str, Any]] = {}
|
|
65
|
+
self._instances: dict[Union[str, tuple[str, tuple[tuple[str, Any], ...]]], ObjectStoreProtocol] = {}
|
|
66
|
+
|
|
67
|
+
def register_alias(
|
|
68
|
+
self,
|
|
69
|
+
alias: str,
|
|
70
|
+
uri: str,
|
|
71
|
+
*,
|
|
72
|
+
backend: Optional[type[ObjectStoreProtocol]] = None,
|
|
73
|
+
base_path: str = "",
|
|
74
|
+
config: Optional[dict[str, Any]] = None,
|
|
75
|
+
**kwargs: Any,
|
|
76
|
+
) -> None:
|
|
77
|
+
"""Register a named alias for a storage configuration.
|
|
78
|
+
|
|
79
|
+
Args:
|
|
80
|
+
alias: Unique alias name for the configuration
|
|
81
|
+
uri: Storage URI (e.g., "s3://bucket", "file:///path")
|
|
82
|
+
backend: Backend class to use (auto-detected from URI if not provided)
|
|
83
|
+
base_path: Base path to prepend to all operations
|
|
84
|
+
config: Additional configuration dict
|
|
85
|
+
**kwargs: Backend-specific configuration options
|
|
86
|
+
"""
|
|
87
|
+
if backend is None:
|
|
88
|
+
# Auto-detect from URI using new intelligent selection
|
|
89
|
+
backend = self._determine_backend_class(uri)
|
|
90
|
+
|
|
91
|
+
config = config or {}
|
|
92
|
+
config.update(kwargs)
|
|
93
|
+
|
|
94
|
+
# Store the actual config that will be passed to backend
|
|
95
|
+
backend_config = dict(config)
|
|
96
|
+
if base_path:
|
|
97
|
+
backend_config["base_path"] = base_path
|
|
98
|
+
|
|
99
|
+
# Store backend class, URI, and config separately
|
|
100
|
+
self._alias_configs[alias] = (backend, uri, backend_config)
|
|
101
|
+
|
|
102
|
+
# Store config with URI for test compatibility
|
|
103
|
+
test_config = dict(backend_config)
|
|
104
|
+
test_config["uri"] = uri
|
|
105
|
+
self._aliases[alias] = test_config
|
|
106
|
+
|
|
107
|
+
def get(self, uri_or_alias: Union[str, Path], **kwargs: Any) -> ObjectStoreProtocol:
|
|
108
|
+
"""Get backend instance using URI-first routing with intelligent backend selection.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
uri_or_alias: URI to resolve directly OR named alias (secondary feature)
|
|
112
|
+
**kwargs: Additional backend-specific configuration options
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
Backend instance with automatic ObStore preference and FSSpec fallback
|
|
116
|
+
|
|
117
|
+
Raises:
|
|
118
|
+
ImproperConfigurationError: If alias not found or invalid input
|
|
119
|
+
"""
|
|
120
|
+
# Handle None case - raise AttributeError for test compatibility
|
|
121
|
+
if uri_or_alias is None:
|
|
122
|
+
msg = "uri_or_alias cannot be None"
|
|
123
|
+
raise AttributeError(msg)
|
|
124
|
+
|
|
125
|
+
# Handle empty string
|
|
126
|
+
if not uri_or_alias:
|
|
127
|
+
msg = "Unknown storage alias: ''"
|
|
128
|
+
raise ImproperConfigurationError(msg)
|
|
129
|
+
|
|
130
|
+
# Handle Path objects - convert to file:// URI
|
|
131
|
+
if isinstance(uri_or_alias, Path):
|
|
132
|
+
uri_or_alias = f"file://{uri_or_alias.resolve()}"
|
|
133
|
+
|
|
134
|
+
# Check cache first
|
|
135
|
+
cache_key: Union[str, tuple[str, tuple[tuple[str, Any], ...]]] = (
|
|
136
|
+
(uri_or_alias, tuple(sorted(kwargs.items()))) if kwargs else uri_or_alias
|
|
137
|
+
)
|
|
138
|
+
if cache_key in self._instances:
|
|
139
|
+
return self._instances[cache_key]
|
|
140
|
+
|
|
141
|
+
# PRIMARY: Try URI-first routing
|
|
142
|
+
if "://" in uri_or_alias:
|
|
143
|
+
backend = self._resolve_from_uri(uri_or_alias, **kwargs)
|
|
144
|
+
# Cache the instance for future use
|
|
145
|
+
self._instances[cache_key] = backend
|
|
146
|
+
return backend
|
|
147
|
+
|
|
148
|
+
# SECONDARY: Check if it's a registered alias
|
|
149
|
+
if uri_or_alias in self._alias_configs:
|
|
150
|
+
backend_cls, stored_uri, config = self._alias_configs[uri_or_alias]
|
|
151
|
+
# Merge kwargs with alias config (kwargs override)
|
|
152
|
+
merged_config = dict(config)
|
|
153
|
+
merged_config.update(kwargs)
|
|
154
|
+
# URI is passed as first positional arg
|
|
155
|
+
instance = backend_cls(stored_uri, **merged_config)
|
|
156
|
+
self._instances[cache_key] = instance
|
|
157
|
+
return instance
|
|
158
|
+
|
|
159
|
+
# Not a URI and not an alias
|
|
160
|
+
msg = f"Unknown storage alias: '{uri_or_alias}'"
|
|
161
|
+
raise ImproperConfigurationError(msg)
|
|
162
|
+
|
|
163
|
+
def _resolve_from_uri(self, uri: str, **kwargs: Any) -> ObjectStoreProtocol:
|
|
164
|
+
"""Resolve backend from URI.
|
|
165
|
+
|
|
166
|
+
Tries ObStore first for supported schemes, then falls back to FSSpec.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
uri: URI to resolve backend for
|
|
170
|
+
**kwargs: Additional backend-specific configuration
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
Backend instance
|
|
174
|
+
|
|
175
|
+
Raises:
|
|
176
|
+
MissingDependencyError: If no suitable backend can be created
|
|
177
|
+
"""
|
|
178
|
+
# Schemes that ObStore doesn't support
|
|
179
|
+
|
|
180
|
+
# Extract scheme
|
|
181
|
+
scheme = self._get_scheme(uri)
|
|
182
|
+
|
|
183
|
+
last_exc: Optional[Exception] = None
|
|
184
|
+
|
|
185
|
+
# If scheme is FSSpec-only, skip ObStore
|
|
186
|
+
if scheme not in FSSPEC_ONLY_SCHEMES and OBSTORE_INSTALLED:
|
|
187
|
+
try:
|
|
188
|
+
return self._create_backend("obstore", uri, **kwargs)
|
|
189
|
+
except (ImportError, ValueError) as e:
|
|
190
|
+
logger.debug("ObStore backend failed for %s: %s", uri, e)
|
|
191
|
+
last_exc = e
|
|
192
|
+
|
|
193
|
+
if FSSPEC_INSTALLED:
|
|
194
|
+
try:
|
|
195
|
+
return self._create_backend("fsspec", uri, **kwargs)
|
|
196
|
+
except (ImportError, ValueError) as e:
|
|
197
|
+
logger.debug("FSSpec backend failed for %s: %s", uri, e)
|
|
198
|
+
last_exc = e
|
|
199
|
+
|
|
200
|
+
msg = f"No storage backend available for URI '{uri}'. Install 'obstore' or 'fsspec' and ensure dependencies for your filesystem are installed."
|
|
201
|
+
raise MissingDependencyError(msg) from last_exc
|
|
202
|
+
|
|
203
|
+
def _determine_backend_class(self, uri: str) -> type[ObjectStoreProtocol]:
|
|
204
|
+
"""Determine the best backend class for a URI based on availability.
|
|
205
|
+
|
|
206
|
+
Prefers ObStore, falls back to FSSpec.
|
|
207
|
+
|
|
208
|
+
Args:
|
|
209
|
+
uri: URI to determine backend for.
|
|
210
|
+
|
|
211
|
+
Returns:
|
|
212
|
+
Backend class (not instance)
|
|
213
|
+
"""
|
|
214
|
+
if OBSTORE_INSTALLED:
|
|
215
|
+
return self._get_backend_class("obstore")
|
|
216
|
+
if FSSPEC_INSTALLED:
|
|
217
|
+
return self._get_backend_class("fsspec")
|
|
218
|
+
|
|
219
|
+
scheme = uri.split("://", maxsplit=1)[0].lower()
|
|
220
|
+
msg = f"No backend available for URI scheme '{scheme}'. Install obstore or fsspec."
|
|
221
|
+
raise MissingDependencyError(msg)
|
|
222
|
+
|
|
223
|
+
def _get_backend_class(self, backend_type: str) -> type[ObjectStoreProtocol]:
|
|
224
|
+
"""Get backend class by type name.
|
|
225
|
+
|
|
226
|
+
Args:
|
|
227
|
+
backend_type: Backend type ('obstore' or 'fsspec')
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
Backend class
|
|
231
|
+
|
|
232
|
+
Raises:
|
|
233
|
+
ValueError: If unknown backend type
|
|
234
|
+
"""
|
|
235
|
+
if backend_type == "obstore":
|
|
236
|
+
from sqlspec.storage.backends.obstore import ObStoreBackend
|
|
237
|
+
|
|
238
|
+
return cast("type[ObjectStoreProtocol]", ObStoreBackend)
|
|
239
|
+
if backend_type == "fsspec":
|
|
240
|
+
from sqlspec.storage.backends.fsspec import FSSpecBackend
|
|
241
|
+
|
|
242
|
+
return cast("type[ObjectStoreProtocol]", FSSpecBackend)
|
|
243
|
+
msg = f"Unknown backend type: {backend_type}. Supported types: 'obstore', 'fsspec'"
|
|
244
|
+
raise ValueError(msg)
|
|
245
|
+
|
|
246
|
+
def _create_backend(self, backend_type: str, uri: str, **kwargs: Any) -> ObjectStoreProtocol:
|
|
247
|
+
"""Create backend instance for URI.
|
|
248
|
+
|
|
249
|
+
Args:
|
|
250
|
+
backend_type: Backend type ('obstore' or 'fsspec')
|
|
251
|
+
uri: URI to create backend for
|
|
252
|
+
**kwargs: Additional backend-specific configuration
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
Backend instance
|
|
256
|
+
"""
|
|
257
|
+
backend_cls = self._get_backend_class(backend_type)
|
|
258
|
+
# Both backends accept URI as first positional parameter
|
|
259
|
+
return backend_cls(uri, **kwargs)
|
|
260
|
+
|
|
261
|
+
def _get_scheme(self, uri: str) -> str:
|
|
262
|
+
"""Extract scheme from URI.
|
|
263
|
+
|
|
264
|
+
Args:
|
|
265
|
+
uri: URI to extract scheme from
|
|
266
|
+
|
|
267
|
+
Returns:
|
|
268
|
+
Scheme (e.g., 's3', 'gs', 'file')
|
|
269
|
+
"""
|
|
270
|
+
# Handle file paths without explicit file:// scheme
|
|
271
|
+
if not uri or "://" not in uri:
|
|
272
|
+
# Local path (absolute or relative)
|
|
273
|
+
return "file"
|
|
274
|
+
|
|
275
|
+
# Extract scheme from URI
|
|
276
|
+
return uri.split("://", maxsplit=1)[0].lower()
|
|
277
|
+
|
|
278
|
+
# Utility methods
|
|
279
|
+
def is_alias_registered(self, alias: str) -> bool:
|
|
280
|
+
"""Check if a named alias is registered."""
|
|
281
|
+
return alias in self._alias_configs
|
|
282
|
+
|
|
283
|
+
def list_aliases(self) -> list[str]:
|
|
284
|
+
"""List all registered aliases."""
|
|
285
|
+
return list(self._alias_configs.keys())
|
|
286
|
+
|
|
287
|
+
def clear_cache(self, uri_or_alias: Optional[str] = None) -> None:
|
|
288
|
+
"""Clear resolved backend cache.
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
uri_or_alias: Specific URI or alias to clear, or None to clear all
|
|
292
|
+
"""
|
|
293
|
+
if uri_or_alias:
|
|
294
|
+
self._instances.pop(uri_or_alias, None)
|
|
295
|
+
else:
|
|
296
|
+
self._instances.clear()
|
|
297
|
+
|
|
298
|
+
def clear(self) -> None:
|
|
299
|
+
"""Clear all aliases and instances."""
|
|
300
|
+
self._alias_configs.clear()
|
|
301
|
+
self._aliases.clear()
|
|
302
|
+
self._instances.clear()
|
|
303
|
+
|
|
304
|
+
def clear_instances(self) -> None:
|
|
305
|
+
"""Clear only cached instances, keeping aliases."""
|
|
306
|
+
self._instances.clear()
|
|
307
|
+
|
|
308
|
+
def clear_aliases(self) -> None:
|
|
309
|
+
"""Clear only aliases, keeping cached instances."""
|
|
310
|
+
self._alias_configs.clear()
|
|
311
|
+
self._aliases.clear()
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
# Global registry instance
|
|
315
|
+
storage_registry = StorageRegistry()
|
sqlspec/typing.py
CHANGED
|
@@ -1,32 +1,53 @@
|
|
|
1
|
+
from collections.abc import Iterable, Mapping
|
|
2
|
+
from collections.abc import Set as AbstractSet
|
|
1
3
|
from dataclasses import Field, fields
|
|
2
4
|
from functools import lru_cache
|
|
3
|
-
from typing import TYPE_CHECKING, Annotated, Any, Optional,
|
|
5
|
+
from typing import TYPE_CHECKING, Annotated, Any, Optional, Union, cast
|
|
4
6
|
|
|
5
|
-
from
|
|
7
|
+
from sqlglot import exp
|
|
8
|
+
from typing_extensions import TypeAlias, TypeGuard, TypeVar
|
|
6
9
|
|
|
7
10
|
from sqlspec._typing import (
|
|
11
|
+
AIOSQL_INSTALLED,
|
|
12
|
+
FSSPEC_INSTALLED,
|
|
8
13
|
LITESTAR_INSTALLED,
|
|
9
14
|
MSGSPEC_INSTALLED,
|
|
15
|
+
OBSTORE_INSTALLED,
|
|
16
|
+
OPENTELEMETRY_INSTALLED,
|
|
17
|
+
PGVECTOR_INSTALLED,
|
|
18
|
+
PROMETHEUS_INSTALLED,
|
|
10
19
|
PYARROW_INSTALLED,
|
|
11
20
|
PYDANTIC_INSTALLED,
|
|
12
21
|
UNSET,
|
|
22
|
+
AiosqlAsyncProtocol, # pyright: ignore[reportAttributeAccessIssue]
|
|
23
|
+
AiosqlParamType, # pyright: ignore[reportAttributeAccessIssue]
|
|
24
|
+
AiosqlProtocol, # pyright: ignore[reportAttributeAccessIssue]
|
|
25
|
+
AiosqlSQLOperationType, # pyright: ignore[reportAttributeAccessIssue]
|
|
26
|
+
AiosqlSyncProtocol, # pyright: ignore[reportAttributeAccessIssue]
|
|
27
|
+
ArrowRecordBatch,
|
|
13
28
|
ArrowTable,
|
|
14
29
|
BaseModel,
|
|
30
|
+
Counter, # pyright: ignore[reportAttributeAccessIssue]
|
|
15
31
|
DataclassProtocol,
|
|
16
32
|
DTOData,
|
|
17
33
|
Empty,
|
|
18
34
|
EmptyType,
|
|
35
|
+
Gauge, # pyright: ignore[reportAttributeAccessIssue]
|
|
36
|
+
Histogram, # pyright: ignore[reportAttributeAccessIssue]
|
|
37
|
+
Span, # pyright: ignore[reportAttributeAccessIssue]
|
|
38
|
+
Status, # pyright: ignore[reportAttributeAccessIssue]
|
|
39
|
+
StatusCode, # pyright: ignore[reportAttributeAccessIssue]
|
|
19
40
|
Struct,
|
|
41
|
+
Tracer, # pyright: ignore[reportAttributeAccessIssue]
|
|
20
42
|
TypeAdapter,
|
|
21
43
|
UnsetType,
|
|
22
|
-
|
|
44
|
+
aiosql,
|
|
45
|
+
convert, # pyright: ignore[reportAttributeAccessIssue]
|
|
46
|
+
trace,
|
|
23
47
|
)
|
|
24
48
|
|
|
25
49
|
if TYPE_CHECKING:
|
|
26
50
|
from collections.abc import Iterable, Sequence
|
|
27
|
-
from collections.abc import Set as AbstractSet
|
|
28
|
-
|
|
29
|
-
from sqlspec.filters import StatementFilter
|
|
30
51
|
|
|
31
52
|
|
|
32
53
|
PYDANTIC_USE_FAILFAST = False # leave permanently disabled for now
|
|
@@ -54,16 +75,29 @@ ModelT = TypeVar("ModelT", bound="Union[dict[str, Any], Struct, BaseModel, Datac
|
|
|
54
75
|
:class:`dict[str, Any]` | :class:`msgspec.Struct` | :class:`pydantic.BaseModel` | :class:`DataclassProtocol`
|
|
55
76
|
"""
|
|
56
77
|
|
|
57
|
-
FilterTypeT = TypeVar("FilterTypeT", bound="StatementFilter")
|
|
58
|
-
"""Type variable for filter types.
|
|
59
78
|
|
|
60
|
-
:
|
|
61
|
-
"""
|
|
79
|
+
DictRow: TypeAlias = "dict[str, Any]"
|
|
80
|
+
"""Type variable for DictRow types."""
|
|
81
|
+
TupleRow: TypeAlias = "tuple[Any, ...]"
|
|
82
|
+
"""Type variable for TupleRow types."""
|
|
83
|
+
RowT = TypeVar("RowT", default=dict[str, Any])
|
|
84
|
+
|
|
62
85
|
SupportedSchemaModel: TypeAlias = "Union[Struct, BaseModel, DataclassProtocol]"
|
|
63
86
|
"""Type alias for pydantic or msgspec models.
|
|
64
87
|
|
|
65
88
|
:class:`msgspec.Struct` | :class:`pydantic.BaseModel` | :class:`DataclassProtocol`
|
|
66
89
|
"""
|
|
90
|
+
StatementParameters: TypeAlias = "Union[Any, dict[str, Any], list[Any], tuple[Any, ...], None]"
|
|
91
|
+
"""Type alias for statement parameters.
|
|
92
|
+
|
|
93
|
+
Represents:
|
|
94
|
+
- :type:`dict[str, Any]`
|
|
95
|
+
- :type:`list[Any]`
|
|
96
|
+
- :type:`tuple[Any, ...]`
|
|
97
|
+
- :type:`None`
|
|
98
|
+
"""
|
|
99
|
+
# Backward compatibility alias
|
|
100
|
+
SQLParameterType: TypeAlias = StatementParameters
|
|
67
101
|
ModelDTOT = TypeVar("ModelDTOT", bound="SupportedSchemaModel")
|
|
68
102
|
"""Type variable for model DTOs.
|
|
69
103
|
|
|
@@ -97,18 +131,8 @@ Represents:
|
|
|
97
131
|
- :class:`DTOData`[:type:`list[ModelT]`]
|
|
98
132
|
"""
|
|
99
133
|
|
|
100
|
-
StatementParameterType: TypeAlias = "Union[Any, dict[str, Any], list[Any], tuple[Any, ...], None]"
|
|
101
|
-
"""Type alias for parameter types.
|
|
102
|
-
|
|
103
|
-
Represents:
|
|
104
|
-
- :type:`dict[str, Any]`
|
|
105
|
-
- :type:`list[Any]`
|
|
106
|
-
- :type:`tuple[Any, ...]`
|
|
107
|
-
- :type:`None`
|
|
108
|
-
"""
|
|
109
|
-
|
|
110
134
|
|
|
111
|
-
def is_dataclass_instance(obj: Any) ->
|
|
135
|
+
def is_dataclass_instance(obj: Any) -> TypeGuard[DataclassProtocol]:
|
|
112
136
|
"""Check if an object is a dataclass instance.
|
|
113
137
|
|
|
114
138
|
Args:
|
|
@@ -117,7 +141,9 @@ def is_dataclass_instance(obj: Any) -> "TypeGuard[DataclassProtocol]":
|
|
|
117
141
|
Returns:
|
|
118
142
|
True if the object is a dataclass instance.
|
|
119
143
|
"""
|
|
120
|
-
|
|
144
|
+
# Ensure obj is an instance and not the class itself,
|
|
145
|
+
# and that its type is a dataclass.
|
|
146
|
+
return not isinstance(obj, type) and hasattr(type(obj), "__dataclass_fields__")
|
|
121
147
|
|
|
122
148
|
|
|
123
149
|
@lru_cache(typed=True)
|
|
@@ -131,9 +157,7 @@ def get_type_adapter(f: "type[T]") -> "TypeAdapter[T]":
|
|
|
131
157
|
:class:`pydantic.TypeAdapter`[:class:`typing.TypeVar`[T]]
|
|
132
158
|
"""
|
|
133
159
|
if PYDANTIC_USE_FAILFAST:
|
|
134
|
-
return TypeAdapter(
|
|
135
|
-
Annotated[f, FailFast()],
|
|
136
|
-
)
|
|
160
|
+
return TypeAdapter(Annotated[f, FailFast()])
|
|
137
161
|
return TypeAdapter(f)
|
|
138
162
|
|
|
139
163
|
|
|
@@ -302,8 +326,7 @@ def is_schema_without_field(obj: "Any", field_name: str) -> "TypeGuard[Supported
|
|
|
302
326
|
|
|
303
327
|
|
|
304
328
|
def is_schema_or_dict_with_field(
|
|
305
|
-
obj: "Any",
|
|
306
|
-
field_name: str,
|
|
329
|
+
obj: "Any", field_name: str
|
|
307
330
|
) -> "TypeGuard[Union[SupportedSchemaModel, dict[str, Any]]]":
|
|
308
331
|
"""Check if a value is a msgspec Struct, Pydantic model, or dict with a specific field.
|
|
309
332
|
|
|
@@ -318,8 +341,7 @@ def is_schema_or_dict_with_field(
|
|
|
318
341
|
|
|
319
342
|
|
|
320
343
|
def is_schema_or_dict_without_field(
|
|
321
|
-
obj: "Any",
|
|
322
|
-
field_name: str,
|
|
344
|
+
obj: "Any", field_name: str
|
|
323
345
|
) -> "TypeGuard[Union[SupportedSchemaModel, dict[str, Any]]]":
|
|
324
346
|
"""Check if a value is a msgspec Struct, Pydantic model, or dict without a specific field.
|
|
325
347
|
|
|
@@ -342,12 +364,13 @@ def is_dataclass(obj: "Any") -> "TypeGuard[DataclassProtocol]":
|
|
|
342
364
|
Returns:
|
|
343
365
|
bool
|
|
344
366
|
"""
|
|
367
|
+
if isinstance(obj, type) and hasattr(obj, "__dataclass_fields__"):
|
|
368
|
+
return True
|
|
345
369
|
return is_dataclass_instance(obj)
|
|
346
370
|
|
|
347
371
|
|
|
348
372
|
def is_dataclass_with_field(
|
|
349
|
-
obj: "Any",
|
|
350
|
-
field_name: str,
|
|
373
|
+
obj: "Any", field_name: str
|
|
351
374
|
) -> "TypeGuard[object]": # Can't specify dataclass type directly
|
|
352
375
|
"""Check if an object is a dataclass and has a specific field.
|
|
353
376
|
|
|
@@ -475,8 +498,7 @@ def dataclass_to_dict(
|
|
|
475
498
|
|
|
476
499
|
|
|
477
500
|
def schema_dump(
|
|
478
|
-
data: "Union[dict[str, Any], DataclassProtocol, Struct, BaseModel]",
|
|
479
|
-
exclude_unset: bool = True,
|
|
501
|
+
data: "Union[dict[str, Any], DataclassProtocol, Struct, BaseModel]", exclude_unset: bool = True
|
|
480
502
|
) -> "dict[str, Any]":
|
|
481
503
|
"""Dump a data object to a dictionary.
|
|
482
504
|
|
|
@@ -515,27 +537,85 @@ def is_dto_data(v: Any) -> TypeGuard[DTOData[Any]]:
|
|
|
515
537
|
return LITESTAR_INSTALLED and isinstance(v, DTOData)
|
|
516
538
|
|
|
517
539
|
|
|
540
|
+
def is_expression(obj: "Any") -> "TypeGuard[exp.Expression]":
|
|
541
|
+
"""Check if a value is a sqlglot Expression.
|
|
542
|
+
|
|
543
|
+
Args:
|
|
544
|
+
obj: Value to check.
|
|
545
|
+
|
|
546
|
+
Returns:
|
|
547
|
+
bool
|
|
548
|
+
"""
|
|
549
|
+
return isinstance(obj, exp.Expression)
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def MixinOf(base: type[T]) -> type[T]: # noqa: N802
|
|
553
|
+
"""Useful function to make mixins with baseclass type hint
|
|
554
|
+
|
|
555
|
+
```
|
|
556
|
+
class StorageMixin(MixinOf(DriverProtocol)): ...
|
|
557
|
+
```
|
|
558
|
+
"""
|
|
559
|
+
if TYPE_CHECKING:
|
|
560
|
+
return base
|
|
561
|
+
return type("<MixinOf>", (base,), {})
|
|
562
|
+
|
|
563
|
+
|
|
518
564
|
__all__ = (
|
|
565
|
+
"AIOSQL_INSTALLED",
|
|
566
|
+
"FSSPEC_INSTALLED",
|
|
519
567
|
"LITESTAR_INSTALLED",
|
|
520
568
|
"MSGSPEC_INSTALLED",
|
|
569
|
+
"OBSTORE_INSTALLED",
|
|
570
|
+
"OPENTELEMETRY_INSTALLED",
|
|
571
|
+
"PGVECTOR_INSTALLED",
|
|
572
|
+
"PROMETHEUS_INSTALLED",
|
|
521
573
|
"PYARROW_INSTALLED",
|
|
522
574
|
"PYDANTIC_INSTALLED",
|
|
523
575
|
"PYDANTIC_USE_FAILFAST",
|
|
524
576
|
"UNSET",
|
|
577
|
+
"AiosqlAsyncProtocol",
|
|
578
|
+
"AiosqlParamType",
|
|
579
|
+
"AiosqlProtocol",
|
|
580
|
+
"AiosqlSQLOperationType",
|
|
581
|
+
"AiosqlSyncProtocol",
|
|
582
|
+
"ArrowRecordBatch",
|
|
525
583
|
"ArrowTable",
|
|
526
584
|
"BaseModel",
|
|
585
|
+
"BulkModelDict",
|
|
586
|
+
"ConnectionT",
|
|
587
|
+
"Counter",
|
|
527
588
|
"DataclassProtocol",
|
|
589
|
+
"DictRow",
|
|
528
590
|
"Empty",
|
|
529
591
|
"EmptyType",
|
|
530
592
|
"FailFast",
|
|
531
|
-
"
|
|
593
|
+
"Gauge",
|
|
594
|
+
"Histogram",
|
|
595
|
+
"Mapping",
|
|
596
|
+
"MixinOf",
|
|
597
|
+
"ModelDTOT",
|
|
598
|
+
"ModelDict",
|
|
532
599
|
"ModelDict",
|
|
533
600
|
"ModelDictList",
|
|
534
|
-
"
|
|
601
|
+
"ModelDictList",
|
|
602
|
+
"ModelT",
|
|
603
|
+
"PoolT",
|
|
604
|
+
"PoolT_co",
|
|
605
|
+
"PydanticOrMsgspecT",
|
|
606
|
+
"RowT",
|
|
607
|
+
"SQLParameterType",
|
|
608
|
+
"Span",
|
|
609
|
+
"StatementParameters",
|
|
610
|
+
"Status",
|
|
611
|
+
"StatusCode",
|
|
535
612
|
"Struct",
|
|
536
613
|
"SupportedSchemaModel",
|
|
614
|
+
"Tracer",
|
|
615
|
+
"TupleRow",
|
|
537
616
|
"TypeAdapter",
|
|
538
617
|
"UnsetType",
|
|
618
|
+
"aiosql",
|
|
539
619
|
"convert",
|
|
540
620
|
"dataclass_to_dict",
|
|
541
621
|
"extract_dataclass_fields",
|
|
@@ -549,6 +629,7 @@ __all__ = (
|
|
|
549
629
|
"is_dict_with_field",
|
|
550
630
|
"is_dict_without_field",
|
|
551
631
|
"is_dto_data",
|
|
632
|
+
"is_expression",
|
|
552
633
|
"is_msgspec_struct",
|
|
553
634
|
"is_msgspec_struct_with_field",
|
|
554
635
|
"is_msgspec_struct_without_field",
|
|
@@ -562,6 +643,7 @@ __all__ = (
|
|
|
562
643
|
"is_schema_with_field",
|
|
563
644
|
"is_schema_without_field",
|
|
564
645
|
"schema_dump",
|
|
646
|
+
"trace",
|
|
565
647
|
)
|
|
566
648
|
|
|
567
649
|
if TYPE_CHECKING:
|
|
@@ -576,10 +658,49 @@ if TYPE_CHECKING:
|
|
|
576
658
|
from msgspec import UNSET, Struct, UnsetType, convert # noqa: TC004
|
|
577
659
|
|
|
578
660
|
if not PYARROW_INSTALLED:
|
|
579
|
-
from sqlspec._typing import ArrowTable
|
|
661
|
+
from sqlspec._typing import ArrowRecordBatch, ArrowTable
|
|
580
662
|
else:
|
|
663
|
+
from pyarrow import RecordBatch as ArrowRecordBatch # noqa: TC004
|
|
581
664
|
from pyarrow import Table as ArrowTable # noqa: TC004
|
|
582
665
|
if not LITESTAR_INSTALLED:
|
|
583
666
|
from sqlspec._typing import DTOData
|
|
584
667
|
else:
|
|
585
668
|
from litestar.dto import DTOData # noqa: TC004
|
|
669
|
+
if not OPENTELEMETRY_INSTALLED:
|
|
670
|
+
from sqlspec._typing import Span, Status, StatusCode, Tracer, trace # noqa: TC004 # pyright: ignore
|
|
671
|
+
else:
|
|
672
|
+
from opentelemetry.trace import ( # pyright: ignore[reportMissingImports] # noqa: TC004
|
|
673
|
+
Span,
|
|
674
|
+
Status,
|
|
675
|
+
StatusCode,
|
|
676
|
+
Tracer,
|
|
677
|
+
)
|
|
678
|
+
if not PROMETHEUS_INSTALLED:
|
|
679
|
+
from sqlspec._typing import Counter, Gauge, Histogram # pyright: ignore
|
|
680
|
+
else:
|
|
681
|
+
from prometheus_client import Counter, Gauge, Histogram # noqa: TC004 # pyright: ignore # noqa: TC004
|
|
682
|
+
|
|
683
|
+
if not AIOSQL_INSTALLED:
|
|
684
|
+
from sqlspec._typing import (
|
|
685
|
+
AiosqlAsyncProtocol, # pyright: ignore[reportAttributeAccessIssue]
|
|
686
|
+
AiosqlParamType, # pyright: ignore[reportAttributeAccessIssue]
|
|
687
|
+
AiosqlProtocol, # pyright: ignore[reportAttributeAccessIssue]
|
|
688
|
+
AiosqlSQLOperationType, # pyright: ignore[reportAttributeAccessIssue]
|
|
689
|
+
AiosqlSyncProtocol, # pyright: ignore[reportAttributeAccessIssue]
|
|
690
|
+
aiosql,
|
|
691
|
+
)
|
|
692
|
+
else:
|
|
693
|
+
import aiosql # noqa: TC004 # pyright: ignore
|
|
694
|
+
from aiosql.types import ( # noqa: TC004 # pyright: ignore[reportMissingImports]
|
|
695
|
+
AsyncDriverAdapterProtocol as AiosqlAsyncProtocol,
|
|
696
|
+
)
|
|
697
|
+
from aiosql.types import ( # noqa: TC004 # pyright: ignore[reportMissingImports]
|
|
698
|
+
DriverAdapterProtocol as AiosqlProtocol,
|
|
699
|
+
)
|
|
700
|
+
from aiosql.types import ParamType as AiosqlParamType # noqa: TC004 # pyright: ignore[reportMissingImports]
|
|
701
|
+
from aiosql.types import (
|
|
702
|
+
SQLOperationType as AiosqlSQLOperationType, # noqa: TC004 # pyright: ignore[reportMissingImports]
|
|
703
|
+
)
|
|
704
|
+
from aiosql.types import ( # noqa: TC004 # pyright: ignore[reportMissingImports]
|
|
705
|
+
SyncDriverAdapterProtocol as AiosqlSyncProtocol,
|
|
706
|
+
)
|