sdmxlib 0.27.2__tar.gz → 0.28.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/PKG-INFO +1 -1
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/pyproject.toml +1 -1
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/local/_data_store.py +8 -2
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/local/_registry.py +187 -80
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/code_query.py +37 -17
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/hierarchy_query.py +35 -17
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/lazy.py +157 -68
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/readers.py +54 -20
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/README.md +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/client.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/federated.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/filters.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/policy.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/providers.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/query.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/registry.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/session.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/catalog.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/data_store.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_csv/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_csv/reader.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_csv/writer.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/metadata.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/reader.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/writer.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/namespaces.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/reader.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/writer.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/metadata.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/namespaces.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/reader.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/writer.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/local/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/annotations.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/base.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/binding.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/category.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/codelist.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/collections.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/concept.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/constraint.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/convert.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/dataflow.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/dataset.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/datastructure.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/expr.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/hierarchy.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/in_memory_registry.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/istring.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/mapping.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/message.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/metadataflow.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/metadataset.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/metadatastructure.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/organisation.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/provision.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/ref.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/registry.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/representation.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/urn.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/validation.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/polars.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/py.typed +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/rest.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/sql.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/__init__.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/schema.py +0 -0
- {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/writers.py +0 -0
|
@@ -267,7 +267,8 @@ class DuckDBDataStore:
|
|
|
267
267
|
order_by=order_by,
|
|
268
268
|
limit=limit,
|
|
269
269
|
)
|
|
270
|
-
|
|
270
|
+
# Per-call cursor for issue #75 (concurrent reads).
|
|
271
|
+
cur = conn.cursor().execute(sql, sql_params)
|
|
271
272
|
col_names = [d[0] for d in cur.description] if cur.description else []
|
|
272
273
|
rows = cur.fetchall()
|
|
273
274
|
if not rows:
|
|
@@ -365,7 +366,12 @@ class DuckDBDataStore:
|
|
|
365
366
|
order_by=order_by,
|
|
366
367
|
limit=None,
|
|
367
368
|
)
|
|
368
|
-
|
|
369
|
+
# Per-call cursor for issue #75 — concurrent ``query_stream``
|
|
370
|
+
# callers must not share the connection's default-cursor row
|
|
371
|
+
# buffer. The Arrow reader streams from this cursor and holds
|
|
372
|
+
# the reference until the generator is exhausted (or GC'd).
|
|
373
|
+
cur = conn.cursor()
|
|
374
|
+
reader = cur.execute(sql, sql_params).to_arrow_reader(chunk_rows)
|
|
369
375
|
yield from reader
|
|
370
376
|
|
|
371
377
|
|
|
@@ -30,8 +30,9 @@ stashes the URI; the DuckDB connection is opened by ``open()`` /
|
|
|
30
30
|
local.add(...)
|
|
31
31
|
"""
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
from collections.abc import Iterable
|
|
33
|
+
import threading
|
|
34
|
+
from collections.abc import Iterable, Iterator
|
|
35
|
+
from contextlib import contextmanager
|
|
35
36
|
from pathlib import Path
|
|
36
37
|
from typing import Any, Self
|
|
37
38
|
|
|
@@ -61,18 +62,30 @@ _ITEM_TO_SCHEME: dict[str, tuple[str, str]] = {
|
|
|
61
62
|
DEFAULT_RESOLVE_CACHE_SIZE = 256
|
|
62
63
|
|
|
63
64
|
|
|
64
|
-
class
|
|
65
|
-
"""Bounded
|
|
65
|
+
class _FIFOCache(dict[str, Any]):
|
|
66
|
+
"""Bounded resolver cache. FIFO eviction, lock-free reads.
|
|
66
67
|
|
|
67
|
-
Subclasses :class:`
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
68
|
+
Subclasses :class:`dict` (not ``OrderedDict``) so that ``get`` /
|
|
69
|
+
``__contains__`` / ``__getitem__`` are plain dict ops — GIL-atomic
|
|
70
|
+
in CPython and safe for multiple concurrent reader threads with no
|
|
71
|
+
locking. Mutations (insert, evict, clear) hold a per-instance
|
|
72
|
+
:class:`threading.Lock`.
|
|
72
73
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
74
|
+
Eviction is FIFO (insertion order) rather than LRU. FIFO is a
|
|
75
|
+
deliberate choice over LRU here: tracking recency would require
|
|
76
|
+
mutating the cache on every hit, which would force every cache hit
|
|
77
|
+
onto the lock and serialise reader threads. At realistic working-set
|
|
78
|
+
sizes for SDMX parent-scheme reuse, the hit-rate difference between
|
|
79
|
+
FIFO and LRU is small; the lock-free read path is more valuable.
|
|
80
|
+
Documented as a known trade-off in ``docs/concurrency.md`` — revisit
|
|
81
|
+
if ``resolve_cache_info()`` ever shows poor hit rates against a
|
|
82
|
+
working set that should otherwise fit.
|
|
83
|
+
|
|
84
|
+
``maxsize == 0`` disables caching — inserts are silently dropped.
|
|
85
|
+
|
|
86
|
+
Hit/miss counters are best-effort under concurrency: lockless ``+=``
|
|
87
|
+
on the counter ints races, so under contention the reported numbers
|
|
88
|
+
may slightly under-count. They remain useful as a diagnostic.
|
|
76
89
|
"""
|
|
77
90
|
|
|
78
91
|
def __init__(self, maxsize: int) -> None:
|
|
@@ -83,63 +96,72 @@ class _LRUDict(OrderedDict[str, Any]):
|
|
|
83
96
|
self.maxsize = maxsize
|
|
84
97
|
self.hits = 0
|
|
85
98
|
self.misses = 0
|
|
99
|
+
self._lock = threading.Lock()
|
|
86
100
|
|
|
87
101
|
def __setitem__(self, key: str, value: Any) -> None:
|
|
88
102
|
if self.maxsize == 0:
|
|
89
103
|
return
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
self.
|
|
104
|
+
with self._lock:
|
|
105
|
+
super().__setitem__(key, value)
|
|
106
|
+
# Evict insertion-oldest entries past the cap. ``next(iter(self))``
|
|
107
|
+
# is the first-inserted key for plain dicts in Python 3.7+.
|
|
108
|
+
while len(self) > self.maxsize:
|
|
109
|
+
oldest = next(iter(self))
|
|
110
|
+
super().__delitem__(oldest)
|
|
95
111
|
|
|
96
112
|
def get(self, key: str, default: Any = None) -> Any: # type: ignore[override]
|
|
113
|
+
# Lock-free read: plain dict ``__contains__`` / ``__getitem__``
|
|
114
|
+
# are GIL-atomic in CPython. The counter increments race under
|
|
115
|
+
# concurrency — accepted (see class docstring).
|
|
97
116
|
if super().__contains__(key):
|
|
98
117
|
self.hits += 1
|
|
99
|
-
self.move_to_end(key)
|
|
100
118
|
return super().__getitem__(key)
|
|
101
119
|
self.misses += 1
|
|
102
120
|
return default
|
|
103
121
|
|
|
122
|
+
def clear(self) -> None:
|
|
123
|
+
with self._lock:
|
|
124
|
+
super().clear()
|
|
125
|
+
|
|
104
126
|
def reset_stats(self) -> None:
|
|
105
127
|
self.hits = 0
|
|
106
128
|
self.misses = 0
|
|
107
129
|
|
|
108
130
|
|
|
109
131
|
class _CallScopedMemo(dict[str, Any]):
|
|
110
|
-
"""Per-call resolver memo layered over
|
|
132
|
+
"""Per-call resolver memo layered over the persistent FIFO cache.
|
|
111
133
|
|
|
112
|
-
Two reasons for the overlay rather than passing the
|
|
134
|
+
Two reasons for the overlay rather than passing the cache directly:
|
|
113
135
|
|
|
114
136
|
1. **Correctness under tight cache sizes (incl. ``maxsize=0``).** The
|
|
115
137
|
resolver writes URN→target during one phase and reads them back in
|
|
116
138
|
a later phase of the same call. If those writes went straight to
|
|
117
|
-
the
|
|
139
|
+
the cache, an entry could be evicted before the read — leaving Refs
|
|
118
140
|
unresolved for callers using a small (or disabled) cache. The
|
|
119
141
|
overlay holds in-flight entries in an unbounded per-call dict so
|
|
120
142
|
single-call resolution is independent of cache size.
|
|
121
143
|
2. **Promote-on-success.** Entries written during the call enter the
|
|
122
|
-
|
|
123
|
-
partial-failure semantics clean: if a call raises
|
|
124
|
-
is unchanged.
|
|
144
|
+
persistent cache only at the end of the call, via :meth:`promote`.
|
|
145
|
+
That keeps partial-failure semantics clean: if a call raises
|
|
146
|
+
midway, the persistent cache is unchanged.
|
|
125
147
|
"""
|
|
126
148
|
|
|
127
|
-
def __init__(self,
|
|
149
|
+
def __init__(self, cache: _FIFOCache) -> None:
|
|
128
150
|
super().__init__()
|
|
129
|
-
self.
|
|
151
|
+
self._cache = cache
|
|
130
152
|
|
|
131
153
|
def __contains__(self, key: object) -> bool:
|
|
132
|
-
return super().__contains__(key) or self.
|
|
154
|
+
return super().__contains__(key) or self._cache.__contains__(key)
|
|
133
155
|
|
|
134
156
|
def get(self, key: str, default: Any = None) -> Any: # type: ignore[override]
|
|
135
157
|
if super().__contains__(key):
|
|
136
158
|
return super().__getitem__(key)
|
|
137
|
-
return self.
|
|
159
|
+
return self._cache.get(key, default)
|
|
138
160
|
|
|
139
161
|
def promote(self) -> None:
|
|
140
|
-
"""Copy this call's resolved entries into the persistent
|
|
162
|
+
"""Copy this call's resolved entries into the persistent cache."""
|
|
141
163
|
for k, v in self.items():
|
|
142
|
-
self.
|
|
164
|
+
self._cache[k] = v
|
|
143
165
|
|
|
144
166
|
|
|
145
167
|
class Registry:
|
|
@@ -164,6 +186,15 @@ class Registry:
|
|
|
164
186
|
DuckLake catalog the caller already opened) is done via
|
|
165
187
|
:meth:`from_connection`, which doesn't take ownership of the
|
|
166
188
|
connection's lifecycle.
|
|
189
|
+
|
|
190
|
+
**Concurrency.** Reads (``get`` / ``get_all`` / ``codelist`` /
|
|
191
|
+
``code_map`` / ``dsd_code_map`` / ``to_registry`` / ``__contains__``
|
|
192
|
+
/ ``__len__``) are safe to call from multiple threads against the
|
|
193
|
+
same ``Registry``. Writes (``add`` / ``drop`` / ``replace`` /
|
|
194
|
+
``add_all`` / ``ingest``) are single-threaded — serialise externally
|
|
195
|
+
if you need to mutate from multiple threads. See
|
|
196
|
+
``docs/design/concurrency.md`` for the model and deployment
|
|
197
|
+
recipes.
|
|
167
198
|
"""
|
|
168
199
|
|
|
169
200
|
def __init__(
|
|
@@ -175,18 +206,20 @@ class Registry:
|
|
|
175
206
|
) -> None:
|
|
176
207
|
"""Open or wrap a DuckDB-backed registry.
|
|
177
208
|
|
|
178
|
-
``resolve_cache_size`` bounds the
|
|
179
|
-
:meth:`get` / :meth:`get_all` / :meth:`to_registry`
|
|
180
|
-
calls. Pass ``0`` to disable caching (matches
|
|
181
|
-
— one full read per Ref). The cache is
|
|
182
|
-
(``add`` / ``drop`` / ``replace``) for
|
|
209
|
+
``resolve_cache_size`` bounds the FIFO cache of resolved Ref
|
|
210
|
+
targets that :meth:`get` / :meth:`get_all` / :meth:`to_registry`
|
|
211
|
+
share across calls. Pass ``0`` to disable caching (matches
|
|
212
|
+
pre-v0.27.2 behaviour — one full read per Ref). The cache is
|
|
213
|
+
cleared on every mutation (``add`` / ``drop`` / ``replace``) for
|
|
214
|
+
invalidation correctness. FIFO eviction (not LRU) keeps cache
|
|
215
|
+
reads lock-free; see ``docs/concurrency.md``.
|
|
183
216
|
"""
|
|
184
217
|
self._uri = Path(uri) if isinstance(uri, str) else uri
|
|
185
218
|
self._source_registry = source_registry
|
|
186
219
|
self._conn: duckdb.DuckDBPyConnection | None = None
|
|
187
220
|
self._cursor: duckdb.DuckDBPyConnection | None = None
|
|
188
221
|
self._owns_connection = True
|
|
189
|
-
self._resolve_cache:
|
|
222
|
+
self._resolve_cache: _FIFOCache = _FIFOCache(resolve_cache_size)
|
|
190
223
|
|
|
191
224
|
@property
|
|
192
225
|
def uri(self) -> Path:
|
|
@@ -250,7 +283,7 @@ class Registry:
|
|
|
250
283
|
instance._conn = conn # noqa: SLF001
|
|
251
284
|
instance._cursor = None # noqa: SLF001
|
|
252
285
|
instance._owns_connection = False # noqa: SLF001
|
|
253
|
-
instance._resolve_cache =
|
|
286
|
+
instance._resolve_cache = _FIFOCache(resolve_cache_size) # noqa: SLF001
|
|
254
287
|
instance._initialise_schema() # noqa: SLF001
|
|
255
288
|
return instance
|
|
256
289
|
|
|
@@ -266,6 +299,11 @@ class Registry:
|
|
|
266
299
|
else:
|
|
267
300
|
self._cursor = self._conn.cursor()
|
|
268
301
|
self._cursor.execute("SET search_path = 'sdmx'")
|
|
302
|
+
# Also set search_path on the connection's default cursor — used
|
|
303
|
+
# by code paths that call ``conn.execute()`` directly without
|
|
304
|
+
# going through ``_read_cursor`` (lazy facades' ``code_map``
|
|
305
|
+
# forwarder, etc.). Cheap one-shot.
|
|
306
|
+
self._conn.execute("SET search_path = 'sdmx'")
|
|
269
307
|
|
|
270
308
|
def __enter__(self) -> Self:
|
|
271
309
|
if self._conn is None:
|
|
@@ -289,10 +327,10 @@ class Registry:
|
|
|
289
327
|
def _c(self) -> duckdb.DuckDBPyConnection:
|
|
290
328
|
"""Internal accessor — returns the live cursor or raises a clear error.
|
|
291
329
|
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
330
|
+
Used by write paths (``add`` / ``drop`` / ``replace``) which are
|
|
331
|
+
documented as single-threaded. Read paths must go through
|
|
332
|
+
:meth:`_read_cursor` instead so concurrent reads each get an
|
|
333
|
+
isolated cursor — see ``docs/concurrency.md`` and issue #75.
|
|
296
334
|
"""
|
|
297
335
|
if self._cursor is None:
|
|
298
336
|
msg = (
|
|
@@ -302,6 +340,49 @@ class Registry:
|
|
|
302
340
|
raise RuntimeError(msg)
|
|
303
341
|
return self._cursor
|
|
304
342
|
|
|
343
|
+
def _require_open(self) -> duckdb.DuckDBPyConnection:
|
|
344
|
+
"""Return the live connection or raise a clear lifecycle error.
|
|
345
|
+
|
|
346
|
+
Use when a caller needs the long-lived connection itself (e.g. to
|
|
347
|
+
hand to a lazy facade) rather than a short-lived cursor.
|
|
348
|
+
"""
|
|
349
|
+
if self._conn is None:
|
|
350
|
+
msg = (
|
|
351
|
+
f"Registry({self._uri!r}) is not open. Use it as a context manager "
|
|
352
|
+
"(``with Registry(uri) as reg: ...``) or call ``.open()`` first."
|
|
353
|
+
)
|
|
354
|
+
raise RuntimeError(msg)
|
|
355
|
+
return self._conn
|
|
356
|
+
|
|
357
|
+
@contextmanager
|
|
358
|
+
def _read_cursor(self) -> Iterator[duckdb.DuckDBPyConnection]:
|
|
359
|
+
"""Open a fresh DuckDB cursor for the duration of one read call.
|
|
360
|
+
|
|
361
|
+
DuckDB serialises ``execute()`` calls on a single connection but
|
|
362
|
+
each connection has *one* default cursor whose row buffer is
|
|
363
|
+
overwritten by every ``execute()``. Reads that fan out across
|
|
364
|
+
threads must therefore each hold their own cursor — otherwise
|
|
365
|
+
thread A's ``fetchone()`` can race thread B's ``execute()`` and
|
|
366
|
+
return rows belonging to B's query. Fixed in #75/#78.
|
|
367
|
+
|
|
368
|
+
Cursors on a read-only DuckDB connection are cheap (no
|
|
369
|
+
transaction state); per-call create/close has negligible
|
|
370
|
+
overhead at typical request rates. Sharing the connection — not
|
|
371
|
+
the cursor — keeps DuckLake / cataloglib federation working.
|
|
372
|
+
"""
|
|
373
|
+
if self._conn is None:
|
|
374
|
+
msg = (
|
|
375
|
+
f"Registry({self._uri!r}) is not open. Use it as a context manager "
|
|
376
|
+
"(``with Registry(uri) as reg: ...``) or call ``.open()`` first."
|
|
377
|
+
)
|
|
378
|
+
raise RuntimeError(msg)
|
|
379
|
+
cur = self._conn.cursor()
|
|
380
|
+
try:
|
|
381
|
+
cur.execute("SET search_path = 'sdmx'")
|
|
382
|
+
yield cur
|
|
383
|
+
finally:
|
|
384
|
+
cur.close()
|
|
385
|
+
|
|
305
386
|
# ── Add ───────────────────────────────────────────────────────────────
|
|
306
387
|
|
|
307
388
|
def add(self, artefact: Any, *, source_registry: str | None = None) -> None:
|
|
@@ -313,6 +394,12 @@ class Registry:
|
|
|
313
394
|
Clears the resolver cache: any cached resolved target may now be
|
|
314
395
|
stale (e.g. a freshly-added Codelist replaces the one a previous
|
|
315
396
|
``get`` had cached).
|
|
397
|
+
|
|
398
|
+
**Single-threaded.** Mutations are not safe under concurrent
|
|
399
|
+
callers — do not invoke ``add`` / ``drop`` / ``replace`` from
|
|
400
|
+
multiple threads against the same ``Registry``. Concurrent reads
|
|
401
|
+
through ``get`` / ``get_all`` / ``code_map`` etc. are safe. See
|
|
402
|
+
``docs/concurrency.md``.
|
|
316
403
|
"""
|
|
317
404
|
tag = source_registry if source_registry is not None else self._source_registry
|
|
318
405
|
writers.put_artefact(self._c, artefact, source_registry=tag)
|
|
@@ -342,7 +429,12 @@ class Registry:
|
|
|
342
429
|
"""
|
|
343
430
|
from sdmxlib.storage.readers import make_lazy_codelist # noqa: PLC0415
|
|
344
431
|
|
|
345
|
-
|
|
432
|
+
# make_lazy_codelist opens its own internal cursor for spine
|
|
433
|
+
# reads; the returned LazyCodelist holds the long-lived
|
|
434
|
+
# connection for later method calls. ``_require_open`` enforces
|
|
435
|
+
# the lifecycle precondition without creating a cursor we don't
|
|
436
|
+
# need.
|
|
437
|
+
return make_lazy_codelist(self._require_open(), str(urn))
|
|
346
438
|
|
|
347
439
|
def get[T](self, urn: "SdmxUrn[T]", *, eager: bool = False) -> "T | None":
|
|
348
440
|
"""Look up an artefact by URN.
|
|
@@ -357,14 +449,15 @@ class Registry:
|
|
|
357
449
|
dsd.dimensions["FREQ"].codelist() # -> LazyCodelist
|
|
358
450
|
dsd.code_map(lang="en") # works
|
|
359
451
|
"""
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
452
|
+
with self._read_cursor() as cur:
|
|
453
|
+
result = readers.get_artefact(cur, str(urn), eager=eager, lazy_conn=self._conn)
|
|
454
|
+
if result is None:
|
|
455
|
+
return None
|
|
456
|
+
if not isinstance(result, (LazyCodelist, LazyHierarchy)):
|
|
457
|
+
memo = _CallScopedMemo(self._resolve_cache)
|
|
458
|
+
self._resolve_refs(cur, result, memo=memo)
|
|
459
|
+
memo.promote()
|
|
460
|
+
return result # type: ignore[return-value]
|
|
368
461
|
|
|
369
462
|
def get_all[T](self, artefact_type: "type[T]") -> "list[T]":
|
|
370
463
|
"""All artefacts of the given Python type.
|
|
@@ -374,18 +467,20 @@ class Registry:
|
|
|
374
467
|
parent scheme referenced by multiple artefacts is loaded once,
|
|
375
468
|
not once per artefact.
|
|
376
469
|
"""
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
470
|
+
with self._read_cursor() as cur:
|
|
471
|
+
results = readers.get_all(cur, artefact_type, lazy_conn=self._conn)
|
|
472
|
+
memo = _CallScopedMemo(self._resolve_cache)
|
|
473
|
+
for r in results:
|
|
474
|
+
if not isinstance(r, (LazyCodelist, LazyHierarchy)):
|
|
475
|
+
self._resolve_refs(cur, r, memo=memo)
|
|
476
|
+
memo.promote()
|
|
477
|
+
return results # type: ignore[return-value]
|
|
384
478
|
|
|
385
479
|
# ── Ref resolution ─────────────────────────────────────────────────
|
|
386
480
|
|
|
387
481
|
def _resolve_refs( # noqa: C901
|
|
388
482
|
self,
|
|
483
|
+
cur: duckdb.DuckDBPyConnection,
|
|
389
484
|
obj: Any,
|
|
390
485
|
*,
|
|
391
486
|
memo: "dict[str, Any] | None" = None,
|
|
@@ -438,7 +533,7 @@ class Registry:
|
|
|
438
533
|
needed.append(ref.urn)
|
|
439
534
|
|
|
440
535
|
if needed:
|
|
441
|
-
self._resolve_urns(needed, memo)
|
|
536
|
+
self._resolve_urns(cur, needed, memo)
|
|
442
537
|
|
|
443
538
|
for owner, name, ref in pending:
|
|
444
539
|
if ref._resolved is not None: # noqa: SLF001
|
|
@@ -479,6 +574,7 @@ class Registry:
|
|
|
479
574
|
|
|
480
575
|
def _resolve_urns( # noqa: C901
|
|
481
576
|
self,
|
|
577
|
+
cur: duckdb.DuckDBPyConnection,
|
|
482
578
|
urns: "list[SdmxUrn[Any]]",
|
|
483
579
|
memo: "dict[str, Any]",
|
|
484
580
|
) -> None:
|
|
@@ -507,7 +603,7 @@ class Registry:
|
|
|
507
603
|
if scheme_info is None:
|
|
508
604
|
continue
|
|
509
605
|
_scheme_class_name, items_attr = scheme_info
|
|
510
|
-
parent = self._load_parent_scheme(parent_urn_str, sample, memo)
|
|
606
|
+
parent = self._load_parent_scheme(cur, parent_urn_str, sample, memo)
|
|
511
607
|
if parent is None:
|
|
512
608
|
continue
|
|
513
609
|
items = getattr(parent, items_attr, None)
|
|
@@ -522,7 +618,7 @@ class Registry:
|
|
|
522
618
|
urn_str = str(parsed)
|
|
523
619
|
if urn_str in memo:
|
|
524
620
|
continue
|
|
525
|
-
target = readers.get_artefact(
|
|
621
|
+
target = readers.get_artefact(cur, urn_str, lazy_conn=self._conn)
|
|
526
622
|
if target is not None:
|
|
527
623
|
memo[urn_str] = target
|
|
528
624
|
|
|
@@ -547,6 +643,7 @@ class Registry:
|
|
|
547
643
|
|
|
548
644
|
def _load_parent_scheme(
|
|
549
645
|
self,
|
|
646
|
+
cur: duckdb.DuckDBPyConnection,
|
|
550
647
|
parent_urn_str: str,
|
|
551
648
|
sample: "SdmxUrn[Any]",
|
|
552
649
|
memo: "dict[str, Any]",
|
|
@@ -559,12 +656,12 @@ class Registry:
|
|
|
559
656
|
cached = memo.get(parent_urn_str)
|
|
560
657
|
if cached is not None:
|
|
561
658
|
return cached
|
|
562
|
-
parent = readers.get_artefact(
|
|
659
|
+
parent = readers.get_artefact(cur, parent_urn_str, eager=True)
|
|
563
660
|
if parent is None and sample.version == "latest":
|
|
564
661
|
scheme_info = _ITEM_TO_SCHEME.get(sample.artefact_class)
|
|
565
662
|
if scheme_info is not None:
|
|
566
663
|
scheme_class_name, _ = scheme_info
|
|
567
|
-
row =
|
|
664
|
+
row = cur.execute(
|
|
568
665
|
"""
|
|
569
666
|
SELECT urn FROM artefact
|
|
570
667
|
WHERE artefact_type = ? AND agency_id = ? AND resource_id = ?
|
|
@@ -572,7 +669,7 @@ class Registry:
|
|
|
572
669
|
[scheme_class_name, sample.agency, sample.id],
|
|
573
670
|
).fetchone()
|
|
574
671
|
if row:
|
|
575
|
-
parent = readers.get_artefact(
|
|
672
|
+
parent = readers.get_artefact(cur, row[0], eager=True)
|
|
576
673
|
if parent is not None:
|
|
577
674
|
memo[parent_urn_str] = parent
|
|
578
675
|
return parent
|
|
@@ -587,19 +684,22 @@ class Registry:
|
|
|
587
684
|
|
|
588
685
|
def __contains__(self, urn: object) -> bool:
|
|
589
686
|
if isinstance(urn, SdmxUrn):
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
687
|
+
with self._read_cursor() as cur:
|
|
688
|
+
row = cur.execute(
|
|
689
|
+
"SELECT 1 FROM artefact WHERE urn = ?",
|
|
690
|
+
[str(urn)],
|
|
691
|
+
).fetchone()
|
|
692
|
+
return row is not None
|
|
595
693
|
return False
|
|
596
694
|
|
|
597
695
|
def __len__(self) -> int:
|
|
598
|
-
|
|
599
|
-
|
|
696
|
+
with self._read_cursor() as cur:
|
|
697
|
+
row = cur.execute("SELECT COUNT(*) FROM artefact").fetchone()
|
|
698
|
+
return row[0] # type: ignore[index]
|
|
600
699
|
|
|
601
700
|
def __repr__(self) -> str:
|
|
602
|
-
|
|
701
|
+
with self._read_cursor() as cur:
|
|
702
|
+
rows = cur.execute("SELECT artefact_type, COUNT(*) FROM artefact GROUP BY artefact_type").fetchall()
|
|
603
703
|
counts = ", ".join(f"{name}={cnt}" for name, cnt in rows)
|
|
604
704
|
return f"Registry({counts or 'empty'})"
|
|
605
705
|
|
|
@@ -610,6 +710,8 @@ class Registry:
|
|
|
610
710
|
|
|
611
711
|
Clears the resolver cache (cached resolved targets may reference
|
|
612
712
|
the dropped artefact).
|
|
713
|
+
|
|
714
|
+
**Single-threaded** — see :meth:`add`.
|
|
613
715
|
"""
|
|
614
716
|
result = writers.drop_artefact(self._c, str(urn))
|
|
615
717
|
self._resolve_cache.clear()
|
|
@@ -620,6 +722,8 @@ class Registry:
|
|
|
620
722
|
|
|
621
723
|
Each leg clears the resolver cache via :meth:`add` / :meth:`drop`;
|
|
622
724
|
the net effect is a single fully-cleared cache at the end.
|
|
725
|
+
|
|
726
|
+
**Single-threaded** — see :meth:`add`.
|
|
623
727
|
"""
|
|
624
728
|
writers.drop_artefact(self._c, str(old_urn))
|
|
625
729
|
self.add(new)
|
|
@@ -655,11 +759,13 @@ class Registry:
|
|
|
655
759
|
|
|
656
760
|
def code_map(self, codelist_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, str]:
|
|
657
761
|
"""Return {code_id: label} for a codelist without creating Code objects."""
|
|
658
|
-
|
|
762
|
+
with self._read_cursor() as cur:
|
|
763
|
+
return readers.code_map(cur, str(codelist_urn), lang=lang)
|
|
659
764
|
|
|
660
765
|
def dsd_code_map(self, dsd_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, dict[str, str]]:
|
|
661
766
|
"""Return {component_id: {code_id: label}} for all enumerated dimensions."""
|
|
662
|
-
|
|
767
|
+
with self._read_cursor() as cur:
|
|
768
|
+
return readers.code_map_for_dsd(cur, str(dsd_urn), lang=lang)
|
|
663
769
|
|
|
664
770
|
# ── Bulk ──────────────────────────────────────────────────────────────
|
|
665
771
|
|
|
@@ -687,12 +793,13 @@ class Registry:
|
|
|
687
793
|
"""
|
|
688
794
|
reg = InMemoryRegistry()
|
|
689
795
|
memo = _CallScopedMemo(self._resolve_cache)
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
796
|
+
with self._read_cursor() as cur:
|
|
797
|
+
rows = cur.execute("SELECT urn, artefact_type FROM artefact").fetchall()
|
|
798
|
+
for urn_str, _artefact_type in rows:
|
|
799
|
+
obj = readers.get_artefact(cur, urn_str, eager=True)
|
|
800
|
+
if obj is None:
|
|
801
|
+
continue
|
|
802
|
+
self._resolve_refs(cur, obj, memo=memo)
|
|
803
|
+
reg.add(obj)
|
|
697
804
|
memo.promote()
|
|
698
805
|
return reg
|
|
@@ -35,6 +35,11 @@ if TYPE_CHECKING:
|
|
|
35
35
|
import duckdb
|
|
36
36
|
|
|
37
37
|
|
|
38
|
+
def _cur(conn: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyConnection:
|
|
39
|
+
"""Open a fresh per-call cursor (issue #75). SQL here is sdmx.-qualified."""
|
|
40
|
+
return conn.cursor()
|
|
41
|
+
|
|
42
|
+
|
|
38
43
|
# ── Field expressions (AST nodes) ───────────────────────────────────────
|
|
39
44
|
|
|
40
45
|
|
|
@@ -445,7 +450,7 @@ class CodeQuery:
|
|
|
445
450
|
dominates on large codelists.
|
|
446
451
|
"""
|
|
447
452
|
sql, params = _compile(self)
|
|
448
|
-
rows = self._conn.execute(sql, params).fetchall()
|
|
453
|
+
rows = _cur(self._conn).execute(sql, params).fetchall()
|
|
449
454
|
if not rows:
|
|
450
455
|
return []
|
|
451
456
|
if self.annotation_types:
|
|
@@ -458,7 +463,7 @@ class CodeQuery:
|
|
|
458
463
|
def ids(self) -> list[str]:
|
|
459
464
|
"""Compile to SQL, execute, and return just the code ids."""
|
|
460
465
|
sql, params = _compile(self, select_clause="c.code_id")
|
|
461
|
-
rows = self._conn.execute(sql, params).fetchall()
|
|
466
|
+
rows = _cur(self._conn).execute(sql, params).fetchall()
|
|
462
467
|
return [r[0] for r in rows]
|
|
463
468
|
|
|
464
469
|
def to_sql(self) -> tuple[str, list[object]]:
|
|
@@ -481,7 +486,7 @@ class CodeQuery:
|
|
|
481
486
|
|
|
482
487
|
def __len__(self) -> int:
|
|
483
488
|
sql, params = _compile(self, select_clause="COUNT(*)", omit_order=True)
|
|
484
|
-
row = self._conn.execute(sql, params).fetchone()
|
|
489
|
+
row = _cur(self._conn).execute(sql, params).fetchone()
|
|
485
490
|
return row[0] if row else 0 # type: ignore[index]
|
|
486
491
|
|
|
487
492
|
@overload
|
|
@@ -492,29 +497,37 @@ class CodeQuery:
|
|
|
492
497
|
|
|
493
498
|
def __getitem__(self, key: int | str) -> Code:
|
|
494
499
|
if isinstance(key, str):
|
|
495
|
-
row =
|
|
496
|
-
|
|
500
|
+
row = (
|
|
501
|
+
_cur(self._conn)
|
|
502
|
+
.execute(
|
|
503
|
+
"""
|
|
497
504
|
SELECT c.code_id, c.parent_id, c.depth, c.name, c.code_urn, c.description
|
|
498
505
|
FROM sdmx.code c
|
|
499
506
|
WHERE c.codelist_urn = ? AND c.code_id = ?
|
|
500
507
|
""",
|
|
501
|
-
|
|
502
|
-
|
|
508
|
+
[self.codelist_urn, key],
|
|
509
|
+
)
|
|
510
|
+
.fetchone()
|
|
511
|
+
)
|
|
503
512
|
if row is None:
|
|
504
513
|
raise KeyError(key)
|
|
505
514
|
return _row_to_code(row, self._anns_for_one(row[4]))
|
|
506
515
|
|
|
507
516
|
offset = key if key >= 0 else len(self) + key
|
|
508
|
-
row =
|
|
509
|
-
|
|
517
|
+
row = (
|
|
518
|
+
_cur(self._conn)
|
|
519
|
+
.execute(
|
|
520
|
+
"""
|
|
510
521
|
SELECT c.code_id, c.parent_id, c.depth, c.name, c.code_urn, c.description
|
|
511
522
|
FROM sdmx.code c
|
|
512
523
|
WHERE c.codelist_urn = ?
|
|
513
524
|
ORDER BY c.position NULLS LAST, c.code_id
|
|
514
525
|
LIMIT 1 OFFSET ?
|
|
515
526
|
""",
|
|
516
|
-
|
|
517
|
-
|
|
527
|
+
[self.codelist_urn, offset],
|
|
528
|
+
)
|
|
529
|
+
.fetchone()
|
|
530
|
+
)
|
|
518
531
|
if row is None:
|
|
519
532
|
raise IndexError(key)
|
|
520
533
|
return _row_to_code(row, self._anns_for_one(row[4]))
|
|
@@ -535,10 +548,14 @@ class CodeQuery:
|
|
|
535
548
|
|
|
536
549
|
def __contains__(self, key: object) -> bool:
|
|
537
550
|
if isinstance(key, str):
|
|
538
|
-
row =
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
551
|
+
row = (
|
|
552
|
+
_cur(self._conn)
|
|
553
|
+
.execute(
|
|
554
|
+
"SELECT 1 FROM sdmx.code WHERE codelist_urn = ? AND code_id = ?",
|
|
555
|
+
[self.codelist_urn, key],
|
|
556
|
+
)
|
|
557
|
+
.fetchone()
|
|
558
|
+
)
|
|
542
559
|
return row is not None
|
|
543
560
|
if isinstance(key, Code):
|
|
544
561
|
return key.id in self
|
|
@@ -947,13 +964,16 @@ def _fetch_annotations_for(
|
|
|
947
964
|
sql_parts.append(f"AND annotation_type IN ({type_placeholders})")
|
|
948
965
|
sql_params.extend(types)
|
|
949
966
|
sql_parts.append("ORDER BY owner_urn, ordinal")
|
|
950
|
-
|
|
967
|
+
# Use a single per-call cursor across both queries to keep concurrent
|
|
968
|
+
# callers isolated from each other's row buffers (issue #75).
|
|
969
|
+
cur = _cur(conn)
|
|
970
|
+
ann_rows = cur.execute(" ".join(sql_parts), sql_params).fetchall()
|
|
951
971
|
if not ann_rows:
|
|
952
972
|
return {}
|
|
953
973
|
|
|
954
974
|
ann_ids = [r[0] for r in ann_rows]
|
|
955
975
|
text_placeholders = ", ".join("?" for _ in ann_ids)
|
|
956
|
-
text_rows =
|
|
976
|
+
text_rows = cur.execute(
|
|
957
977
|
f"""
|
|
958
978
|
SELECT annotation_id, lang, text
|
|
959
979
|
FROM sdmx.annotation_text
|