sdmxlib 0.27.2__tar.gz → 0.28.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/PKG-INFO +1 -1
  2. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/pyproject.toml +1 -1
  3. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/local/_data_store.py +8 -2
  4. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/local/_registry.py +187 -80
  5. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/code_query.py +37 -17
  6. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/hierarchy_query.py +35 -17
  7. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/lazy.py +157 -68
  8. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/readers.py +54 -20
  9. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/README.md +0 -0
  10. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/__init__.py +0 -0
  11. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/__init__.py +0 -0
  12. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/client.py +0 -0
  13. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/federated.py +0 -0
  14. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/filters.py +0 -0
  15. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/policy.py +0 -0
  16. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/providers.py +0 -0
  17. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/query.py +0 -0
  18. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/registry.py +0 -0
  19. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/api/session.py +0 -0
  20. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/catalog.py +0 -0
  21. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/data_store.py +0 -0
  22. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/__init__.py +0 -0
  23. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_csv/__init__.py +0 -0
  24. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_csv/reader.py +0 -0
  25. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_csv/writer.py +0 -0
  26. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/__init__.py +0 -0
  27. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/metadata.py +0 -0
  28. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/reader.py +0 -0
  29. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_json/writer.py +0 -0
  30. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/__init__.py +0 -0
  31. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/namespaces.py +0 -0
  32. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/reader.py +0 -0
  33. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml21/writer.py +0 -0
  34. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/__init__.py +0 -0
  35. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/metadata.py +0 -0
  36. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/namespaces.py +0 -0
  37. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/reader.py +0 -0
  38. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/formats/sdmx_ml30/writer.py +0 -0
  39. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/local/__init__.py +0 -0
  40. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/__init__.py +0 -0
  41. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/annotations.py +0 -0
  42. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/base.py +0 -0
  43. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/binding.py +0 -0
  44. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/category.py +0 -0
  45. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/codelist.py +0 -0
  46. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/collections.py +0 -0
  47. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/concept.py +0 -0
  48. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/constraint.py +0 -0
  49. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/convert.py +0 -0
  50. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/dataflow.py +0 -0
  51. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/dataset.py +0 -0
  52. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/datastructure.py +0 -0
  53. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/expr.py +0 -0
  54. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/hierarchy.py +0 -0
  55. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/in_memory_registry.py +0 -0
  56. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/istring.py +0 -0
  57. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/mapping.py +0 -0
  58. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/message.py +0 -0
  59. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/metadataflow.py +0 -0
  60. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/metadataset.py +0 -0
  61. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/metadatastructure.py +0 -0
  62. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/organisation.py +0 -0
  63. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/provision.py +0 -0
  64. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/ref.py +0 -0
  65. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/registry.py +0 -0
  66. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/representation.py +0 -0
  67. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/urn.py +0 -0
  68. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/model/validation.py +0 -0
  69. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/polars.py +0 -0
  70. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/py.typed +0 -0
  71. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/rest.py +0 -0
  72. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/sql.py +0 -0
  73. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/__init__.py +0 -0
  74. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/schema.py +0 -0
  75. {sdmxlib-0.27.2 → sdmxlib-0.28.0}/src/sdmxlib/storage/writers.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: sdmxlib
3
- Version: 0.27.2
3
+ Version: 0.28.0
4
4
  Summary: SDMX structural metadata library for Python
5
5
  Keywords: sdmx,statistics,metadata,datastructure
6
6
  Author: gabrielgellner
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sdmxlib"
3
- version = "0.27.2"
3
+ version = "0.28.0"
4
4
  description = "SDMX structural metadata library for Python"
5
5
  readme = "README.md"
6
6
  license = { text = "Apache-2.0" }
@@ -267,7 +267,8 @@ class DuckDBDataStore:
267
267
  order_by=order_by,
268
268
  limit=limit,
269
269
  )
270
- cur = conn.execute(sql, sql_params)
270
+ # Per-call cursor for issue #75 (concurrent reads).
271
+ cur = conn.cursor().execute(sql, sql_params)
271
272
  col_names = [d[0] for d in cur.description] if cur.description else []
272
273
  rows = cur.fetchall()
273
274
  if not rows:
@@ -365,7 +366,12 @@ class DuckDBDataStore:
365
366
  order_by=order_by,
366
367
  limit=None,
367
368
  )
368
- reader = conn.execute(sql, sql_params).to_arrow_reader(chunk_rows)
369
+ # Per-call cursor for issue #75 — concurrent ``query_stream``
370
+ # callers must not share the connection's default-cursor row
371
+ # buffer. The Arrow reader streams from this cursor and holds
372
+ # the reference until the generator is exhausted (or GC'd).
373
+ cur = conn.cursor()
374
+ reader = cur.execute(sql, sql_params).to_arrow_reader(chunk_rows)
369
375
  yield from reader
370
376
 
371
377
 
@@ -30,8 +30,9 @@ stashes the URI; the DuckDB connection is opened by ``open()`` /
30
30
  local.add(...)
31
31
  """
32
32
 
33
- from collections import OrderedDict
34
- from collections.abc import Iterable
33
+ import threading
34
+ from collections.abc import Iterable, Iterator
35
+ from contextlib import contextmanager
35
36
  from pathlib import Path
36
37
  from typing import Any, Self
37
38
 
@@ -61,18 +62,30 @@ _ITEM_TO_SCHEME: dict[str, tuple[str, str]] = {
61
62
  DEFAULT_RESOLVE_CACHE_SIZE = 256
62
63
 
63
64
 
64
- class _LRUDict(OrderedDict[str, Any]):
65
- """Bounded LRU mapping used as the registry-level resolver cache.
65
+ class _FIFOCache(dict[str, Any]):
66
+ """Bounded resolver cache. FIFO eviction, lock-free reads.
66
67
 
67
- Subclasses :class:`OrderedDict` so it can be passed wherever the
68
- resolver expects ``dict[str, Any]``. Inserts evict least-recently-used
69
- entries past ``maxsize``; reads via :meth:`get` move keys to the end.
70
- ``maxsize == 0`` disables caching inserts are silently dropped, so
71
- the structure stays empty.
68
+ Subclasses :class:`dict` (not ``OrderedDict``) so that ``get`` /
69
+ ``__contains__`` / ``__getitem__`` are plain dict ops GIL-atomic
70
+ in CPython and safe for multiple concurrent reader threads with no
71
+ locking. Mutations (insert, evict, clear) hold a per-instance
72
+ :class:`threading.Lock`.
72
73
 
73
- Tracks ``hits`` / ``misses`` for :meth:`Registry.resolve_cache_info`;
74
- ``__contains__`` is treated as internal control flow (the resolver
75
- uses it for dedup) and is not counted.
74
+ Eviction is FIFO (insertion order) rather than LRU. FIFO is a
75
+ deliberate choice over LRU here: tracking recency would require
76
+ mutating the cache on every hit, which would force every cache hit
77
+ onto the lock and serialise reader threads. At realistic working-set
78
+ sizes for SDMX parent-scheme reuse, the hit-rate difference between
79
+ FIFO and LRU is small; the lock-free read path is more valuable.
80
+ Documented as a known trade-off in ``docs/concurrency.md`` — revisit
81
+ if ``resolve_cache_info()`` ever shows poor hit rates against a
82
+ working set that should otherwise fit.
83
+
84
+ ``maxsize == 0`` disables caching — inserts are silently dropped.
85
+
86
+ Hit/miss counters are best-effort under concurrency: lockless ``+=``
87
+ on the counter ints races, so under contention the reported numbers
88
+ may slightly under-count. They remain useful as a diagnostic.
76
89
  """
77
90
 
78
91
  def __init__(self, maxsize: int) -> None:
@@ -83,63 +96,72 @@ class _LRUDict(OrderedDict[str, Any]):
83
96
  self.maxsize = maxsize
84
97
  self.hits = 0
85
98
  self.misses = 0
99
+ self._lock = threading.Lock()
86
100
 
87
101
  def __setitem__(self, key: str, value: Any) -> None:
88
102
  if self.maxsize == 0:
89
103
  return
90
- if key in self:
91
- self.move_to_end(key)
92
- super().__setitem__(key, value)
93
- while len(self) > self.maxsize:
94
- self.popitem(last=False)
104
+ with self._lock:
105
+ super().__setitem__(key, value)
106
+ # Evict insertion-oldest entries past the cap. ``next(iter(self))``
107
+ # is the first-inserted key for plain dicts in Python 3.7+.
108
+ while len(self) > self.maxsize:
109
+ oldest = next(iter(self))
110
+ super().__delitem__(oldest)
95
111
 
96
112
  def get(self, key: str, default: Any = None) -> Any: # type: ignore[override]
113
+ # Lock-free read: plain dict ``__contains__`` / ``__getitem__``
114
+ # are GIL-atomic in CPython. The counter increments race under
115
+ # concurrency — accepted (see class docstring).
97
116
  if super().__contains__(key):
98
117
  self.hits += 1
99
- self.move_to_end(key)
100
118
  return super().__getitem__(key)
101
119
  self.misses += 1
102
120
  return default
103
121
 
122
+ def clear(self) -> None:
123
+ with self._lock:
124
+ super().clear()
125
+
104
126
  def reset_stats(self) -> None:
105
127
  self.hits = 0
106
128
  self.misses = 0
107
129
 
108
130
 
109
131
  class _CallScopedMemo(dict[str, Any]):
110
- """Per-call resolver memo layered over a persistent LRU cache.
132
+ """Per-call resolver memo layered over the persistent FIFO cache.
111
133
 
112
- Two reasons for the overlay rather than passing the LRU directly:
134
+ Two reasons for the overlay rather than passing the cache directly:
113
135
 
114
136
  1. **Correctness under tight cache sizes (incl. ``maxsize=0``).** The
115
137
  resolver writes URN→target during one phase and reads them back in
116
138
  a later phase of the same call. If those writes went straight to
117
- the LRU, an entry could be evicted before the read — leaving Refs
139
+ the cache, an entry could be evicted before the read — leaving Refs
118
140
  unresolved for callers using a small (or disabled) cache. The
119
141
  overlay holds in-flight entries in an unbounded per-call dict so
120
142
  single-call resolution is independent of cache size.
121
143
  2. **Promote-on-success.** Entries written during the call enter the
122
- LRU only at the end of the call, via :meth:`promote`. That keeps
123
- partial-failure semantics clean: if a call raises midway, the LRU
124
- is unchanged.
144
+ persistent cache only at the end of the call, via :meth:`promote`.
145
+ That keeps partial-failure semantics clean: if a call raises
146
+ midway, the persistent cache is unchanged.
125
147
  """
126
148
 
127
- def __init__(self, lru: _LRUDict) -> None:
149
+ def __init__(self, cache: _FIFOCache) -> None:
128
150
  super().__init__()
129
- self._lru = lru
151
+ self._cache = cache
130
152
 
131
153
  def __contains__(self, key: object) -> bool:
132
- return super().__contains__(key) or self._lru.__contains__(key)
154
+ return super().__contains__(key) or self._cache.__contains__(key)
133
155
 
134
156
  def get(self, key: str, default: Any = None) -> Any: # type: ignore[override]
135
157
  if super().__contains__(key):
136
158
  return super().__getitem__(key)
137
- return self._lru.get(key, default)
159
+ return self._cache.get(key, default)
138
160
 
139
161
  def promote(self) -> None:
140
- """Copy this call's resolved entries into the persistent LRU."""
162
+ """Copy this call's resolved entries into the persistent cache."""
141
163
  for k, v in self.items():
142
- self._lru[k] = v
164
+ self._cache[k] = v
143
165
 
144
166
 
145
167
  class Registry:
@@ -164,6 +186,15 @@ class Registry:
164
186
  DuckLake catalog the caller already opened) is done via
165
187
  :meth:`from_connection`, which doesn't take ownership of the
166
188
  connection's lifecycle.
189
+
190
+ **Concurrency.** Reads (``get`` / ``get_all`` / ``codelist`` /
191
+ ``code_map`` / ``dsd_code_map`` / ``to_registry`` / ``__contains__``
192
+ / ``__len__``) are safe to call from multiple threads against the
193
+ same ``Registry``. Writes (``add`` / ``drop`` / ``replace`` /
194
+ ``add_all`` / ``ingest``) are single-threaded — serialise externally
195
+ if you need to mutate from multiple threads. See
196
+ ``docs/design/concurrency.md`` for the model and deployment
197
+ recipes.
167
198
  """
168
199
 
169
200
  def __init__(
@@ -175,18 +206,20 @@ class Registry:
175
206
  ) -> None:
176
207
  """Open or wrap a DuckDB-backed registry.
177
208
 
178
- ``resolve_cache_size`` bounds the LRU of resolved Ref targets that
179
- :meth:`get` / :meth:`get_all` / :meth:`to_registry` share across
180
- calls. Pass ``0`` to disable caching (matches pre-v0.27.2 behaviour
181
- — one full read per Ref). The cache is cleared on every mutation
182
- (``add`` / ``drop`` / ``replace``) for invalidation correctness.
209
+ ``resolve_cache_size`` bounds the FIFO cache of resolved Ref
210
+ targets that :meth:`get` / :meth:`get_all` / :meth:`to_registry`
211
+ share across calls. Pass ``0`` to disable caching (matches
212
+ pre-v0.27.2 behaviour — one full read per Ref). The cache is
213
+ cleared on every mutation (``add`` / ``drop`` / ``replace``) for
214
+ invalidation correctness. FIFO eviction (not LRU) keeps cache
215
+ reads lock-free; see ``docs/concurrency.md``.
183
216
  """
184
217
  self._uri = Path(uri) if isinstance(uri, str) else uri
185
218
  self._source_registry = source_registry
186
219
  self._conn: duckdb.DuckDBPyConnection | None = None
187
220
  self._cursor: duckdb.DuckDBPyConnection | None = None
188
221
  self._owns_connection = True
189
- self._resolve_cache: _LRUDict = _LRUDict(resolve_cache_size)
222
+ self._resolve_cache: _FIFOCache = _FIFOCache(resolve_cache_size)
190
223
 
191
224
  @property
192
225
  def uri(self) -> Path:
@@ -250,7 +283,7 @@ class Registry:
250
283
  instance._conn = conn # noqa: SLF001
251
284
  instance._cursor = None # noqa: SLF001
252
285
  instance._owns_connection = False # noqa: SLF001
253
- instance._resolve_cache = _LRUDict(resolve_cache_size) # noqa: SLF001
286
+ instance._resolve_cache = _FIFOCache(resolve_cache_size) # noqa: SLF001
254
287
  instance._initialise_schema() # noqa: SLF001
255
288
  return instance
256
289
 
@@ -266,6 +299,11 @@ class Registry:
266
299
  else:
267
300
  self._cursor = self._conn.cursor()
268
301
  self._cursor.execute("SET search_path = 'sdmx'")
302
+ # Also set search_path on the connection's default cursor — used
303
+ # by code paths that call ``conn.execute()`` directly without
304
+ # going through ``_read_cursor`` (lazy facades' ``code_map``
305
+ # forwarder, etc.). Cheap one-shot.
306
+ self._conn.execute("SET search_path = 'sdmx'")
269
307
 
270
308
  def __enter__(self) -> Self:
271
309
  if self._conn is None:
@@ -289,10 +327,10 @@ class Registry:
289
327
  def _c(self) -> duckdb.DuckDBPyConnection:
290
328
  """Internal accessor — returns the live cursor or raises a clear error.
291
329
 
292
- Replaces direct ``self._cursor`` use after the Path-style refactor;
293
- every method that touches the database routes through this so the
294
- error message points at the lifecycle problem instead of the
295
- downstream ``AttributeError`` you'd see with a bare ``None``.
330
+ Used by write paths (``add`` / ``drop`` / ``replace``) which are
331
+ documented as single-threaded. Read paths must go through
332
+ :meth:`_read_cursor` instead so concurrent reads each get an
333
+ isolated cursor see ``docs/concurrency.md`` and issue #75.
296
334
  """
297
335
  if self._cursor is None:
298
336
  msg = (
@@ -302,6 +340,49 @@ class Registry:
302
340
  raise RuntimeError(msg)
303
341
  return self._cursor
304
342
 
343
+ def _require_open(self) -> duckdb.DuckDBPyConnection:
344
+ """Return the live connection or raise a clear lifecycle error.
345
+
346
+ Use when a caller needs the long-lived connection itself (e.g. to
347
+ hand to a lazy facade) rather than a short-lived cursor.
348
+ """
349
+ if self._conn is None:
350
+ msg = (
351
+ f"Registry({self._uri!r}) is not open. Use it as a context manager "
352
+ "(``with Registry(uri) as reg: ...``) or call ``.open()`` first."
353
+ )
354
+ raise RuntimeError(msg)
355
+ return self._conn
356
+
357
+ @contextmanager
358
+ def _read_cursor(self) -> Iterator[duckdb.DuckDBPyConnection]:
359
+ """Open a fresh DuckDB cursor for the duration of one read call.
360
+
361
+ DuckDB serialises ``execute()`` calls on a single connection but
362
+ each connection has *one* default cursor whose row buffer is
363
+ overwritten by every ``execute()``. Reads that fan out across
364
+ threads must therefore each hold their own cursor — otherwise
365
+ thread A's ``fetchone()`` can race thread B's ``execute()`` and
366
+ return rows belonging to B's query. Fixed in #75/#78.
367
+
368
+ Cursors on a read-only DuckDB connection are cheap (no
369
+ transaction state); per-call create/close has negligible
370
+ overhead at typical request rates. Sharing the connection — not
371
+ the cursor — keeps DuckLake / cataloglib federation working.
372
+ """
373
+ if self._conn is None:
374
+ msg = (
375
+ f"Registry({self._uri!r}) is not open. Use it as a context manager "
376
+ "(``with Registry(uri) as reg: ...``) or call ``.open()`` first."
377
+ )
378
+ raise RuntimeError(msg)
379
+ cur = self._conn.cursor()
380
+ try:
381
+ cur.execute("SET search_path = 'sdmx'")
382
+ yield cur
383
+ finally:
384
+ cur.close()
385
+
305
386
  # ── Add ───────────────────────────────────────────────────────────────
306
387
 
307
388
  def add(self, artefact: Any, *, source_registry: str | None = None) -> None:
@@ -313,6 +394,12 @@ class Registry:
313
394
  Clears the resolver cache: any cached resolved target may now be
314
395
  stale (e.g. a freshly-added Codelist replaces the one a previous
315
396
  ``get`` had cached).
397
+
398
+ **Single-threaded.** Mutations are not safe under concurrent
399
+ callers — do not invoke ``add`` / ``drop`` / ``replace`` from
400
+ multiple threads against the same ``Registry``. Concurrent reads
401
+ through ``get`` / ``get_all`` / ``code_map`` etc. are safe. See
402
+ ``docs/concurrency.md``.
316
403
  """
317
404
  tag = source_registry if source_registry is not None else self._source_registry
318
405
  writers.put_artefact(self._c, artefact, source_registry=tag)
@@ -342,7 +429,12 @@ class Registry:
342
429
  """
343
430
  from sdmxlib.storage.readers import make_lazy_codelist # noqa: PLC0415
344
431
 
345
- return make_lazy_codelist(self._c, str(urn))
432
+ # make_lazy_codelist opens its own internal cursor for spine
433
+ # reads; the returned LazyCodelist holds the long-lived
434
+ # connection for later method calls. ``_require_open`` enforces
435
+ # the lifecycle precondition without creating a cursor we don't
436
+ # need.
437
+ return make_lazy_codelist(self._require_open(), str(urn))
346
438
 
347
439
  def get[T](self, urn: "SdmxUrn[T]", *, eager: bool = False) -> "T | None":
348
440
  """Look up an artefact by URN.
@@ -357,14 +449,15 @@ class Registry:
357
449
  dsd.dimensions["FREQ"].codelist() # -> LazyCodelist
358
450
  dsd.code_map(lang="en") # works
359
451
  """
360
- result = readers.get_artefact(self._c, str(urn), eager=eager)
361
- if result is None:
362
- return None
363
- if not isinstance(result, (LazyCodelist, LazyHierarchy)):
364
- memo = _CallScopedMemo(self._resolve_cache)
365
- self._resolve_refs(result, memo=memo)
366
- memo.promote()
367
- return result # type: ignore[return-value]
452
+ with self._read_cursor() as cur:
453
+ result = readers.get_artefact(cur, str(urn), eager=eager, lazy_conn=self._conn)
454
+ if result is None:
455
+ return None
456
+ if not isinstance(result, (LazyCodelist, LazyHierarchy)):
457
+ memo = _CallScopedMemo(self._resolve_cache)
458
+ self._resolve_refs(cur, result, memo=memo)
459
+ memo.promote()
460
+ return result # type: ignore[return-value]
368
461
 
369
462
  def get_all[T](self, artefact_type: "type[T]") -> "list[T]":
370
463
  """All artefacts of the given Python type.
@@ -374,18 +467,20 @@ class Registry:
374
467
  parent scheme referenced by multiple artefacts is loaded once,
375
468
  not once per artefact.
376
469
  """
377
- results = readers.get_all(self._c, artefact_type)
378
- memo = _CallScopedMemo(self._resolve_cache)
379
- for r in results:
380
- if not isinstance(r, (LazyCodelist, LazyHierarchy)):
381
- self._resolve_refs(r, memo=memo)
382
- memo.promote()
383
- return results # type: ignore[return-value]
470
+ with self._read_cursor() as cur:
471
+ results = readers.get_all(cur, artefact_type, lazy_conn=self._conn)
472
+ memo = _CallScopedMemo(self._resolve_cache)
473
+ for r in results:
474
+ if not isinstance(r, (LazyCodelist, LazyHierarchy)):
475
+ self._resolve_refs(cur, r, memo=memo)
476
+ memo.promote()
477
+ return results # type: ignore[return-value]
384
478
 
385
479
  # ── Ref resolution ─────────────────────────────────────────────────
386
480
 
387
481
  def _resolve_refs( # noqa: C901
388
482
  self,
483
+ cur: duckdb.DuckDBPyConnection,
389
484
  obj: Any,
390
485
  *,
391
486
  memo: "dict[str, Any] | None" = None,
@@ -438,7 +533,7 @@ class Registry:
438
533
  needed.append(ref.urn)
439
534
 
440
535
  if needed:
441
- self._resolve_urns(needed, memo)
536
+ self._resolve_urns(cur, needed, memo)
442
537
 
443
538
  for owner, name, ref in pending:
444
539
  if ref._resolved is not None: # noqa: SLF001
@@ -479,6 +574,7 @@ class Registry:
479
574
 
480
575
  def _resolve_urns( # noqa: C901
481
576
  self,
577
+ cur: duckdb.DuckDBPyConnection,
482
578
  urns: "list[SdmxUrn[Any]]",
483
579
  memo: "dict[str, Any]",
484
580
  ) -> None:
@@ -507,7 +603,7 @@ class Registry:
507
603
  if scheme_info is None:
508
604
  continue
509
605
  _scheme_class_name, items_attr = scheme_info
510
- parent = self._load_parent_scheme(parent_urn_str, sample, memo)
606
+ parent = self._load_parent_scheme(cur, parent_urn_str, sample, memo)
511
607
  if parent is None:
512
608
  continue
513
609
  items = getattr(parent, items_attr, None)
@@ -522,7 +618,7 @@ class Registry:
522
618
  urn_str = str(parsed)
523
619
  if urn_str in memo:
524
620
  continue
525
- target = readers.get_artefact(self._c, urn_str)
621
+ target = readers.get_artefact(cur, urn_str, lazy_conn=self._conn)
526
622
  if target is not None:
527
623
  memo[urn_str] = target
528
624
 
@@ -547,6 +643,7 @@ class Registry:
547
643
 
548
644
  def _load_parent_scheme(
549
645
  self,
646
+ cur: duckdb.DuckDBPyConnection,
550
647
  parent_urn_str: str,
551
648
  sample: "SdmxUrn[Any]",
552
649
  memo: "dict[str, Any]",
@@ -559,12 +656,12 @@ class Registry:
559
656
  cached = memo.get(parent_urn_str)
560
657
  if cached is not None:
561
658
  return cached
562
- parent = readers.get_artefact(self._c, parent_urn_str, eager=True)
659
+ parent = readers.get_artefact(cur, parent_urn_str, eager=True)
563
660
  if parent is None and sample.version == "latest":
564
661
  scheme_info = _ITEM_TO_SCHEME.get(sample.artefact_class)
565
662
  if scheme_info is not None:
566
663
  scheme_class_name, _ = scheme_info
567
- row = self._c.execute(
664
+ row = cur.execute(
568
665
  """
569
666
  SELECT urn FROM artefact
570
667
  WHERE artefact_type = ? AND agency_id = ? AND resource_id = ?
@@ -572,7 +669,7 @@ class Registry:
572
669
  [scheme_class_name, sample.agency, sample.id],
573
670
  ).fetchone()
574
671
  if row:
575
- parent = readers.get_artefact(self._c, row[0], eager=True)
672
+ parent = readers.get_artefact(cur, row[0], eager=True)
576
673
  if parent is not None:
577
674
  memo[parent_urn_str] = parent
578
675
  return parent
@@ -587,19 +684,22 @@ class Registry:
587
684
 
588
685
  def __contains__(self, urn: object) -> bool:
589
686
  if isinstance(urn, SdmxUrn):
590
- row = self._c.execute(
591
- "SELECT 1 FROM artefact WHERE urn = ?",
592
- [str(urn)],
593
- ).fetchone()
594
- return row is not None
687
+ with self._read_cursor() as cur:
688
+ row = cur.execute(
689
+ "SELECT 1 FROM artefact WHERE urn = ?",
690
+ [str(urn)],
691
+ ).fetchone()
692
+ return row is not None
595
693
  return False
596
694
 
597
695
  def __len__(self) -> int:
598
- row = self._c.execute("SELECT COUNT(*) FROM artefact").fetchone()
599
- return row[0] # type: ignore[index]
696
+ with self._read_cursor() as cur:
697
+ row = cur.execute("SELECT COUNT(*) FROM artefact").fetchone()
698
+ return row[0] # type: ignore[index]
600
699
 
601
700
  def __repr__(self) -> str:
602
- rows = self._c.execute("SELECT artefact_type, COUNT(*) FROM artefact GROUP BY artefact_type").fetchall()
701
+ with self._read_cursor() as cur:
702
+ rows = cur.execute("SELECT artefact_type, COUNT(*) FROM artefact GROUP BY artefact_type").fetchall()
603
703
  counts = ", ".join(f"{name}={cnt}" for name, cnt in rows)
604
704
  return f"Registry({counts or 'empty'})"
605
705
 
@@ -610,6 +710,8 @@ class Registry:
610
710
 
611
711
  Clears the resolver cache (cached resolved targets may reference
612
712
  the dropped artefact).
713
+
714
+ **Single-threaded** — see :meth:`add`.
613
715
  """
614
716
  result = writers.drop_artefact(self._c, str(urn))
615
717
  self._resolve_cache.clear()
@@ -620,6 +722,8 @@ class Registry:
620
722
 
621
723
  Each leg clears the resolver cache via :meth:`add` / :meth:`drop`;
622
724
  the net effect is a single fully-cleared cache at the end.
725
+
726
+ **Single-threaded** — see :meth:`add`.
623
727
  """
624
728
  writers.drop_artefact(self._c, str(old_urn))
625
729
  self.add(new)
@@ -655,11 +759,13 @@ class Registry:
655
759
 
656
760
  def code_map(self, codelist_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, str]:
657
761
  """Return {code_id: label} for a codelist without creating Code objects."""
658
- return readers.code_map(self._c, str(codelist_urn), lang=lang)
762
+ with self._read_cursor() as cur:
763
+ return readers.code_map(cur, str(codelist_urn), lang=lang)
659
764
 
660
765
  def dsd_code_map(self, dsd_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, dict[str, str]]:
661
766
  """Return {component_id: {code_id: label}} for all enumerated dimensions."""
662
- return readers.code_map_for_dsd(self._c, str(dsd_urn), lang=lang)
767
+ with self._read_cursor() as cur:
768
+ return readers.code_map_for_dsd(cur, str(dsd_urn), lang=lang)
663
769
 
664
770
  # ── Bulk ──────────────────────────────────────────────────────────────
665
771
 
@@ -687,12 +793,13 @@ class Registry:
687
793
  """
688
794
  reg = InMemoryRegistry()
689
795
  memo = _CallScopedMemo(self._resolve_cache)
690
- rows = self._c.execute("SELECT urn, artefact_type FROM artefact").fetchall()
691
- for urn_str, _artefact_type in rows:
692
- obj = readers.get_artefact(self._c, urn_str, eager=True)
693
- if obj is None:
694
- continue
695
- self._resolve_refs(obj, memo=memo)
696
- reg.add(obj)
796
+ with self._read_cursor() as cur:
797
+ rows = cur.execute("SELECT urn, artefact_type FROM artefact").fetchall()
798
+ for urn_str, _artefact_type in rows:
799
+ obj = readers.get_artefact(cur, urn_str, eager=True)
800
+ if obj is None:
801
+ continue
802
+ self._resolve_refs(cur, obj, memo=memo)
803
+ reg.add(obj)
697
804
  memo.promote()
698
805
  return reg
@@ -35,6 +35,11 @@ if TYPE_CHECKING:
35
35
  import duckdb
36
36
 
37
37
 
38
+ def _cur(conn: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyConnection:
39
+ """Open a fresh per-call cursor (issue #75). SQL here is sdmx.-qualified."""
40
+ return conn.cursor()
41
+
42
+
38
43
  # ── Field expressions (AST nodes) ───────────────────────────────────────
39
44
 
40
45
 
@@ -445,7 +450,7 @@ class CodeQuery:
445
450
  dominates on large codelists.
446
451
  """
447
452
  sql, params = _compile(self)
448
- rows = self._conn.execute(sql, params).fetchall()
453
+ rows = _cur(self._conn).execute(sql, params).fetchall()
449
454
  if not rows:
450
455
  return []
451
456
  if self.annotation_types:
@@ -458,7 +463,7 @@ class CodeQuery:
458
463
  def ids(self) -> list[str]:
459
464
  """Compile to SQL, execute, and return just the code ids."""
460
465
  sql, params = _compile(self, select_clause="c.code_id")
461
- rows = self._conn.execute(sql, params).fetchall()
466
+ rows = _cur(self._conn).execute(sql, params).fetchall()
462
467
  return [r[0] for r in rows]
463
468
 
464
469
  def to_sql(self) -> tuple[str, list[object]]:
@@ -481,7 +486,7 @@ class CodeQuery:
481
486
 
482
487
  def __len__(self) -> int:
483
488
  sql, params = _compile(self, select_clause="COUNT(*)", omit_order=True)
484
- row = self._conn.execute(sql, params).fetchone()
489
+ row = _cur(self._conn).execute(sql, params).fetchone()
485
490
  return row[0] if row else 0 # type: ignore[index]
486
491
 
487
492
  @overload
@@ -492,29 +497,37 @@ class CodeQuery:
492
497
 
493
498
  def __getitem__(self, key: int | str) -> Code:
494
499
  if isinstance(key, str):
495
- row = self._conn.execute(
496
- """
500
+ row = (
501
+ _cur(self._conn)
502
+ .execute(
503
+ """
497
504
  SELECT c.code_id, c.parent_id, c.depth, c.name, c.code_urn, c.description
498
505
  FROM sdmx.code c
499
506
  WHERE c.codelist_urn = ? AND c.code_id = ?
500
507
  """,
501
- [self.codelist_urn, key],
502
- ).fetchone()
508
+ [self.codelist_urn, key],
509
+ )
510
+ .fetchone()
511
+ )
503
512
  if row is None:
504
513
  raise KeyError(key)
505
514
  return _row_to_code(row, self._anns_for_one(row[4]))
506
515
 
507
516
  offset = key if key >= 0 else len(self) + key
508
- row = self._conn.execute(
509
- """
517
+ row = (
518
+ _cur(self._conn)
519
+ .execute(
520
+ """
510
521
  SELECT c.code_id, c.parent_id, c.depth, c.name, c.code_urn, c.description
511
522
  FROM sdmx.code c
512
523
  WHERE c.codelist_urn = ?
513
524
  ORDER BY c.position NULLS LAST, c.code_id
514
525
  LIMIT 1 OFFSET ?
515
526
  """,
516
- [self.codelist_urn, offset],
517
- ).fetchone()
527
+ [self.codelist_urn, offset],
528
+ )
529
+ .fetchone()
530
+ )
518
531
  if row is None:
519
532
  raise IndexError(key)
520
533
  return _row_to_code(row, self._anns_for_one(row[4]))
@@ -535,10 +548,14 @@ class CodeQuery:
535
548
 
536
549
  def __contains__(self, key: object) -> bool:
537
550
  if isinstance(key, str):
538
- row = self._conn.execute(
539
- "SELECT 1 FROM sdmx.code WHERE codelist_urn = ? AND code_id = ?",
540
- [self.codelist_urn, key],
541
- ).fetchone()
551
+ row = (
552
+ _cur(self._conn)
553
+ .execute(
554
+ "SELECT 1 FROM sdmx.code WHERE codelist_urn = ? AND code_id = ?",
555
+ [self.codelist_urn, key],
556
+ )
557
+ .fetchone()
558
+ )
542
559
  return row is not None
543
560
  if isinstance(key, Code):
544
561
  return key.id in self
@@ -947,13 +964,16 @@ def _fetch_annotations_for(
947
964
  sql_parts.append(f"AND annotation_type IN ({type_placeholders})")
948
965
  sql_params.extend(types)
949
966
  sql_parts.append("ORDER BY owner_urn, ordinal")
950
- ann_rows = conn.execute(" ".join(sql_parts), sql_params).fetchall()
967
+ # Use a single per-call cursor across both queries to keep concurrent
968
+ # callers isolated from each other's row buffers (issue #75).
969
+ cur = _cur(conn)
970
+ ann_rows = cur.execute(" ".join(sql_parts), sql_params).fetchall()
951
971
  if not ann_rows:
952
972
  return {}
953
973
 
954
974
  ann_ids = [r[0] for r in ann_rows]
955
975
  text_placeholders = ", ".join("?" for _ in ann_ids)
956
- text_rows = conn.execute(
976
+ text_rows = cur.execute(
957
977
  f"""
958
978
  SELECT annotation_id, lang, text
959
979
  FROM sdmx.annotation_text