sdmxlib 0.28.0__tar.gz → 0.28.2__tar.gz

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.28.0
+Version: 0.28.2
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.28.0"
+version = "0.28.2"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/src/sdmxlib/local/_data_store.py

@@ -15,6 +15,7 @@ file-handle juggling.
 
 from __future__ import annotations
 
+import threading
 from collections.abc import Iterator
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal, Self
@@ -98,6 +99,8 @@ class DuckDBDataStore:
         self._registry: Registry | None = None
         self._conn: duckdb.DuckDBPyConnection | None = None
         self._owns_connection = True
+        # Per-thread read cursor (#80); see Registry._read_cursor.
+        self._thread_local: threading.local = threading.local()
 
     @property
     def uri(self) -> Path:
@@ -171,6 +174,7 @@ class DuckDBDataStore:
         instance._registry = registry  # noqa: SLF001
         instance._conn = registry.connection  # noqa: SLF001
         instance._owns_connection = False  # noqa: SLF001
+        instance._thread_local = threading.local()  # noqa: SLF001
         instance._configure_session()  # noqa: SLF001
         return instance
 
@@ -191,6 +195,15 @@ class DuckDBDataStore:
             # Loading extensions can change the search_path; reset to
             # ``sdmx`` so any colocated Registry's internal queries stay clean.
             self._conn.execute("SET search_path = 'sdmx'")
+        if self._needs_azure():
+            # Force the named curl transport adapter. The Azure SDK's
+            # default transport on Linux uses a statically-linked HTTP
+            # stack that doesn't honour the standard ``SSL_CERT_FILE`` /
+            # ``CURL_CA_BUNDLE`` environment variables, so it fails on
+            # any image whose CA store is at a non-default path. The
+            # ``curl`` adapter delegates to libcurl, which reads those
+            # env vars at handshake time.
+            self._conn.execute("SET azure_transport_option_type = 'curl'")
         if self._azure_credentials:
             sql = _azure_create_secret_sql(self._azure_credentials)
             self._conn.execute(sql)
@@ -227,6 +240,22 @@ class DuckDBDataStore:
         if self._conn is not None and self._owns_connection:
             self._conn.close()
         self._conn = None
+        self._thread_local = threading.local()
+
+    def _read_cursor(self) -> duckdb.DuckDBPyConnection:
+        """Return this thread's read cursor for ``query`` / ``query_stream``.
+
+        Mirror of :meth:`Registry._read_cursor` (#80). Lazily created on
+        first call per thread, reused for the thread's lifetime.
+        """
+        cur = getattr(self._thread_local, "cur", None)
+        if cur is not None:
+            return cur
+        conn = self.connection
+        cur = conn.cursor()
+        cur.execute("SET search_path = 'sdmx'")
+        self._thread_local.cur = cur
+        return cur
 
     # ── DataStore protocol ───────────────────────────────────────────────
 
@@ -267,8 +296,8 @@ class DuckDBDataStore:
             order_by=order_by,
             limit=limit,
         )
-        # Per-call cursor for issue #75 (concurrent reads).
-        cur = conn.cursor().execute(sql, sql_params)
+        # Thread-local cursor for issue #75 / #80 (concurrent reads).
+        cur = self._read_cursor().execute(sql, sql_params)
         col_names = [d[0] for d in cur.description] if cur.description else []
         rows = cur.fetchall()
         if not rows:

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/src/sdmxlib/local/_registry.py

@@ -31,8 +31,7 @@ stashes the URI; the DuckDB connection is opened by ``open()`` /
 """
 
 import threading
-from collections.abc import Iterable, Iterator
-from contextlib import contextmanager
+from collections.abc import Iterable
 from pathlib import Path
 from typing import Any, Self
 
@@ -220,6 +219,10 @@ class Registry:
         self._cursor: duckdb.DuckDBPyConnection | None = None
         self._owns_connection = True
         self._resolve_cache: _FIFOCache = _FIFOCache(resolve_cache_size)
+        # Per-thread read cursor (#80). Each thread that reads from this
+        # Registry lazily gets its own cursor on first call; subsequent
+        # calls on the same thread reuse it. Cleared on close().
+        self._thread_local: threading.local = threading.local()
 
     @property
     def uri(self) -> Path:
@@ -284,6 +287,7 @@ class Registry:
         instance._cursor = None  # noqa: SLF001
         instance._owns_connection = False  # noqa: SLF001
         instance._resolve_cache = _FIFOCache(resolve_cache_size)  # noqa: SLF001
+        instance._thread_local = threading.local()  # noqa: SLF001
         instance._initialise_schema()  # noqa: SLF001
         return instance
 
@@ -317,11 +321,19 @@ class Registry:
         """Close the database connection (only if we own it).
 
         After ``close()``, the registry can be re-opened with ``open()``.
+        Drops the per-thread cursor cache — reopen starts fresh.
         """
         if self._conn is not None and self._owns_connection:
             self._conn.close()
         self._conn = None
         self._cursor = None
+        # Drop per-thread cursors. The cursor objects become orphaned —
+        # DuckDB's close() invalidates them; Python GC reclaims the
+        # handles. We can't iterate other threads' attributes on the
+        # threadlocal, but reassigning replaces the storage for *this*
+        # thread and lets the prior thread-local mapping get GC'd
+        # (other threads' attrs become unreachable too).
+        self._thread_local = threading.local()
 
     @property
     def _c(self) -> duckdb.DuckDBPyConnection:
@@ -354,22 +366,25 @@ class Registry:
             raise RuntimeError(msg)
         return self._conn
 
-    @contextmanager
-    def _read_cursor(self) -> Iterator[duckdb.DuckDBPyConnection]:
-        """Open a fresh DuckDB cursor for the duration of one read call.
-
-        DuckDB serialises ``execute()`` calls on a single connection but
-        each connection has *one* default cursor whose row buffer is
-        overwritten by every ``execute()``. Reads that fan out across
-        threads must therefore each hold their own cursor — otherwise
-        thread A's ``fetchone()`` can race thread B's ``execute()`` and
-        return rows belonging to B's query. Fixed in #75/#78.
-
-        Cursors on a read-only DuckDB connection are cheap (no
-        transaction state); per-call create/close has negligible
-        overhead at typical request rates. Sharing the connection — not
-        the cursor — keeps DuckLake / cataloglib federation working.
+    def _read_cursor(self) -> duckdb.DuckDBPyConnection:
+        """Return this thread's read cursor, creating it on first call.
+
+        Each thread gets its own cursor lazily; subsequent calls on the
+        same thread reuse it. DuckDB requires per-thread cursors (the
+        connection's default-cursor row buffer is overwritten by every
+        ``execute()`` — see #75), but it does *not* require recreating
+        the cursor per call. Thread-local reuse amortises the
+        ``cursor()`` + ``SET search_path`` cost across the thread's
+        lifetime — typically the entire process for a long-lived
+        FastAPI/uvicorn threadpool.
+
+        Cleanup: the thread-local storage is dropped by :meth:`close`;
+        otherwise it lives until the thread dies (Python GC closes the
+        cursor object).
         """
+        cur = getattr(self._thread_local, "cur", None)
+        if cur is not None:
+            return cur
         if self._conn is None:
             msg = (
                 f"Registry({self._uri!r}) is not open. Use it as a context manager "
@@ -377,11 +392,9 @@ class Registry:
             )
             raise RuntimeError(msg)
         cur = self._conn.cursor()
-        try:
-            cur.execute("SET search_path = 'sdmx'")
-            yield cur
-        finally:
-            cur.close()
+        cur.execute("SET search_path = 'sdmx'")
+        self._thread_local.cur = cur
+        return cur
 
     # ── Add ───────────────────────────────────────────────────────────────
 
@@ -449,15 +462,15 @@ class Registry:
             dsd.dimensions["FREQ"].codelist()  # -> LazyCodelist
             dsd.code_map(lang="en")  # works
         """
-        with self._read_cursor() as cur:
-            result = readers.get_artefact(cur, str(urn), eager=eager, lazy_conn=self._conn)
-            if result is None:
-                return None
-            if not isinstance(result, (LazyCodelist, LazyHierarchy)):
-                memo = _CallScopedMemo(self._resolve_cache)
-                self._resolve_refs(cur, result, memo=memo)
-                memo.promote()
-            return result  # type: ignore[return-value]
+        cur = self._read_cursor()
+        result = readers.get_artefact(cur, str(urn), eager=eager, lazy_conn=self._conn)
+        if result is None:
+            return None
+        if not isinstance(result, (LazyCodelist, LazyHierarchy)):
+            memo = _CallScopedMemo(self._resolve_cache)
+            self._resolve_refs(cur, result, memo=memo)
+            memo.promote()
+        return result  # type: ignore[return-value]
 
     def get_all[T](self, artefact_type: "type[T]") -> "list[T]":
         """All artefacts of the given Python type.
@@ -467,14 +480,14 @@ class Registry:
         parent scheme referenced by multiple artefacts is loaded once,
         not once per artefact.
         """
-        with self._read_cursor() as cur:
-            results = readers.get_all(cur, artefact_type, lazy_conn=self._conn)
-            memo = _CallScopedMemo(self._resolve_cache)
-            for r in results:
-                if not isinstance(r, (LazyCodelist, LazyHierarchy)):
-                    self._resolve_refs(cur, r, memo=memo)
-            memo.promote()
-            return results  # type: ignore[return-value]
+        cur = self._read_cursor()
+        results = readers.get_all(cur, artefact_type, lazy_conn=self._conn)
+        memo = _CallScopedMemo(self._resolve_cache)
+        for r in results:
+            if not isinstance(r, (LazyCodelist, LazyHierarchy)):
+                self._resolve_refs(cur, r, memo=memo)
+        memo.promote()
+        return results  # type: ignore[return-value]
 
     # ── Ref resolution ─────────────────────────────────────────────────
 
@@ -684,22 +697,22 @@ class Registry:
 
     def __contains__(self, urn: object) -> bool:
         if isinstance(urn, SdmxUrn):
-            with self._read_cursor() as cur:
-                row = cur.execute(
-                    "SELECT 1 FROM artefact WHERE urn = ?",
-                    [str(urn)],
-                ).fetchone()
-                return row is not None
+            cur = self._read_cursor()
+            row = cur.execute(
+                "SELECT 1 FROM artefact WHERE urn = ?",
+                [str(urn)],
+            ).fetchone()
+            return row is not None
         return False
 
     def __len__(self) -> int:
-        with self._read_cursor() as cur:
-            row = cur.execute("SELECT COUNT(*) FROM artefact").fetchone()
-            return row[0]  # type: ignore[index]
+        cur = self._read_cursor()
+        row = cur.execute("SELECT COUNT(*) FROM artefact").fetchone()
+        return row[0]  # type: ignore[index]
 
     def __repr__(self) -> str:
-        with self._read_cursor() as cur:
-            rows = cur.execute("SELECT artefact_type, COUNT(*) FROM artefact GROUP BY artefact_type").fetchall()
+        cur = self._read_cursor()
+        rows = cur.execute("SELECT artefact_type, COUNT(*) FROM artefact GROUP BY artefact_type").fetchall()
         counts = ", ".join(f"{name}={cnt}" for name, cnt in rows)
         return f"Registry({counts or 'empty'})"
 
@@ -759,13 +772,11 @@ class Registry:
 
     def code_map(self, codelist_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, str]:
         """Return {code_id: label} for a codelist without creating Code objects."""
-        with self._read_cursor() as cur:
-            return readers.code_map(cur, str(codelist_urn), lang=lang)
+        return readers.code_map(self._read_cursor(), str(codelist_urn), lang=lang)
 
     def dsd_code_map(self, dsd_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, dict[str, str]]:
         """Return {component_id: {code_id: label}} for all enumerated dimensions."""
-        with self._read_cursor() as cur:
-            return readers.code_map_for_dsd(cur, str(dsd_urn), lang=lang)
+        return readers.code_map_for_dsd(self._read_cursor(), str(dsd_urn), lang=lang)
 
     # ── Bulk ──────────────────────────────────────────────────────────────
 

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/src/sdmxlib/model/code_query.py

@@ -35,9 +35,24 @@ if TYPE_CHECKING:
     import duckdb
 
 
+import threading as _threading
+import weakref as _weakref
+
+# Per-thread per-connection cursor store (#80). See ``sdmxlib.storage.lazy._cur``.
+_thread_local = _threading.local()
+
+
 def _cur(conn: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyConnection:
-    """Open a fresh per-call cursor (issue #75). SQL here is sdmx.-qualified."""
-    return conn.cursor()
+    """Return this thread's cursor for ``conn`` (issue #80)."""
+    table = getattr(_thread_local, "table", None)
+    if table is None:
+        table = _weakref.WeakKeyDictionary()
+        _thread_local.table = table
+    cur = table.get(conn)
+    if cur is None:
+        cur = conn.cursor()
+        table[conn] = cur
+    return cur
 
 
 # ── Field expressions (AST nodes) ───────────────────────────────────────

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/src/sdmxlib/model/hierarchy_query.py

@@ -30,9 +30,24 @@ if TYPE_CHECKING:
     import duckdb
 
 
+import threading as _threading
+import weakref as _weakref
+
+# Per-thread per-connection cursor store (#80). See ``sdmxlib.storage.lazy._cur``.
+_thread_local = _threading.local()
+
+
 def _cur(conn: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyConnection:
-    """Open a fresh per-call cursor (issue #75). SQL here is sdmx.-qualified."""
-    return conn.cursor()
+    """Return this thread's cursor for ``conn`` (issue #80)."""
+    table = getattr(_thread_local, "table", None)
+    if table is None:
+        table = _weakref.WeakKeyDictionary()
+        _thread_local.table = table
+    cur = table.get(conn)
+    if cur is None:
+        cur = conn.cursor()
+        table[conn] = cur
+    return cur
 
 
 # ── Field expressions ────────────────────────────────────────────────────

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/src/sdmxlib/storage/lazy.py

@@ -8,6 +8,9 @@ Hierarchy navigation (``roots``, ``leaves``, ``narrower``,
 appropriate persisted table (``sdmx.code`` or ``sdmx.hierarchy_node``).
 """
 
+import threading
+import weakref
+
 import duckdb
 
 from sdmxlib.model.code_query import (
@@ -46,25 +49,38 @@ def _map_to_istring(m: dict[str, str] | None) -> InternationalString:
     return InternationalString.of(m)
 
 
+# Per-thread per-connection cursor store (#75 / #80). One cursor per
+# (thread, connection) pair, created lazily on first use, reused for the
+# thread's lifetime. Connections are held weakly so closing a Registry
+# (which closes its connection and drops references) lets the entry GC
+# without needing explicit cleanup hooks — no stale cursors after id
+# reuse.
+_thread_local = threading.local()
+
+
 def _cur(conn: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyConnection:
-    """Open a fresh per-call cursor (issue #75).
+    """Return this thread's cursor for ``conn``, creating on first call.
 
     Lazy facades and query classes share a long-lived connection across
-    method calls, but each ``execute()`` needs its own cursor: DuckDB
-    reuses a single default-cursor row buffer per connection, and
+    method calls, but each ``execute()`` needs its own per-thread cursor:
+    DuckDB reuses a single default-cursor row buffer per connection, and
     concurrent ``.execute()`` calls on the same buffer overwrite each
-    other's results.
-
-    No ``SET search_path`` here — all SQL in this module qualifies
-    tables with ``sdmx.`` explicitly, so a fresh cursor's lack of
-    inherited search path doesn't matter. (Other code paths that issue
-    unqualified SQL set search_path themselves.)
+    other's results (issue #75). Thread-local reuse amortises the
+    ``cursor()`` cost across the thread's lifetime (issue #80).
 
-    Caller pattern: ``rows = _cur(self._conn).execute(SQL, params).fetchall()``.
-    The cursor object stays referenced through the chained call, then
-    falls out of scope and is reclaimed.
+    No ``SET search_path`` — all SQL in this module qualifies tables with
+    ``sdmx.`` explicitly. (Other code paths that issue unqualified SQL
+    set search_path themselves.)
     """
-    return conn.cursor()
+    table = getattr(_thread_local, "table", None)
+    if table is None:
+        table = weakref.WeakKeyDictionary()
+        _thread_local.table = table
+    cur = table.get(conn)
+    if cur is None:
+        cur = conn.cursor()
+        table[conn] = cur
+    return cur
 
 
 class LazyCodelist:

{sdmxlib-0.28.0 → sdmxlib-0.28.2}/src/sdmxlib/storage/readers.py

@@ -13,6 +13,8 @@ Example::
     cm = readers.code_map(conn, codelist_urn, lang="en")  # {code_id: label}
 """
 
+import threading
+import weakref
 from typing import Any, cast
 
 import duckdb
@@ -67,6 +69,26 @@ from sdmxlib.storage.writers import _CLASS_TO_NAME
 _TYPE_REGISTRY: dict[str, type] = {v: k for k, v in _CLASS_TO_NAME.items()}
 
 
+# Per-thread cursor store for ``code_map`` / ``code_map_for_dsd`` (#80).
+# Held weakly so a closed Registry's connection doesn't pin a stale
+# cursor entry (avoids the id-reuse hazard a plain id-keyed dict would
+# have when Python recycles the connection's id after GC).
+_thread_local = threading.local()
+
+
+def _thread_cursor(conn_or_cur: duckdb.DuckDBPyConnection) -> duckdb.DuckDBPyConnection:
+    table = getattr(_thread_local, "table", None)
+    if table is None:
+        table = weakref.WeakKeyDictionary()
+        _thread_local.table = table
+    cur = table.get(conn_or_cur)
+    if cur is None:
+        cur = conn_or_cur.cursor()
+        cur.execute("SET search_path = 'sdmx'")
+        table[conn_or_cur] = cur
+    return cur
+
+
 # ── Shared helpers ──────────────────────────────────────────────────────
 
 
@@ -1547,7 +1569,7 @@ def code_map(
     upfront configuration. Opens its own per-call cursor (issue #75):
     safe to call concurrently from threads sharing the same connection.
     """
-    cur = conn.cursor()
+    cur = _thread_cursor(conn)
     rows = cur.execute(
         f"""
         SELECT code_id, name['{lang}'] AS label
@@ -1570,7 +1592,7 @@ def code_map_for_dsd(
 
     Single JOIN query — no N+1. Per-call cursor as in :func:`code_map`.
     """
-    cur = conn.cursor()
+    cur = _thread_cursor(conn)
     rows = cur.execute(
         f"""
         SELECT dc.component_id, c.code_id, c.name['{lang}'] AS label