sdmxlib 0.41.2__tar.gz → 0.41.3__tar.gz

{sdmxlib-0.41.2 → sdmxlib-0.41.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.41.2
+Version: 0.41.3
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Dist: attrs>=26.1.0
-Requires-Dist: duckdb>=1.0.0
+Requires-Dist: duckdb>=1.5.3
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: lxml>=6.0.2
 Requires-Dist: orjson>=3.9

{sdmxlib-0.41.2 → sdmxlib-0.41.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.41.2"
+version = "0.41.3"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -17,7 +17,11 @@ classifiers = [
 ]
 dependencies = [
     "attrs>=26.1.0",
-    "duckdb>=1.0.0",
+    # >=1.5.3: ducklake extension must support DuckLake catalog format 1.0,
+    # which is what statcan-catalog-builder (duckdb 1.5.3) writes into
+    # published bundles. duckdb 1.5.1's ducklake tops out at format 0.4 and
+    # fails to ATTACH such bundles with "DuckLake catalog version mismatch".
+    "duckdb>=1.5.3",
     "httpx>=0.28.1",
     "lxml>=6.0.2",
     "orjson>=3.9",

{sdmxlib-0.41.2 → sdmxlib-0.41.3}/src/sdmxlib/__init__.py

@@ -11,7 +11,7 @@ from sdmxlib.data_store import DataStore
 from sdmxlib.sql import CompileContext, Predicate, SqlRelation
 from sdmxlib.formats import parse, read, serialize, to_json, to_xml
 from sdmxlib.local import LazyCodelist, LazyHierarchy, Registry
-from sdmxlib.local.data_store import DuckDBDataStore
+from sdmxlib.local.data_store import DuckDBDataStore, DuckLakeVersionError
 from sdmxlib.rest import ComponentFilter, parse_components
 from sdmxlib.model.convert import from_dict, to_dict
 from sdmxlib.model.validation import (
@@ -167,6 +167,7 @@ __all__ = [
     "DriftError",
     "DriftIssue",
     "DuckDBDataStore",
+    "DuckLakeVersionError",
     "Facet",
     "FailurePolicy",
     "FederatedCatalog",

{sdmxlib-0.41.2 → sdmxlib-0.41.3}/src/sdmxlib/local/data_store.py

@@ -37,6 +37,29 @@ if TYPE_CHECKING:
     WriteSource = pl.DataFrame | pl.LazyFrame | pa.Table | duckdb.DuckDBPyRelation
 
 
+class DuckLakeVersionError(Exception):  # mirrors RegistryUnavailable etc.
+    """A DuckLake catalog's format is newer than the installed extension.
+
+    Raised when ATTACH-ing a bundle whose DuckLake catalog format exceeds
+    what the loaded ``ducklake`` extension supports. The fix is to align
+    versions (upgrade ``duckdb`` >= 1.5.3 and/or the ``ducklake``
+    extension), not to migrate the bundle in place. See issue #156.
+    """
+
+
+def _is_ducklake_version_mismatch(exc: Exception) -> bool:
+    """True if ``exc`` is a DuckDB error about an unsupported DuckLake format.
+
+    Matches the two shapes DuckDB raises depending on the skew:
+
+    - ``DuckLake catalog version mismatch: catalog version is X, but the
+      extension requires version Y``
+    - ``Only DuckLake versions ... are supported``
+    """
+    msg = str(exc).lower()
+    return "ducklake" in msg and ("catalog version mismatch" in msg or "are supported" in msg)
+
+
 class DuckDBDataStore:
     """A `DataStore` backed by DuckDB.
 
@@ -215,13 +238,37 @@ class DuckDBDataStore:
             sql = _azure_create_secret_sql(self._azure_credentials)
             self._conn.execute(sql)
         if self._attach:
-            for alias, spec in self._attach.items():
-                url, options = _split_attach_spec(spec)
-                sql = f"ATTACH {url} AS {_sql_identifier(alias)}"
-                if options:
-                    sql += f" {options}"
-                self._conn.execute(sql)
-            self._conn.execute("SET search_path = 'sdmx'")
+            self._run_attaches()
+
+    def _run_attaches(self) -> None:
+        """ATTACH each configured catalog, translating DuckLake skew errors.
+
+        A DuckLake catalog whose format is newer than the installed
+        ``ducklake`` extension surfaces as an opaque DuckDB exception; we
+        re-raise it as `DuckLakeVersionError` with the fix. Any other
+        ATTACH failure propagates unchanged. See issue #156.
+        """
+        conn = self._conn
+        if conn is None:  # guarded by caller; narrows the type here too
+            return
+        for alias, spec in (self._attach or {}).items():
+            url, options = _split_attach_spec(spec)
+            sql = f"ATTACH {url} AS {_sql_identifier(alias)}"
+            if options:
+                sql += f" {options}"
+            try:
+                conn.execute(sql)
+            except duckdb.Error as exc:
+                if _is_ducklake_version_mismatch(exc):
+                    msg = (
+                        f"cannot attach DuckLake catalog {alias!r}: its catalog format is "
+                        "newer than the installed ducklake extension supports. Upgrade "
+                        "duckdb (>=1.5.3) and/or the ducklake extension, then retry. "
+                        f"(underlying: {exc})"
+                    )
+                    raise DuckLakeVersionError(msg) from exc
+                raise
+        conn.execute("SET search_path = 'sdmx'")
 
     def _needs_azure(self) -> bool:
         """True if Azure secret/extension setup is required for this session.

{sdmxlib-0.41.2 → sdmxlib-0.41.3}/src/sdmxlib/polars.py

@@ -19,7 +19,6 @@ directly for standalone use::
 
 from __future__ import annotations
 
-import weakref
 from typing import TYPE_CHECKING, Any
 
 import polars as pl
@@ -175,25 +174,15 @@ def validation_report(
 # slpl.label() and _label_view() are hot paths. Building the id→label
 # dict by iterating ``codelist.codes`` materialises a Code object per
 # row, which dominates on large codelists (CL_GEOGRAPHY: 78k codes →
-# ~16s per call). We cache the dict per (codelist instance, lang) pair.
+# ~16s per call).
 #
-# Identity is the codelist *object* (weakref so the cache doesn't pin
-# instances). When the codelist is garbage-collected the cache entry
-# goes with it. A fresh ingest produces a new instance and a fresh
-# cache slot, so re-ingestion is automatically correct without a
-# manual invalidation call.
-
-_LABEL_MAP_CACHE: weakref.WeakKeyDictionary[object, dict[str, dict[str, str]]] = weakref.WeakKeyDictionary()
-
-
-def clear_label_cache() -> None:
-    """Drop the cached id→label maps used by ``label`` / ``_label_view``.
-
-    The cache is normally invalidated automatically when the underlying
-    codelist instance is garbage-collected. Call this explicitly in
-    tests or other contexts that want to force a rebuild.
-    """
-    _LABEL_MAP_CACHE.clear()
+# Caching lives at the *source*, not here (#155): `LazyCodelist.code_map`
+# memoises per instance per lang with a single-flight build lock, so every
+# code_map caller — not just label() — dedups, and concurrent first-time
+# callers share one build instead of stampeding the DuckDB connection.
+# Plain in-memory `Codelist` has no code_map and is rebuilt each call, but
+# that path is pure-Python dict construction (no SQL) — cheap enough not to
+# warrant a second cache. See docs/design/caching.md.
 
 
 def _component_label_map(component: object, lang: str) -> dict[str, str] | None:
@@ -202,8 +191,9 @@ def _component_label_map(component: object, lang: str) -> dict[str, str] | None:
     Fast path: when ``component.representation()`` is a
     `LazyCodelist` (or anything with a
     ``code_map(*, lang)`` method), use direct-SQL projection — single
-    DuckDB query, no Code/InternationalString construction. Falls back
-    to in-memory iteration for plain `Codelist` instances.
+    DuckDB query (cached + single-flight at the source), no
+    Code/InternationalString construction. Falls back to in-memory
+    iteration for plain `Codelist` instances.
 
     Returns ``None`` when the component is not enumerated or its
     representation Ref is unresolved.
@@ -214,33 +204,13 @@ def _component_label_map(component: object, lang: str) -> dict[str, str] | None:
     if not (isinstance(rep, Ref) and rep.resolved):
         return None
     cl: object = rep()
-    try:
-        per_lang = _LABEL_MAP_CACHE.get(cl)
-    except TypeError:
-        # Some codelist instances (e.g. attrs frozen) aren't weakly
-        # referenceable. Skip caching for those.
-        per_lang = None
-    if per_lang is not None and lang in per_lang:
-        return per_lang[lang]
     code_map = getattr(cl, "code_map", None)
-    mapping: dict[str, str]
     if callable(code_map):
-        # Fast path: direct SQL projection (LazyCodelist).
-        result = cast("dict[str, str]", code_map(lang=lang))
-        mapping = dict(result)
-    else:
-        # Fallback: in-memory codelist iteration.
-        codes = cast("Any", cl).codes
-        mapping = {c.id: c.name.get(lang) or c.id for c in codes}
-    try:
-        if per_lang is None:
-            _LABEL_MAP_CACHE[cl] = {lang: mapping}
-        else:
-            per_lang[lang] = mapping
-    except TypeError:
-        # Codelist isn't weakly referenceable — return uncached.
-        pass
-    return mapping
+        # Fast path: direct SQL projection (LazyCodelist), cached at source.
+        return cast("dict[str, str]", code_map(lang=lang))
+    # Fallback: in-memory codelist iteration (no SQL, cheap).
+    codes = cast("Any", cl).codes
+    return {c.id: c.name.get(lang) or c.id for c in codes}
 
 
 def _label_view(
@@ -298,9 +268,11 @@ def label(
     appends a ``{col}{suffix}`` column with the code label in ``lang``.
     Codes with no label fall back to the code id.
 
-    Label maps are cached per ``(codelist_urn, lang)`` after the first
-    call — call `clear_label_cache` after re-ingesting a codelist
-    in long-running processes.
+    Label maps are cached per (codelist instance, lang) at the source
+    (`LazyCodelist.code_map`) with a single-flight build lock, so the
+    first caller builds the map and concurrent callers share it rather
+    than each re-querying. A fresh ingest produces a new codelist
+    instance with a fresh cache, so re-ingestion is automatically correct.
 
     Example:
         labelled = slpl.label(pl.scan_csv("data.csv"), dsd, lang="en").collect()

{sdmxlib-0.41.2 → sdmxlib-0.41.3}/src/sdmxlib/storage/lazy.py

@@ -92,8 +92,10 @@ class LazyCodelist:
     """
 
     __slots__ = (
-        "__weakref__",  # required so slpl.label's per-codelist cache can hold a weakref
+        "__weakref__",
         "_agency",
+        "_code_map_cache",
+        "_code_map_lock",
         "_conn",
         "_description",
         "_id",
@@ -123,6 +125,13 @@ class LazyCodelist:
         self._name = name
         self._description = description
         self._is_final = is_final
+        # Per-instance, per-language label-map cache + single-flight build
+        # lock (#155). The lock collapses the thundering herd: N threads
+        # asking the same instance for the same lang run the SQL once, not
+        # N times. Cache lifetime is the instance's — a fresh ingest yields
+        # a new LazyCodelist, so re-ingestion is automatically correct.
+        self._code_map_cache: dict[str, dict[str, str]] = {}
+        self._code_map_lock = threading.Lock()
 
     # ── MaintainableArtefact-compatible properties ────────────────────────
 
@@ -393,10 +402,27 @@ class LazyCodelist:
     # ── Direct SQL helpers ───────────────────────────────────────────────
 
     def code_map(self, *, lang: str) -> dict[str, str]:
-        """Direct SQL: ``{code_id: label}`` without creating Code objects."""
-        from sdmxlib.storage.readers import code_map  # noqa: PLC0415
+        """Direct SQL: ``{code_id: label}`` without creating Code objects.
 
-        return code_map(self._conn, self._urn, lang=lang)
+        Memoised per instance per ``lang`` with a single-flight build lock
+        (#155): the first caller runs the SQL projection, concurrent
+        callers for the same ``lang`` block on the lock and then return the
+        hot cache rather than each re-running the query. The returned dict
+        is the cached instance — treat it as read-only.
+        """
+        cached = self._code_map_cache.get(lang)
+        if cached is not None:
+            return cached
+        with self._code_map_lock:
+            # Double-checked: a prior holder may have built it while we waited.
+            cached = self._code_map_cache.get(lang)
+            if cached is not None:
+                return cached
+            from sdmxlib.storage.readers import code_map  # noqa: PLC0415
+
+            mapping = code_map(self._conn, self._urn, lang=lang)
+            self._code_map_cache[lang] = mapping
+            return mapping
 
     def materialize(self) -> Codelist:
         """Load all codes and return a real `Codelist`."""