sdmxlib 0.27.1__tar.gz → 0.27.2__tar.gz

{sdmxlib-0.27.1 → sdmxlib-0.27.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.27.1
+Version: 0.27.2
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.27.1 → sdmxlib-0.27.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.27.1"
+version = "0.27.2"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.27.1 → sdmxlib-0.27.2}/src/sdmxlib/local/_registry.py

@@ -30,6 +30,7 @@ stashes the URI; the DuckDB connection is opened by ``open()`` /
         local.add(...)
 """
 
+from collections import OrderedDict
 from collections.abc import Iterable
 from pathlib import Path
 from typing import Any, Self
@@ -57,6 +58,90 @@ _ITEM_TO_SCHEME: dict[str, tuple[str, str]] = {
 }
 
 
+DEFAULT_RESOLVE_CACHE_SIZE = 256
+
+
+class _LRUDict(OrderedDict[str, Any]):
+    """Bounded LRU mapping used as the registry-level resolver cache.
+
+    Subclasses :class:`OrderedDict` so it can be passed wherever the
+    resolver expects ``dict[str, Any]``. Inserts evict least-recently-used
+    entries past ``maxsize``; reads via :meth:`get` move keys to the end.
+    ``maxsize == 0`` disables caching — inserts are silently dropped, so
+    the structure stays empty.
+
+    Tracks ``hits`` / ``misses`` for :meth:`Registry.resolve_cache_info`;
+    ``__contains__`` is treated as internal control flow (the resolver
+    uses it for dedup) and is not counted.
+    """
+
+    def __init__(self, maxsize: int) -> None:
+        super().__init__()
+        if maxsize < 0:
+            msg = f"resolve_cache_size must be >= 0, got {maxsize}"
+            raise ValueError(msg)
+        self.maxsize = maxsize
+        self.hits = 0
+        self.misses = 0
+
+    def __setitem__(self, key: str, value: Any) -> None:
+        if self.maxsize == 0:
+            return
+        if key in self:
+            self.move_to_end(key)
+        super().__setitem__(key, value)
+        while len(self) > self.maxsize:
+            self.popitem(last=False)
+
+    def get(self, key: str, default: Any = None) -> Any:  # type: ignore[override]
+        if super().__contains__(key):
+            self.hits += 1
+            self.move_to_end(key)
+            return super().__getitem__(key)
+        self.misses += 1
+        return default
+
+    def reset_stats(self) -> None:
+        self.hits = 0
+        self.misses = 0
+
+
+class _CallScopedMemo(dict[str, Any]):
+    """Per-call resolver memo layered over a persistent LRU cache.
+
+    Two reasons for the overlay rather than passing the LRU directly:
+
+    1. **Correctness under tight cache sizes (incl. ``maxsize=0``).** The
+       resolver writes URN→target during one phase and reads them back in
+       a later phase of the same call. If those writes went straight to
+       the LRU, an entry could be evicted before the read — leaving Refs
+       unresolved for callers using a small (or disabled) cache. The
+       overlay holds in-flight entries in an unbounded per-call dict so
+       single-call resolution is independent of cache size.
+    2. **Promote-on-success.** Entries written during the call enter the
+       LRU only at the end of the call, via :meth:`promote`. That keeps
+       partial-failure semantics clean: if a call raises midway, the LRU
+       is unchanged.
+    """
+
+    def __init__(self, lru: _LRUDict) -> None:
+        super().__init__()
+        self._lru = lru
+
+    def __contains__(self, key: object) -> bool:
+        return super().__contains__(key) or self._lru.__contains__(key)
+
+    def get(self, key: str, default: Any = None) -> Any:  # type: ignore[override]
+        if super().__contains__(key):
+            return super().__getitem__(key)
+        return self._lru.get(key, default)
+
+    def promote(self) -> None:
+        """Copy this call's resolved entries into the persistent LRU."""
+        for k, v in self.items():
+            self._lru[k] = v
+
+
 class Registry:
     """DuckDB-backed persistent SDMX artefact store.
 
@@ -86,12 +171,22 @@ class Registry:
         uri: str | Path,
         *,
         source_registry: str | None = None,
+        resolve_cache_size: int = DEFAULT_RESOLVE_CACHE_SIZE,
     ) -> None:
+        """Open or wrap a DuckDB-backed registry.
+
+        ``resolve_cache_size`` bounds the LRU of resolved Ref targets that
+        :meth:`get` / :meth:`get_all` / :meth:`to_registry` share across
+        calls. Pass ``0`` to disable caching (matches pre-v0.27.2 behaviour
+        — one full read per Ref). The cache is cleared on every mutation
+        (``add`` / ``drop`` / ``replace``) for invalidation correctness.
+        """
         self._uri = Path(uri) if isinstance(uri, str) else uri
         self._source_registry = source_registry
         self._conn: duckdb.DuckDBPyConnection | None = None
         self._cursor: duckdb.DuckDBPyConnection | None = None
         self._owns_connection = True
+        self._resolve_cache: _LRUDict = _LRUDict(resolve_cache_size)
 
     @property
     def uri(self) -> Path:
@@ -137,6 +232,7 @@ class Registry:
         conn: duckdb.DuckDBPyConnection,
         *,
         source_registry: str | None = None,
+        resolve_cache_size: int = DEFAULT_RESOLVE_CACHE_SIZE,
     ) -> Self:
         """Wrap an existing DuckDB connection.
 
@@ -144,6 +240,9 @@ class Registry:
         Registry leaves the connection open. Use this when the connection
         is shared with another component (e.g. a co-located DuckLake catalog)
         that owns its lifecycle.
+
+        ``resolve_cache_size`` configures the resolver LRU; see
+        :meth:`__init__`.
         """
         instance = cls.__new__(cls)
         instance._uri = Path(":connection:")  # noqa: SLF001 — alternate constructor sets private state
@@ -151,6 +250,7 @@ class Registry:
         instance._conn = conn  # noqa: SLF001
         instance._cursor = None  # noqa: SLF001
         instance._owns_connection = False  # noqa: SLF001
+        instance._resolve_cache = _LRUDict(resolve_cache_size)  # noqa: SLF001
         instance._initialise_schema()  # noqa: SLF001
         return instance
 
@@ -209,9 +309,14 @@ class Registry:
 
         ``source_registry`` overrides the registry-level default for this
         call. ``None`` (default) means "use the registry-level default."
+
+        Clears the resolver cache: any cached resolved target may now be
+        stale (e.g. a freshly-added Codelist replaces the one a previous
+        ``get`` had cached).
         """
         tag = source_registry if source_registry is not None else self._source_registry
         writers.put_artefact(self._c, artefact, source_registry=tag)
+        self._resolve_cache.clear()
 
     def add_all(
         self,
@@ -256,22 +361,25 @@ class Registry:
         if result is None:
             return None
         if not isinstance(result, (LazyCodelist, LazyHierarchy)):
-            self._resolve_refs(result)
+            memo = _CallScopedMemo(self._resolve_cache)
+            self._resolve_refs(result, memo=memo)
+            memo.promote()
         return result  # type: ignore[return-value]
 
     def get_all[T](self, artefact_type: "type[T]") -> "list[T]":
         """All artefacts of the given Python type.
 
-        Shares a single resolver memo across the batch so any parent
-        scheme referenced by multiple artefacts is loaded once, not once
-        per artefact.
+        Resolution shares one call-scoped memo across the batch and is
+        backed by the registry's persistent ``_resolve_cache``, so any
+        parent scheme referenced by multiple artefacts is loaded once,
+        not once per artefact.
         """
         results = readers.get_all(self._c, artefact_type)
-        memo: dict[str, Any] = {}
-        walked: set[int] = set()
+        memo = _CallScopedMemo(self._resolve_cache)
         for r in results:
             if not isinstance(r, (LazyCodelist, LazyHierarchy)):
-                self._resolve_refs(r, memo=memo, walked=walked)
+                self._resolve_refs(r, memo=memo)
+        memo.promote()
         return results  # type: ignore[return-value]
 
     # ── Ref resolution ─────────────────────────────────────────────────
@@ -498,14 +606,51 @@ class Registry:
     # ── Mutations ─────────────────────────────────────────────────────────
 
     def drop(self, urn: "SdmxUrn[Any]") -> bool:
-        """Remove an artefact by URN. Returns True if it existed."""
-        return writers.drop_artefact(self._c, str(urn))
+        """Remove an artefact by URN. Returns True if it existed.
+
+        Clears the resolver cache (cached resolved targets may reference
+        the dropped artefact).
+        """
+        result = writers.drop_artefact(self._c, str(urn))
+        self._resolve_cache.clear()
+        return result
 
     def replace(self, old_urn: "SdmxUrn[Any]", new: Any) -> None:
-        """Remove the old artefact and add the new one."""
+        """Remove the old artefact and add the new one.
+
+        Each leg clears the resolver cache via :meth:`add` / :meth:`drop`;
+        the net effect is a single fully-cleared cache at the end.
+        """
         writers.drop_artefact(self._c, str(old_urn))
         self.add(new)
 
+    # ── Resolver cache ────────────────────────────────────────────────────
+
+    def clear_resolve_cache(self) -> None:
+        """Purge the resolver cache and reset hit/miss counters.
+
+        Useful for explicit invalidation hooks (e.g. a webhook telling the
+        service that upstream metadata changed without going through this
+        registry's mutation API). Cache also clears automatically on every
+        ``add`` / ``drop`` / ``replace``.
+        """
+        self._resolve_cache.clear()
+        self._resolve_cache.reset_stats()
+
+    def resolve_cache_info(self) -> dict[str, int]:
+        """Return resolver cache stats: size, max_size, hits, misses.
+
+        Intended for ops dashboards and integration tests asserting cache
+        behaviour. ``size == 0`` after construction or any mutation; rises
+        as ``get`` / ``get_all`` populate it, capped at ``max_size``.
+        """
+        return {
+            "size": len(self._resolve_cache),
+            "max_size": self._resolve_cache.maxsize,
+            "hits": self._resolve_cache.hits,
+            "misses": self._resolve_cache.misses,
+        }
+
     # ── Direct SQL helpers ────────────────────────────────────────────────
 
     def code_map(self, codelist_urn: "str | SdmxUrn[Any]", *, lang: str) -> dict[str, str]:
@@ -536,17 +681,18 @@ class Registry:
     def to_registry(self) -> InMemoryRegistry:
         """Load everything into an in-memory Registry with full Ref interning.
 
-        Shares a resolver memo across the bulk load so each parent scheme
-        is read at most once even when many artefacts reference it.
+        Resolution uses the registry's persistent ``_resolve_cache`` so each
+        parent scheme is read at most once even when many artefacts
+        reference it.
         """
         reg = InMemoryRegistry()
-        memo: dict[str, Any] = {}
-        walked: set[int] = set()
+        memo = _CallScopedMemo(self._resolve_cache)
         rows = self._c.execute("SELECT urn, artefact_type FROM artefact").fetchall()
         for urn_str, _artefact_type in rows:
             obj = readers.get_artefact(self._c, urn_str, eager=True)
             if obj is None:
                 continue
-            self._resolve_refs(obj, memo=memo, walked=walked)
+            self._resolve_refs(obj, memo=memo)
             reg.add(obj)
+        memo.promote()
         return reg