sdmxlib 0.35.0__tar.gz → 0.35.2__tar.gz

{sdmxlib-0.35.0 → sdmxlib-0.35.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.35.0
+Version: 0.35.2
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.35.0 → sdmxlib-0.35.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.35.0"
+version = "0.35.2"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.35.0 → sdmxlib-0.35.2}/src/sdmxlib/local/registry.py

@@ -30,7 +30,9 @@ stashes the URI; the DuckDB connection is opened by ``open()`` /
         local.add(...)
 """
 
+import logging
 import threading
+import time
 from collections.abc import Iterable, Iterator
 from contextlib import contextmanager
 from pathlib import Path
@@ -42,7 +44,7 @@ import duckdb
 from sdmxlib._duckdb import scalar
 from sdmxlib.model.collections import ItemList
 from sdmxlib.model.ref import Ref
-from sdmxlib.model.registry import InMemoryRegistry
+from sdmxlib.model.registry import InMemoryRegistry, artefact_urn
 from sdmxlib.model.urn import SdmxUrn
 from sdmxlib.storage import readers, writers
 from sdmxlib.storage._kinds import NAME_TO_CLASS as _TYPE_REGISTRY
@@ -68,6 +70,26 @@ _ITEM_TO_SCHEME: dict[str, tuple[str, str]] = {
 DEFAULT_RESOLVE_CACHE_SIZE = 256
 
 
+_logger = logging.getLogger(__name__)
+"""Per-artefact write timings are emitted at DEBUG level.
+
+Enable with::
+
+    logging.getLogger("sdmxlib.local.registry").setLevel(logging.DEBUG)
+
+Each ``add()`` call emits one line::
+
+    artefact_persist class=Codelist wall=0.0431 urn=urn:sdmx:...=ESTAT:CL_FREQ(1.0)
+
+A ``transaction()`` block emits one summary at exit::
+
+    transaction_summary count=523 wall=187.4 by_class=[Codelist:1 Hierarchy:12 MetadataSet:498 …]
+
+Default level is WARNING, so production callers see no output and pay
+no measurement cost (the timer is gated on ``isEnabledFor(DEBUG)``).
+"""
+
+
 class _FIFOCache(dict[str, Any]):
     """Bounded resolver cache. FIFO eviction, lock-free reads.
 
@@ -227,6 +249,9 @@ class Registry:
         self._owns_connection = True
         self._resolve_cache: _FIFOCache = _FIFOCache(resolve_cache_size)
         self._in_transaction: bool = False
+        # Per-transaction telemetry collected when DEBUG logging is on.
+        # Tuple is (count, total_wall_seconds) keyed by artefact class.
+        self._txn_stats: dict[str, tuple[int, float]] = {}
         # Per-thread read cursor (#80). Each thread that reads from this
         # Registry lazily gets its own cursor on first call; subsequent
         # calls on the same thread reuse it. Cleared on close().
@@ -296,6 +321,7 @@ class Registry:
         instance._owns_connection = False  # noqa: SLF001
         instance._resolve_cache = _FIFOCache(resolve_cache_size)  # noqa: SLF001
         instance._in_transaction = False  # noqa: SLF001
+        instance._txn_stats = {}  # noqa: SLF001
         instance._thread_local = threading.local()  # noqa: SLF001
         instance._initialise_schema()  # noqa: SLF001
         return instance
@@ -428,6 +454,10 @@ class Registry:
         already-open block does not emit a second BEGIN — DuckDB
         rejects nested transactions, and silently joining the outer
         block is what callers usually want.
+
+        When the ``sdmxlib.local.registry`` logger is set to DEBUG,
+        a one-line ``transaction_summary`` is emitted at exit with the
+        artefact-count and total-wall-time breakdown by class.
         """
         if self._in_transaction:
             yield
@@ -435,6 +465,8 @@ class Registry:
         conn = self._c
         conn.begin()
         self._in_transaction = True
+        self._txn_stats.clear()
+        t_start = time.perf_counter() if _logger.isEnabledFor(logging.DEBUG) else 0.0
         try:
             yield
             conn.commit()
@@ -444,6 +476,12 @@ class Registry:
         finally:
             self._in_transaction = False
             self._resolve_cache.clear()
+            if _logger.isEnabledFor(logging.DEBUG) and self._txn_stats:
+                total = time.perf_counter() - t_start
+                count = sum(n for n, _ in self._txn_stats.values())
+                by_class = " ".join(f"{cls}:{n}({wall:.2f}s)" for cls, (n, wall) in sorted(self._txn_stats.items()))
+                _logger.debug("transaction_summary count=%d wall=%.2f by_class=[%s]", count, total, by_class)
+                self._txn_stats.clear()
 
     def add(self, artefact: Any, *, source_registry: str | None = None) -> None:
         """Add or replace an artefact in the store.
@@ -462,7 +500,16 @@ class Registry:
         ``docs/concurrency.md``.
         """
         tag = source_registry if source_registry is not None else self._source_registry
+        timed = _logger.isEnabledFor(logging.DEBUG)
+        t0 = time.perf_counter() if timed else 0.0
         writers.put_artefact(self._c, artefact, source_registry=tag)
+        if timed:
+            wall = time.perf_counter() - t0
+            cls = type(artefact).__name__
+            _logger.debug("artefact_persist class=%s wall=%.4f urn=%s", cls, wall, artefact_urn(artefact))
+            if self._in_transaction:
+                n, total = self._txn_stats.get(cls, (0, 0.0))
+                self._txn_stats[cls] = (n + 1, total + wall)
         # Inside an open ``transaction()`` block the cache is cleared
         # once at exit; per-call clearing would just churn it.
         if not self._in_transaction:

{sdmxlib-0.35.0 → sdmxlib-0.35.2}/src/sdmxlib/storage/writers.py

@@ -294,16 +294,31 @@ def _put_refs(
     """
     if not refs:
         return
-    # executemany lets DuckDB amortize the PK index check across the
-    # batch — collapses ~2M individual INSERTs on StatCan-shape bundles
-    # (issue #139) into one statement bind/execute cycle.
-    conn.executemany(
-        """
-        INSERT INTO artefact_ref (source_urn, target_urn, role, ordinal)
-        VALUES (?, ?, ?, ?)
-        ON CONFLICT (source_urn, target_urn, role) DO NOTHING
-        """,
-        refs,
+    # Arrow columnar bulk insert. The 0.35.0 executemany rewrite assumed
+    # DuckDB would amortise the ON CONFLICT PK lookup across the batch;
+    # the v0.35.1 instrumentation showed it doesn't — per-row cost on a
+    # 295k-edge hierarchy was 11.5 ms (issue #146 / #140). The Arrow
+    # path inserts the whole batch in one SELECT and never pays the
+    # per-row index lookup.
+    #
+    # ON CONFLICT is dropped: within one ``_put_refs`` call, source_urn
+    # is constant (one artefact's outgoing edges) and
+    # ``_delete_artefact_cascade`` has already cleared this artefact's
+    # prior rows, so no PK collision is possible across calls. Intra-
+    # call duplicates (a DSD referencing the same Codelist via two
+    # dimensions) are dedup'd here in Python, keeping the first ordinal
+    # seen — same set-semantics behaviour the ON CONFLICT clause used to
+    # give. The ordinal on this table is only a positional hint; the
+    # canonical per-dimension position lives in ``dsd_component``.
+    seen: dict[tuple[str, str, str], int | None] = {}
+    for src, tgt, role, ordinal in refs:
+        seen.setdefault((src, tgt, role), ordinal)
+    rows: list[list[Any]] = [[src, tgt, role, ordinal] for (src, tgt, role), ordinal in seen.items()]
+    _execute_bulk_insert(
+        conn,
+        "artefact_ref",
+        ["source_urn", "target_urn", "role", "ordinal"],
+        rows,
     )