sdmxlib 0.35.1__tar.gz → 0.35.2__tar.gz

{sdmxlib-0.35.1 → sdmxlib-0.35.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.35.1
+Version: 0.35.2
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.35.1 → sdmxlib-0.35.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.35.1"
+version = "0.35.2"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.35.1 → sdmxlib-0.35.2}/src/sdmxlib/storage/writers.py

@@ -294,16 +294,31 @@ def _put_refs(
     """
     if not refs:
         return
-    # executemany lets DuckDB amortize the PK index check across the
-    # batch — collapses ~2M individual INSERTs on StatCan-shape bundles
-    # (issue #139) into one statement bind/execute cycle.
-    conn.executemany(
-        """
-        INSERT INTO artefact_ref (source_urn, target_urn, role, ordinal)
-        VALUES (?, ?, ?, ?)
-        ON CONFLICT (source_urn, target_urn, role) DO NOTHING
-        """,
-        refs,
+    # Arrow columnar bulk insert. The 0.35.0 executemany rewrite assumed
+    # DuckDB would amortise the ON CONFLICT PK lookup across the batch;
+    # the v0.35.1 instrumentation showed it doesn't — per-row cost on a
+    # 295k-edge hierarchy was 11.5 ms (issue #146 / #140). The Arrow
+    # path inserts the whole batch in one SELECT and never pays the
+    # per-row index lookup.
+    #
+    # ON CONFLICT is dropped: within one ``_put_refs`` call, source_urn
+    # is constant (one artefact's outgoing edges) and
+    # ``_delete_artefact_cascade`` has already cleared this artefact's
+    # prior rows, so no PK collision is possible across calls. Intra-
+    # call duplicates (a DSD referencing the same Codelist via two
+    # dimensions) are dedup'd here in Python, keeping the first ordinal
+    # seen — same set-semantics behaviour the ON CONFLICT clause used to
+    # give. The ordinal on this table is only a positional hint; the
+    # canonical per-dimension position lives in ``dsd_component``.
+    seen: dict[tuple[str, str, str], int | None] = {}
+    for src, tgt, role, ordinal in refs:
+        seen.setdefault((src, tgt, role), ordinal)
+    rows: list[list[Any]] = [[src, tgt, role, ordinal] for (src, tgt, role), ordinal in seen.items()]
+    _execute_bulk_insert(
+        conn,
+        "artefact_ref",
+        ["source_urn", "target_urn", "role", "ordinal"],
+        rows,
     )