sdmxlib 0.26.0__tar.gz → 0.27.0__tar.gz

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.26.0
+Version: 0.27.0
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.26.0"
+version = "0.27.0"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/catalog.py

@@ -21,9 +21,9 @@ Two construction paths:
    Standard layout::
 
        <bundle>/
-         registry.duckdb         # sdmx.* schema
+         meta.duckdb             # sdmx.* + pinax.* schemas
          lake.ducklake           # DuckLake catalog
-         parquet/                # DuckLake-managed data
+         parquet/                # DuckLake-managed data (DATA_PATH)
 
 2. ``FederatedCatalog.from_parts(structural=, bindings=, data=)`` —
    composition. For federated topologies (remote structural registry +
@@ -136,21 +136,22 @@ class FederatedCatalog:
     def open(self) -> Self:
         """Open the bundle's Registry + DuckDBDataStore against the standard layout.
 
-        Looks for ``<uri>/registry.duckdb`` (sdmx artefacts) and
+        Looks for ``<uri>/meta.duckdb`` (sdmx + pinax artefacts) and
         ``<uri>/lake.ducklake`` + ``<uri>/parquet/`` (DuckLake-managed
         observations). The Registry serves both ``structural`` and
         ``bindings`` roles. Idempotent within a single open/close cycle.
         """
         if self._structural is not None:
             return self
-        registry_path = self._uri / "registry.duckdb"
+        registry_path = self._uri / "meta.duckdb"
         lake_path = self._uri / "lake.ducklake"
         parquet_path = self._uri / "parquet"
         registry = Registry(registry_path).open()
-        attach_spec = f"ducklake:'{lake_path}' (DATA_PATH '{parquet_path}/')"
+        attach_url = f"'ducklake:{lake_path}'"
+        attach_options = f"(DATA_PATH '{parquet_path}/')"
         data = DuckDBDataStore.from_registry(
             registry,
-            attach={"lake": attach_spec},
+            attach={"lake": (attach_url, attach_options)},
             extensions=["ducklake"],
         )
         self._owned_registry = registry

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/data_store.py

@@ -19,20 +19,24 @@ structural side.
 from __future__ import annotations
 
 from collections.abc import Iterator
-from typing import TYPE_CHECKING, Protocol, runtime_checkable
+from typing import TYPE_CHECKING, Literal, Protocol, runtime_checkable
 
 import polars as pl
 
 from sdmxlib.model.binding import DataflowBinding
 
+if TYPE_CHECKING:
+    import duckdb
+    import pyarrow as pa
+
+    WriteSource = pl.DataFrame | pl.LazyFrame | pa.Table | duckdb.DuckDBPyRelation
+
 # SqlRelation lives in sdmxlib.sql now (suite-level public contract);
 # re-exported here for backwards compatibility with code that imports
 # ``from sdmxlib.data_store import SqlRelation``.
 from sdmxlib.sql import SqlRelation as SqlRelation  # noqa: PLC0414 — public re-export
 
 if TYPE_CHECKING:
-    import pyarrow as pa
-
     from sdmxlib.rest import ComponentFilter
 
 
@@ -115,3 +119,36 @@ class DataStore(Protocol):
             One :class:`pyarrow.RecordBatch` at a time, in scan order.
         """
         ...
+
+    def write_observations(
+        self,
+        *,
+        schema: str,
+        table: str,
+        source: WriteSource,
+        partition_by: tuple[str, ...] = (),
+        mode: Literal["overwrite", "append"] = "overwrite",
+        lake_alias: str = "lake",
+    ) -> None:
+        """Materialise observation rows into a backend-managed table.
+
+        Implementations route the write to whatever physical storage they
+        manage (DuckLake, Iceberg, Postgres, raw Parquet). The caller
+        side is uniform: schema + table + a source that can be turned
+        into rows.
+
+        Args:
+            schema: Target schema.
+            table: Target table name.
+            source: Rows to write. Accepted shapes vary by implementation
+                but Polars frames and Arrow tables are the portable
+                lowest common denominator.
+            partition_by: SDMX dimension ids the table is partitioned on.
+                Recorded on the matching :class:`DataflowBinding`.
+            mode: ``"overwrite"`` replaces atomically; ``"append"``
+                requires the table to already exist.
+            lake_alias: For ATTACH-based backends, the alias of the
+                target catalog. Implementations that don't use ATTACH
+                may ignore this argument.
+        """
+        ...

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/local/_data_store.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 
 from collections.abc import Iterator
 from pathlib import Path
-from typing import TYPE_CHECKING, Self
+from typing import TYPE_CHECKING, Literal, Self
 
 import duckdb
 import polars as pl
@@ -30,6 +30,11 @@ if TYPE_CHECKING:
     from sdmxlib.local._registry import Registry
     from sdmxlib.rest import ComponentFilter
 
+    # Sources that ``write_observations`` accepts. Type-checking only:
+    # the runtime path branches on isinstance(), so a runtime alias isn't
+    # needed (and pyarrow is an optional dependency).
+    WriteSource = pl.DataFrame | pl.LazyFrame | pa.Table | duckdb.DuckDBPyRelation
+
 
 class DuckDBDataStore:
     """A :class:`~sdmxlib.data_store.DataStore` backed by DuckDB.
@@ -45,10 +50,27 @@ class DuckDBDataStore:
        Lazy codelists keep working across queries, and queries can join
        across the two without cross-connection plumbing.
 
-    The ``attach`` / ``extensions`` knobs configure the DuckDB session
-    at open time. ``attach`` maps ``alias → ATTACH spec`` (each executed
-    as ``ATTACH <spec> AS <alias>``); ``extensions`` INSTALL+LOAD the
-    listed extensions before any ATTACH.
+    The ``attach`` / ``extensions`` / ``azure_credentials`` knobs
+    configure the DuckDB session at open time. ``attach`` maps
+    ``alias → ATTACH spec`` (each executed as
+    ``ATTACH <spec> AS <alias>``); ``extensions`` INSTALL+LOAD the
+    listed extensions before any ATTACH. ``azure_credentials`` is a
+    dict that drives a DuckDB ``CREATE SECRET`` for ADLSv2 access — see
+    below.
+
+    Azure credential dicts accept one of three shapes:
+
+    * ``{"connection_string": "..."}`` → ``PROVIDER 'config'`` with
+      ``CONNECTION_STRING``.
+    * ``{"account_name": ..., "tenant_id": ..., "client_id": ...,
+      "client_secret": ...}`` → ``PROVIDER 'service_principal'``.
+    * ``{"chain": "cli"}`` (or ``"env"``, ``"managed_identity"``) →
+      ``PROVIDER 'credential_chain'`` with the named chain.
+
+    When ``azure_credentials`` is set, or any ATTACH spec contains
+    ``abfss://``, the ``azure`` extension is auto-INSTALL+LOADed so
+    DATA_PATH URIs like ``abfss://acct.dfs.core.windows.net/c/p/`` work
+    out of the box.
 
     Example::
 
@@ -65,12 +87,14 @@ class DuckDBDataStore:
         self,
         uri: str | Path,
         *,
-        attach: dict[str, str] | None = None,
+        attach: dict[str, str | tuple[str, str]] | None = None,
         extensions: list[str] | None = None,
+        azure_credentials: dict[str, str] | None = None,
     ) -> None:
         self._uri: Path = Path(uri) if isinstance(uri, str) else uri
         self._attach = attach
         self._extensions = extensions
+        self._azure_credentials = azure_credentials
         self._registry: Registry | None = None
         self._conn: duckdb.DuckDBPyConnection | None = None
         self._owns_connection = True
@@ -122,8 +146,9 @@ class DuckDBDataStore:
         cls,
         registry: Registry,
         *,
-        attach: dict[str, str] | None = None,
+        attach: dict[str, str | tuple[str, str]] | None = None,
         extensions: list[str] | None = None,
+        azure_credentials: dict[str, str] | None = None,
     ) -> Self:
         """Wrap a :class:`Registry`'s open connection.
 
@@ -135,11 +160,14 @@ class DuckDBDataStore:
             registry: An open :class:`Registry`. Its connection is reused.
             attach: ``alias → ATTACH spec`` mapping. Executed at open time.
             extensions: DuckDB extensions to INSTALL+LOAD before any ATTACH.
+            azure_credentials: Optional Azure secret config. See
+                :meth:`__init__` for accepted keys.
         """
         instance = cls.__new__(cls)
         instance._uri = registry.uri  # noqa: SLF001 — alternate constructor sets private state
         instance._attach = attach  # noqa: SLF001
         instance._extensions = extensions  # noqa: SLF001
+        instance._azure_credentials = azure_credentials  # noqa: SLF001
         instance._registry = registry  # noqa: SLF001
         instance._conn = registry.connection  # noqa: SLF001
         instance._owns_connection = False  # noqa: SLF001
@@ -150,18 +178,42 @@ class DuckDBDataStore:
         """Apply the configured extensions + ATTACH specs to the live connection."""
         if self._conn is None:
             return
-        if self._extensions:
-            for ext in self._extensions:
+        extensions = list(self._extensions or [])
+        # Auto-load the azure extension if we need to talk to abfss:// or
+        # have credentials configured. Callers can override by listing
+        # "azure" in extensions themselves — INSTALL/LOAD are idempotent.
+        if self._needs_azure() and "azure" not in extensions:
+            extensions.append("azure")
+        if extensions:
+            for ext in extensions:
                 self._conn.execute(f"INSTALL {_sql_identifier(ext)}")
                 self._conn.execute(f"LOAD {_sql_identifier(ext)}")
             # Loading extensions can change the search_path; reset to
             # ``sdmx`` so any colocated Registry's internal queries stay clean.
             self._conn.execute("SET search_path = 'sdmx'")
+        if self._azure_credentials:
+            sql = _azure_create_secret_sql(self._azure_credentials)
+            self._conn.execute(sql)
         if self._attach:
             for alias, spec in self._attach.items():
-                self._conn.execute(f"ATTACH {spec} AS {_sql_identifier(alias)}")
+                url, options = _split_attach_spec(spec)
+                sql = f"ATTACH {url} AS {_sql_identifier(alias)}"
+                if options:
+                    sql += f" {options}"
+                self._conn.execute(sql)
             self._conn.execute("SET search_path = 'sdmx'")
 
+    def _needs_azure(self) -> bool:
+        """True if Azure secret/extension setup is required for this session."""
+        if self._azure_credentials:
+            return True
+        if self._attach:
+            for spec in self._attach.values():
+                spec_text = spec if isinstance(spec, str) else " ".join(s for s in spec if s)
+                if "abfss://" in spec_text:
+                    return True
+        return False
+
     def __enter__(self) -> Self:
         if self._conn is None:
             self.open()
@@ -222,6 +274,69 @@ class DuckDBDataStore:
             return pl.DataFrame(schema=col_names)
         return pl.from_records(rows, schema=col_names, orient="row")
 
+    def write_observations(
+        self,
+        *,
+        schema: str,
+        table: str,
+        source: WriteSource,
+        partition_by: tuple[str, ...] = (),
+        mode: Literal["overwrite", "append"] = "overwrite",
+        lake_alias: str = "lake",
+    ) -> None:
+        """Materialise observation rows into a DuckLake-managed table.
+
+        Routes through the ATTACHed lake (``lake_alias``, default ``lake``).
+        Schema is created if missing. ``mode="overwrite"`` uses
+        ``CREATE OR REPLACE TABLE``; ``mode="append"`` uses ``INSERT INTO``
+        and requires the target table to already exist with a compatible
+        schema. ``partition_by`` is currently informational — DuckLake
+        infers partition columns from table definition; we record the
+        layout on the matching :class:`DataflowBinding`.
+
+        Args:
+            schema: Target schema inside the lake (created if absent).
+            table: Target table name. Together with ``schema`` and
+                ``lake_alias`` this resolves to
+                ``<lake_alias>.<schema>.<table>``.
+            source: Rows to write. Anything DuckDB can register as a
+                virtual relation — Polars frames, Arrow tables, or an
+                existing DuckDB relation.
+            partition_by: SDMX dimension ids the table is partitioned on.
+            mode: ``"overwrite"`` replaces the table atomically;
+                ``"append"`` inserts into an existing table.
+            lake_alias: ATTACH alias of the target lake. Defaults to
+                ``"lake"`` to match the conventional bundle layout.
+        """
+        if not _is_valid_identifier(schema):
+            msg = f"Invalid schema identifier: {schema!r}"
+            raise ValueError(msg)
+        if not _is_valid_identifier(table):
+            msg = f"Invalid table identifier: {table!r}"
+            raise ValueError(msg)
+        if not _is_valid_identifier(lake_alias):
+            msg = f"Invalid lake alias: {lake_alias!r}"
+            raise ValueError(msg)
+
+        conn = self.connection
+        # Register the source as a temporary view the CTAS can scan.
+        view_name = f"__sdmxlib_write_obs_{id(source):x}"
+        rel = _as_relation(conn, source)
+        rel.create_view(view_name, replace=True)
+        try:
+            target = f"{lake_alias}.{schema}.{table}"
+            conn.execute(f"CREATE SCHEMA IF NOT EXISTS {lake_alias}.{schema}")
+            if mode == "overwrite":
+                conn.execute(f"CREATE OR REPLACE TABLE {target} AS SELECT * FROM {view_name}")
+            elif mode == "append":
+                conn.execute(f"INSERT INTO {target} SELECT * FROM {view_name}")
+            else:
+                msg = f"Unknown mode: {mode!r} (expected 'overwrite' or 'append')"
+                raise ValueError(msg)
+        finally:
+            conn.execute(f"DROP VIEW IF EXISTS {view_name}")
+        _ = partition_by  # silence unused — see docstring
+
     def query_stream(
         self,
         binding: DataflowBinding,
@@ -282,8 +397,9 @@ def _build_query(
     attribute column maps and ANDed in.
     """
     cols_sql = ", ".join(columns) if columns else "*"
-    main = binding.table_location
-    sidecar = binding.series_attributes_location
+    main = f"{binding.database_role}.{binding.table_location}"
+    sidecar_loc = binding.series_attributes_location
+    sidecar = f"{binding.database_role}.{sidecar_loc}" if sidecar_loc is not None else None
     if sidecar is None:
         sql = f"SELECT {cols_sql} FROM {main}"
     else:
@@ -335,7 +451,102 @@ def _sql_identifier(name: str) -> str:
     and underscores. Reject anything else to avoid SQL injection through
     extension/alias names.
     """
-    if not name or not all(c.isalnum() or c == "_" for c in name):
+    if not _is_valid_identifier(name):
         msg = f"Invalid SQL identifier: {name!r}"
         raise ValueError(msg)
     return name
+
+
+def _is_valid_identifier(name: str) -> bool:
+    """True if ``name`` is a safe SQL identifier (alphanumeric + underscore)."""
+    return bool(name) and all(c.isalnum() or c == "_" for c in name)
+
+
+def _split_attach_spec(spec: object) -> tuple[str, str]:
+    """Normalise an ATTACH spec into ``(url, options)``.
+
+    Two shapes accepted:
+
+    - ``"'ducklake:/path'"`` — connection URL only; no options.
+    - ``("'ducklake:/path'", "(DATA_PATH '/p/')")`` — URL + trailing
+      options clause that goes after ``AS alias``.
+
+    Returns ``(url, options)`` strings; ``options`` is empty when none.
+    """
+    if isinstance(spec, str):
+        return spec, ""
+    if isinstance(spec, tuple) and len(spec) == 2:
+        url, options = spec
+        return str(url), str(options or "")
+    msg = f"Unsupported ATTACH spec shape: {type(spec).__name__}"
+    raise TypeError(msg)
+
+
+def _azure_create_secret_sql(creds: dict[str, str]) -> str:
+    """Render a DuckDB ``CREATE SECRET`` statement for an Azure credential dict.
+
+    Three accepted shapes:
+
+    * ``{"connection_string": "..."}`` → ``PROVIDER 'config'`` with
+      ``CONNECTION_STRING``.
+    * ``{"account_name": ..., "tenant_id": ..., "client_id": ...,
+      "client_secret": ...}`` → ``PROVIDER 'service_principal'``.
+    * ``{"chain": "cli"|"env"|"managed_identity"}`` →
+      ``PROVIDER 'credential_chain'``.
+    """
+    name = "sdmxlib_azure"
+    if "connection_string" in creds:
+        cs = _sql_string_literal(creds["connection_string"])
+        return f"CREATE OR REPLACE SECRET {name} (TYPE AZURE, PROVIDER 'config', CONNECTION_STRING {cs})"
+    if "chain" in creds:
+        chain = _sql_string_literal(creds["chain"])
+        clauses = ["TYPE AZURE", "PROVIDER 'credential_chain'", f"CHAIN {chain}"]
+        if "account_name" in creds:
+            clauses.append(f"ACCOUNT_NAME {_sql_string_literal(creds['account_name'])}")
+        return f"CREATE OR REPLACE SECRET {name} ({', '.join(clauses)})"
+    required = {"account_name", "tenant_id", "client_id", "client_secret"}
+    if required.issubset(creds):
+        return (
+            f"CREATE OR REPLACE SECRET {name} ("
+            f"TYPE AZURE, PROVIDER 'service_principal', "
+            f"TENANT_ID {_sql_string_literal(creds['tenant_id'])}, "
+            f"CLIENT_ID {_sql_string_literal(creds['client_id'])}, "
+            f"CLIENT_SECRET {_sql_string_literal(creds['client_secret'])}, "
+            f"ACCOUNT_NAME {_sql_string_literal(creds['account_name'])}"
+            f")"
+        )
+    msg = (
+        "azure_credentials must be one of: "
+        "{'connection_string': ...}, "
+        "{'account_name': ..., 'tenant_id': ..., 'client_id': ..., 'client_secret': ...}, "
+        "or {'chain': 'cli'|'env'|'managed_identity'}. "
+        f"Got keys: {sorted(creds.keys())}"
+    )
+    raise ValueError(msg)
+
+
+def _sql_string_literal(value: str) -> str:
+    """Quote a value as a SQL string literal — single-quotes, doubled internals."""
+    escaped = value.replace("'", "''")
+    return f"'{escaped}'"
+
+
+def _as_relation(
+    conn: duckdb.DuckDBPyConnection,
+    source: WriteSource,
+) -> duckdb.DuckDBPyRelation:
+    """Materialise ``source`` as a DuckDB relation the connection can scan.
+
+    Handles Polars frames (eager + lazy), Arrow tables, and existing
+    DuckDB relations uniformly. ``LazyFrame`` is collected to a DataFrame
+    first — callers that need streaming behaviour for very large frames
+    can pass an Arrow ``RecordBatchReader`` via :func:`pyarrow.Table.from_batches`.
+    """
+    if isinstance(source, duckdb.DuckDBPyRelation):
+        return source
+    if isinstance(source, pl.LazyFrame):
+        source = source.collect()
+    if isinstance(source, pl.DataFrame):
+        return conn.from_arrow(source.to_arrow())
+    # Last branch: pyarrow.Table. Imported lazily to keep pyarrow optional.
+    return conn.from_arrow(source)

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/model/binding.py

@@ -10,10 +10,18 @@ The binding artefact is local-store-owned: a Dataflow may live in a
 remote registry (FMR, ESTAT, ECB), but the binding to a Parquet/DuckLake
 table lives in the local registry that knows about the data.
 
+The bundle architecture splits the physical address of a table into
+``database_role`` (ATTACH alias of the catalog database) and
+``table_location`` (``schema.table`` inside that database). The resolver
+re-joins them at query time as ``<database_role>.<table_location>``;
+this keeps bindings portable across bundles that may attach the lake
+under different aliases.
+
 Example:
     binding = DataflowBinding(
         dataflow=Ref[Dataflow].unresolved(dataflow_urn),
-        table_location="lake.census_2021.observations",
+        database_role="lake",
+        table_location="census_2021.observations",
         partition_layout=("GEO_LEVEL",),
         dimension_column_map={"REF_AREA": "REF_AREA", "TIME_PERIOD": "TIME_PERIOD"},
         measure_column_map={"OBS_COUNT": "OBS_COUNT"},
@@ -26,7 +34,7 @@ Example:
     local.add(binding)
 """
 
-from datetime import datetime
+from datetime import UTC, datetime
 from typing import TYPE_CHECKING, ClassVar, Self
 
 from attrs import define, field, frozen
@@ -36,6 +44,7 @@ from sdmxlib.model.ref import Ref
 from sdmxlib.model.urn import SdmxUrn
 
 if TYPE_CHECKING:
+    from sdmxlib.model.datastructure import DataStructure
     from sdmxlib.model.registry import RegistryReader
 
 
@@ -45,9 +54,15 @@ class DataflowBinding:
 
     Attributes:
         dataflow: Ref to the bound dataflow (may be remote).
-        table_location: Backend-specific table identifier — for DuckLake
-            this is ``catalog.schema.table``; for raw Parquet this might
-            be a path or URI.
+        database_role: ATTACH alias of the catalog database that owns the
+            table. Defaults to ``"lake"`` to match the conventional
+            bundle layout (``meta.duckdb`` + ``lake.ducklake`` attached
+            as ``lake``). Combined with :attr:`table_location` at query
+            time as ``<database_role>.<table_location>``.
+        table_location: ``schema.table`` inside the database identified
+            by :attr:`database_role`. Does NOT include the database
+            alias; the resolver prepends it. For raw Parquet stores
+            without an ATTACH this can be a path or URI.
         partition_layout: SDMX dimension ids the table is partitioned on,
             in partition order. Drives partition pruning at query time.
         dimension_column_map: SDMX dimension id → table column name.
@@ -73,6 +88,7 @@ class DataflowBinding:
 
     dataflow: Ref[Dataflow]
     table_location: str
+    database_role: str = "lake"
     partition_layout: tuple[str, ...] = field(factory=tuple, converter=tuple)
     dimension_column_map: dict[str, str] = field(factory=dict)
     measure_column_map: dict[str, str] = field(factory=dict)
@@ -124,6 +140,78 @@ class DataflowBinding:
             version=version,
         )
 
+    # ── Convenience constructor ──────────────────────────────────────────
+
+    @classmethod
+    def from_dsd(
+        cls,
+        dsd: "DataStructure",
+        *,
+        dataflow: Ref[Dataflow],
+        schema: str,
+        table: str,
+        partition_layout: tuple[str, ...] = (),
+        source_registry: str = "local",
+        materialized_at: datetime | None = None,
+        bound_at: datetime | None = None,
+        database_role: str = "lake",
+        series_attributes_location: str | None = None,
+    ) -> "DataflowBinding":
+        """Build a DataflowBinding from a DSD with identity column mapping.
+
+        Derives ``dimension_column_map`` / ``measure_column_map`` /
+        ``attribute_column_map`` from the DSD's components (SDMX id →
+        SDMX id), pins ``structural_version`` as
+        ``f"{dsd.agency_id}:{dsd.id}({dsd.version})"``, and addresses the
+        target table via ``database_role`` + ``schema.table`` (no
+        database alias prefix is written into ``table_location``).
+
+        ``materialized_at`` and ``bound_at`` default to ``datetime.now(UTC)``.
+
+        Args:
+            dsd: The :class:`DataStructure` whose components define the
+                column maps.
+            dataflow: Ref to the bound dataflow.
+            schema: Schema portion of ``table_location``.
+            table: Table portion of ``table_location``.
+            partition_layout: SDMX dimension ids the table is partitioned
+                on, in partition order.
+            source_registry: Provenance label for the structural side.
+            materialized_at: When the data was last refreshed. Defaults
+                to ``datetime.now(UTC)``.
+            bound_at: When this binding record was created. Defaults to
+                ``datetime.now(UTC)``.
+            database_role: ATTACH alias of the database that owns the
+                table. Defaults to ``"lake"``.
+            series_attributes_location: Optional sidecar table for
+                series-level attributes (same shape as
+                ``schema.table`` — no database alias prefix).
+
+        Returns:
+            A :class:`DataflowBinding` ready to be added to a registry.
+        """
+        now = datetime.now(UTC)
+        dim_map = {d.id: d.id for d in dsd.dimensions}
+        if dsd.time_dimension is not None:
+            dim_map[dsd.time_dimension.id] = dsd.time_dimension.id
+        measure_map = {m.id: m.id for m in dsd.measures}
+        attr_map = {a.id: a.id for a in dsd.attributes}
+        structural_version = f"{dsd.agency_id}:{dsd.id}({dsd.version})"
+        return cls(
+            dataflow=dataflow,
+            database_role=database_role,
+            table_location=f"{schema}.{table}",
+            partition_layout=partition_layout,
+            dimension_column_map=dim_map,
+            measure_column_map=measure_map,
+            attribute_column_map=attr_map,
+            structural_version=structural_version,
+            source_registry=source_registry,
+            materialized_at=materialized_at if materialized_at is not None else now,
+            bound_at=bound_at if bound_at is not None else now,
+            series_attributes_location=series_attributes_location,
+        )
+
 
 # ── Drift detection ─────────────────────────────────────────────────────
 

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/storage/readers.py

@@ -1176,7 +1176,7 @@ def _read_dataflow_binding(conn: duckdb.DuckDBPyConnection, urn: str) -> Dataflo
 
     row = conn.execute(
         """
-        SELECT dataflow_urn, table_location, partition_layout,
+        SELECT dataflow_urn, table_location, database_role, partition_layout,
                dimension_column_map, measure_column_map, attribute_column_map,
                structural_version, binding_source_registry,
                materialized_at, bound_at, series_attributes_location
@@ -1190,6 +1190,7 @@ def _read_dataflow_binding(conn: duckdb.DuckDBPyConnection, urn: str) -> Dataflo
     (
         dataflow_urn_str,
         table_location,
+        database_role,
         partition_layout,
         dim_map,
         meas_map,
@@ -1204,6 +1205,7 @@ def _read_dataflow_binding(conn: duckdb.DuckDBPyConnection, urn: str) -> Dataflo
     return DataflowBinding(
         dataflow=Ref[Dataflow].unresolved(SdmxUrn.parse(dataflow_urn_str)),
         table_location=table_location,
+        database_role=database_role,
         partition_layout=tuple(partition_layout or ()),
         dimension_column_map=dict(dim_map or {}),
         measure_column_map=dict(meas_map or {}),

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/storage/schema.py

@@ -361,6 +361,7 @@ CREATE TABLE IF NOT EXISTS dataflow_binding (
     urn                          TEXT PRIMARY KEY,
     dataflow_urn                 TEXT NOT NULL,
     table_location               TEXT NOT NULL,
+    database_role                TEXT NOT NULL DEFAULT 'lake',
     partition_layout             TEXT[],
     dimension_column_map         MAP(VARCHAR, VARCHAR),
     measure_column_map           MAP(VARCHAR, VARCHAR),

{sdmxlib-0.26.0 → sdmxlib-0.27.0}/src/sdmxlib/storage/writers.py

@@ -296,7 +296,18 @@ def _put_refs(
     conn: duckdb.DuckDBPyConnection,
     refs: list[tuple[str, str, str, int | None]],
 ) -> None:
-    """Insert artefact_ref rows."""
+    """Insert artefact_ref rows.
+
+    Treats ``(source_urn, target_urn, role)`` as set semantics: a DSD
+    that references the same codelist via multiple dimensions yields a
+    single dependency edge here. The per-dimension ordinal (position) is
+    recoverable from the matching component table
+    (``dsd_component.position`` / ``msd_component.position``), so we
+    don't lose information by collapsing duplicates. ``ON CONFLICT DO
+    NOTHING`` keeps the first ordinal we saw — arbitrary, but ordinal on
+    this table is only a hint for callers that want positional iteration
+    through the ref graph without a join.
+    """
     if not refs:
         return
     for source_urn, target_urn, role, ordinal in refs:
@@ -304,6 +315,7 @@ def _put_refs(
             """
             INSERT INTO artefact_ref (source_urn, target_urn, role, ordinal)
             VALUES (?, ?, ?, ?)
+            ON CONFLICT (source_urn, target_urn, role) DO NOTHING
             """,
             [source_urn, target_urn, role, ordinal],
         )
@@ -1390,17 +1402,18 @@ def put_dataflow_binding(
     conn.execute(
         """
         INSERT INTO dataflow_binding (
-            urn, dataflow_urn, table_location, partition_layout,
+            urn, dataflow_urn, table_location, database_role, partition_layout,
             dimension_column_map, measure_column_map, attribute_column_map,
             structural_version, binding_source_registry,
             materialized_at, bound_at, series_attributes_location
         )
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         """,
         [
             urn,
             str(binding.dataflow.urn),
             binding.table_location,
+            binding.database_role,
             list(binding.partition_layout),
             dict(binding.dimension_column_map),
             dict(binding.measure_column_map),