sdmxlib 0.44.1__tar.gz → 0.45.0__tar.gz

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.44.1
+Version: 0.45.0
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.44.1"
+version = "0.45.0"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/src/sdmxlib/__init__.py

@@ -10,7 +10,7 @@ from sdmxlib.catalog import DataflowEntry, FederatedCatalog
 from sdmxlib.data_store import DataStore
 from sdmxlib.sql import CompileContext, Predicate, SqlRelation
 from sdmxlib.formats import parse, read, serialize, to_json, to_xml
-from sdmxlib.local import LazyCodelist, LazyHierarchy, Registry
+from sdmxlib.local import LazyCodelist, LazyHierarchy, QueryPlan, Registry, SchemaInspector
 from sdmxlib.local.data_store import DuckDBDataStore, DuckLakeVersionError
 from sdmxlib.rest import ComponentFilter, parse_components
 from sdmxlib.model.convert import from_dict, to_dict
@@ -206,6 +206,7 @@ __all__ = [
     "Predicate",
     "Provider",
     "ProvisionAgreement",
+    "QueryPlan",
     "Ref",
     "ReferenceIssue",
     "ReferencePeriod",
@@ -220,6 +221,7 @@ __all__ = [
     "Representation",
     "RepresentationMap",
     "RestRegistry",
+    "SchemaInspector",
     "SdmxRef",
     "SdmxUrn",
     "SqlRelation",

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/src/sdmxlib/formats/sdmx_ml21/reader.py

@@ -83,8 +83,10 @@ _read_constraints = _helpers.read_constraints
 
 
 def _read_concept_role(component_el: etree._Element) -> tuple[str, ...]:
-    """Read ``<str:ConceptRole>`` children → tuple of role ids (from the Ref
-    ``id=`` attribute, or the item id of a URN-text payload).
+    """Read ``<str:ConceptRole>`` children → tuple of role ids.
+
+    Ids come from the Ref ``id=`` attribute, or the item id of a URN-text
+    payload.
     """
     roles: list[str] = []
     for role_el in component_el.findall(_s("ConceptRole")):
@@ -101,8 +103,10 @@ def _read_concept_role(component_el: etree._Element) -> tuple[str, ...]:
 
 
 def _read_contact(c_el: etree._Element) -> Contact:
-    """Read a v2.1 ``<str:Contact>``. Name is common-namespaced; Department /
-    Role are structure-namespaced with ``xml:lang``.
+    """Read a v2.1 ``<str:Contact>``.
+
+    Name is common-namespaced; Department / Role are structure-namespaced
+    with ``xml:lang``.
     """
 
     def _str_istring(tag: str) -> InternationalString:

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/src/sdmxlib/formats/sdmx_ml21/writer.py

@@ -84,11 +84,11 @@ _write_scheme_map = _helpers.write_scheme_map
 
 
 def _write_concept_roles(parent: etree._Element, roles: tuple[str, ...]) -> None:
-    """Emit one ``<str:ConceptRole>`` per role, referencing the SDMX cross-
-    domain concept scheme (the canonical home for the standard roles).
+    """Emit one ``<str:ConceptRole>`` per role.
 
-    Per the v2.1 XSD the component sequence places ConceptRole after
-    LocalRepresentation.
+    References the SDMX cross-domain concept scheme (the canonical home for
+    the standard roles). Per the v2.1 XSD the component sequence places
+    ConceptRole after LocalRepresentation.
     """
     for role in roles:
         cr = etree.SubElement(parent, _s("ConceptRole"))
@@ -102,8 +102,10 @@ def _write_concept_roles(parent: etree._Element, roles: tuple[str, ...]) -> None
 
 
 def _write_contact(parent: etree._Element, contact: Contact) -> None:
-    """Emit a v2.1 ``<str:Contact>``. Name is common-namespaced; Department /
-    Role / the communication channels are structure-namespaced per ContactType.
+    """Emit a v2.1 ``<str:Contact>``.
+
+    Name is common-namespaced; Department / Role / the communication
+    channels are structure-namespaced per ContactType.
     """
     el = etree.SubElement(parent, _s("Contact"))
     if contact.name:

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/src/sdmxlib/local/__init__.py

@@ -1,6 +1,7 @@
 """Local persistent SDMX artefact store backed by DuckDB."""
 
 from sdmxlib.local.registry import Registry
+from sdmxlib.local.sql import QueryPlan, SchemaInspector
 from sdmxlib.storage.lazy import LazyCodelist, LazyHierarchy
 
-__all__ = ["LazyCodelist", "LazyHierarchy", "Registry"]
+__all__ = ["LazyCodelist", "LazyHierarchy", "QueryPlan", "Registry", "SchemaInspector"]

{sdmxlib-0.44.1 → sdmxlib-0.45.0}/src/sdmxlib/local/data_store.py

@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Literal, Self
 import duckdb
 import polars as pl
 
+from sdmxlib.local.sql import SchemaInspector
 from sdmxlib.model.binding import DataflowBinding
 
 if TYPE_CHECKING:
@@ -123,6 +124,9 @@ class DuckDBDataStore:
         self._owns_connection = True
         # Per-thread read cursor (#80); see Registry._read_cursor.
         self._thread_local: threading.local = threading.local()
+        # Lazily-bound schema inspector, shared by the internal query path
+        # and consumers' custom SQL (#163). Reset on close().
+        self._inspector: SchemaInspector | None = None
 
     @property
     def uri(self) -> Path:
@@ -197,6 +201,7 @@ class DuckDBDataStore:
         instance._conn = registry.connection  # noqa: SLF001
         instance._owns_connection = False  # noqa: SLF001
         instance._thread_local = threading.local()  # noqa: SLF001
+        instance._inspector = None  # noqa: SLF001
         instance._configure_session()  # noqa: SLF001
         return instance
 
@@ -306,6 +311,25 @@ class DuckDBDataStore:
             self._conn.close()
         self._conn = None
         self._thread_local = threading.local()
+        self._inspector = None
+
+    def inspect(self) -> SchemaInspector:
+        """Return this store's `SchemaInspector` — the custom-SQL seam (#163).
+
+        The inspector reflects the live DuckDB session (cached ``DESCRIBE``)
+        and turns a `DataflowBinding` into a `QueryPlan`, so consumers who
+        build their own SQL against `connection` reuse the store's table-name
+        convention and sidecar join-key derivation. The returned inspector is
+        the same instance the internal ``query`` path uses, so its column
+        cache is shared.
+
+        Raises:
+            RuntimeError: If the data store hasn't been opened.
+        """
+        conn = self.connection  # raises if not open
+        if self._inspector is None:
+            self._inspector = SchemaInspector(conn)
+        return self._inspector
 
     def _read_cursor(self) -> duckdb.DuckDBPyConnection:
         """Return this thread's read cursor for ``query`` / ``query_stream``.
@@ -350,9 +374,8 @@ class DuckDBDataStore:
         attributes live on a sibling table rather than denormalised into
         the fact table.
         """
-        conn = self.connection
         sql, sql_params = _build_query(
-            conn=conn,
+            inspector=self.inspect(),
             binding=binding,
             where=where,
             params=params,
@@ -430,6 +453,13 @@ class DuckDBDataStore:
                 raise ValueError(msg)
         finally:
             conn.execute(f"DROP VIEW IF EXISTS {view_name}")
+        # A rewrite can change the target table's schema; drop cached column
+        # lists so the inspector (internal query path + custom SQL) re-reflects.
+        # The target may be cached under a differently-qualified name than the
+        # one written here (2- vs 3-part, alias skew), so clear the whole
+        # (tiny) cache rather than guess the exact key. Writes are rare.
+        if self._inspector is not None:
+            self._inspector.invalidate_all()
         _ = partition_by  # silence unused — see docstring
 
     def query_stream(
@@ -451,7 +481,7 @@ class DuckDBDataStore:
         """
         conn = self.connection
         sql, sql_params = _build_query(
-            conn=conn,
+            inspector=self.inspect(),
             binding=binding,
             where=where,
             params=params,
@@ -474,7 +504,7 @@ class DuckDBDataStore:
 
 def _build_query(
     *,
-    conn: duckdb.DuckDBPyConnection,
+    inspector: SchemaInspector,
     binding: DataflowBinding,
     where: list[str] | None,
     params: list[object] | None,
@@ -491,27 +521,15 @@ def _build_query(
     for parameterisation — never f-stringed.
 
     When ``binding.series_attributes_location`` is set, the sidecar table
-    is LEFT JOINed on the columns common to both tables. When
+    is LEFT JOINed on the columns common to both tables (derived by
+    ``inspector.plan``, which shares the store's DESCRIBE cache). When
     ``component_filter`` is provided, it is translated to additional
     ``col IN (?, ?, ...)`` fragments via the binding's dimension and
     attribute column maps and ANDed in.
     """
     cols_sql = ", ".join(columns) if columns else "*"
-    main = f"{binding.database_role}.{binding.table_location}"
-    sidecar_loc = binding.series_attributes_location
-    sidecar = f"{binding.database_role}.{sidecar_loc}" if sidecar_loc is not None else None
-    if sidecar is None:
-        sql = f"SELECT {cols_sql} FROM {main}"
-    else:
-        join_keys = _shared_columns(conn, main, sidecar)
-        if not join_keys:
-            msg = (
-                f"DataflowBinding.series_attributes_location {sidecar!r} "
-                f"shares no columns with {main!r}; cannot derive JOIN keys"
-            )
-            raise ValueError(msg)
-        keys_sql = ", ".join(join_keys)
-        sql = f"SELECT {cols_sql} FROM {main} LEFT JOIN {sidecar} USING ({keys_sql})"
+    plan = inspector.plan(binding, columns=columns)
+    sql = f"SELECT {cols_sql} FROM {plan.from_clause}"
 
     fragments = list(where or [])
     sql_params: list[object] = list(params or [])
@@ -532,18 +550,6 @@ def _build_query(
     return sql, sql_params
 
 
-def _describe_columns(conn: duckdb.DuckDBPyConnection, table: str) -> list[str]:
-    """Return column names for ``table`` via DuckDB ``DESCRIBE``."""
-    rows = conn.execute(f"DESCRIBE {table}").fetchall()
-    return [str(r[0]) for r in rows]
-
-
-def _shared_columns(conn: duckdb.DuckDBPyConnection, table_a: str, table_b: str) -> list[str]:
-    """Columns present in both tables, in ``table_a`` order."""
-    cols_b = set(_describe_columns(conn, table_b))
-    return [c for c in _describe_columns(conn, table_a) if c in cols_b]
-
-
 def _sql_identifier(name: str) -> str:
     """Escape a SQL identifier (extension name, ATTACH alias).
 

sdmxlib-0.45.0/src/sdmxlib/local/sql.py

@@ -0,0 +1,166 @@
+"""Custom-SQL seam for `DuckDBDataStore` — schema reflection + query planning.
+
+Most callers should go through `DuckDBDataStore.query` / `query_stream`. This
+module is the supported escape hatch for consumers that build their **own** SQL
+against the store's DuckDB connection (per-value ``printf`` formatting,
+Hive-partition-pruning filters, wide pivots, Range pagination — things the
+high-level query methods don't express).
+
+Two pieces:
+
+- `SchemaInspector` — reflect a live DuckDB session's schema
+  (``DESCRIBE``-backed, cached) and turn a `DataflowBinding` into a `QueryPlan`
+  so consumers reuse the store's table-name convention and sidecar join-key
+  derivation instead of re-implementing it.
+- `QueryPlan` — the resolved ``FROM`` side of a binding's query (main table,
+  optional sidecar ``LEFT JOIN``, join keys). Splice your own projection and
+  ``WHERE`` onto ``plan.from_clause``.
+
+Obtain an inspector via ``store.inspect()`` (it shares the store's cache with
+the internal query path) rather than constructing one directly.
+
+Example:
+    insp = store.inspect()
+    plan = insp.plan(binding)
+    sql = (
+        f"SELECT REF_AREA, printf('%.2f', OBS_VALUE) AS value "
+        f"FROM {plan.from_clause} WHERE GEO_LEVEL = ?"
+    )
+    rows = store.connection.execute(sql, ["CMA"]).fetchall()
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from attrs import frozen
+
+if TYPE_CHECKING:
+    import duckdb
+
+    from sdmxlib.model.binding import DataflowBinding
+
+
+@frozen(kw_only=True)
+class QueryPlan:
+    """The resolved ``FROM`` side of a `DataflowBinding`'s query.
+
+    Produced by `SchemaInspector.plan`. Consumers build their custom SQL by
+    splicing a projection and ``WHERE`` onto `from_clause`:
+
+        f"SELECT {cols} FROM {plan.from_clause} WHERE {predicate}"
+
+    Attributes:
+        main: Fully-qualified main table (``<database_role>.<table_location>``).
+        sidecar: Fully-qualified series-attributes table, or ``None`` when the
+            binding has no ``series_attributes_location``.
+        join_keys: Columns common to main and sidecar (in main order) that the
+            ``LEFT JOIN ... USING (...)`` is keyed on. Empty when no sidecar.
+        columns: The projected column names the plan was built for, or ``()``
+            for ``SELECT *``. Informational — `from_clause` never includes the
+            projection.
+    """
+
+    main: str
+    sidecar: str | None
+    join_keys: tuple[str, ...]
+    columns: tuple[str, ...] = ()
+
+    @property
+    def from_clause(self) -> str:
+        """The ``FROM`` fragment: ``main`` alone, or ``main LEFT JOIN sidecar USING (...)``."""
+        if self.sidecar is None:
+            return self.main
+        keys_sql = ", ".join(self.join_keys)
+        return f"{self.main} LEFT JOIN {self.sidecar} USING ({keys_sql})"
+
+
+class SchemaInspector:
+    """Cached schema reflection + binding→query planning for a DuckDB session.
+
+    Bound to a single DuckDB connection. ``DESCRIBE`` results are memoized per
+    table for the inspector's lifetime — the schema of a read table is stable,
+    and re-running ``DESCRIBE`` on every query is pure redundant work. When a
+    table is rewritten (e.g. `DuckDBDataStore.write_observations`), call
+    `invalidate` so a stale column list isn't served.
+
+    Obtain one via ``store.inspect()`` so the internal query path and your
+    custom SQL share the same cache; constructing one directly is supported but
+    gets a private cache.
+    """
+
+    def __init__(self, conn: duckdb.DuckDBPyConnection) -> None:
+        self._conn = conn
+        self._columns_cache: dict[str, list[str]] = {}
+
+    def columns(self, table: str) -> list[str]:
+        """Column names for ``table`` via DuckDB ``DESCRIBE`` (cached).
+
+        Args:
+            table: A table reference DuckDB can ``DESCRIBE`` — bare name or
+                ``schema.table`` / ``database.schema.table``.
+        """
+        cached = self._columns_cache.get(table)
+        if cached is not None:
+            return cached
+        rows = self._conn.execute(f"DESCRIBE {table}").fetchall()
+        cols = [str(r[0]) for r in rows]
+        self._columns_cache[table] = cols
+        return cols
+
+    def shared_columns(self, table_a: str, table_b: str) -> list[str]:
+        """Columns present in both tables, in ``table_a`` order.
+
+        These are the join keys for a main⋈sidecar ``LEFT JOIN``.
+        """
+        cols_b = set(self.columns(table_b))
+        return [c for c in self.columns(table_a) if c in cols_b]
+
+    def invalidate(self, table: str) -> None:
+        """Drop the cached column list for ``table`` (e.g. after a rewrite)."""
+        self._columns_cache.pop(table, None)
+
+    def invalidate_all(self) -> None:
+        """Drop all cached column lists."""
+        self._columns_cache.clear()
+
+    def plan(self, binding: DataflowBinding, *, columns: list[str] | None = None) -> QueryPlan:
+        """Resolve ``binding``'s main/sidecar tables into a `QueryPlan`.
+
+        Mirrors what `DuckDBDataStore.query` builds for the ``FROM`` side:
+        ``<database_role>.<table_location>`` for the main table, and — when
+        ``binding.series_attributes_location`` is set — a ``LEFT JOIN`` onto the
+        sidecar keyed on their shared columns.
+
+        Args:
+            binding: The dataflow binding to resolve.
+            columns: Optional projection recorded on the plan (informational;
+                does not affect `QueryPlan.from_clause`).
+
+        Raises:
+            ValueError: If the binding names a sidecar that shares no columns
+                with the main table (no derivable join keys).
+        """
+        main = f"{binding.database_role}.{binding.table_location}"
+        sidecar_loc = binding.series_attributes_location
+        if sidecar_loc is None:
+            return QueryPlan(
+                main=main,
+                sidecar=None,
+                join_keys=(),
+                columns=tuple(columns or ()),
+            )
+        sidecar = f"{binding.database_role}.{sidecar_loc}"
+        join_keys = self.shared_columns(main, sidecar)
+        if not join_keys:
+            msg = (
+                f"DataflowBinding.series_attributes_location {sidecar!r} "
+                f"shares no columns with {main!r}; cannot derive JOIN keys"
+            )
+            raise ValueError(msg)
+        return QueryPlan(
+            main=main,
+            sidecar=sidecar,
+            join_keys=tuple(join_keys),
+            columns=tuple(columns or ()),
+        )