sdmxlib 0.63.0__tar.gz → 0.65.0__tar.gz

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.63.0
+Version: 0.65.0
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.63.0"
+version = "0.65.0"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -140,7 +140,7 @@ mutate_only_covered_lines = false
 
 [dependency-groups]
 dev = [
-    "basedpyright>=1.39.9,<1.40",
+    "basedpyright==1.39.9",
     "catppuccin[pygments]>=2.5.0",
     "types-lxml>=2025.3.30",
     "mkdocs-material>=9.7.6",

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.63.0"
+version = "0.65.0"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }
@@ -220,7 +220,16 @@ dev = [
   # ruff on `select = ["ALL"]` that is not hypothetical: ruff 0.16.0
   # stabilised 12 preview rules and would have turned CI red with no commit
   # of ours in between. Bump these deliberately, in their own MR.
-  "basedpyright>=1.39.9,<1.40",
+  #
+  # basedpyright is pinned *exactly*, because the bound above was not enough:
+  # 1.39.10 is a patch release built on pyright 1.1.412, which stopped solving
+  # a generic class passed as a `Callable[[], R]` factory from the declared
+  # type — `x: Box[int] = make(Box)` is an error there and fine on 1.1.411.
+  # Every `field(factory=ItemList, ...)` in the model is that shape, so a fresh
+  # environment failed `just check` with 44 errors and no commit of ours in
+  # between. Still broken on pyright 1.1.414 / basedpyright 1.40.2. The pin
+  # goes with basedpyright itself in the pyrefly-only migration.
+  "basedpyright==1.39.9",
   "catppuccin[pygments]>=2.5.0",
   "types-lxml>=2025.3.30",
   "mkdocs-material>=9.7.6",

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/__init__.py

@@ -19,7 +19,7 @@ from sdmxlib.session import Session, SessionHook, ValueSet
 from sdmxlib.query import RawSql, RawSqlError, schema
 from sdmxlib.query.accessors import ArtefactRow, CodeRow, HCodeRow, ReportedAttributeRow, q
 from sdmxlib.query._compile import CompiledQuery
-from sdmxlib.query._expr import Agg, AggPred, AnyPred, Col, Cols, Cursor, Pred, RowKind, Win, WinFn, WinPred
+from sdmxlib.query._expr import Agg, AggPred, AnyPred, Col, Cols, Cursor, Pred, RowKind, Untyped, Win, WinFn, WinPred
 from sdmxlib.query.observations import ObsQuery, ObsRow
 from sdmxlib.query.query import Frame, Grouped, Page, Query, union
 from sdmxlib.sql import SqlRelation
@@ -136,6 +136,27 @@ from sdmxlib.model import (
     artefact_urn,
 )
 
+
+def len[R: RowKind]() -> Agg[R]:  # noqa: A001 - polars' `pl.len()`; nothing in this module calls the builtin
+    """The number of rows, as an aggregate — `count(*)`, named for polars' `pl.len()`.
+
+    For `Query.agg`, which folds a whole query into one row, so the row count
+    can come out of the same scan as the other aggregates (#414):
+
+        obs.agg(SEX=obs.col("SEX").unique(), n=sl.len())
+
+    A function rather than a column method because it counts *rows*, not the
+    values of any column — the same reason `group_by(...).len()` is a method on
+    the grouping rather than on a column. The row kind is taken from where it is
+    used, so it fits any query's `agg`.
+
+    Shadows the builtin only as `sl.len`; this module never calls `len()`
+    itself, which is why it is defined here rather than re-exported from a
+    module that would need a second suppression.
+    """
+    return Agg[R]("count", ())
+
+
 __all__ = [
     "Agency",
     "AgencyScheme",
@@ -278,6 +299,7 @@ __all__ = [
     "SupportsUrn",
     "TimeDimension",
     "UnresolvedRef",
+    "Untyped",
     "ValidationError",
     "ValidationIssue",
     "ValueMap",
@@ -290,6 +312,7 @@ __all__ = [
     "dim",
     "from_dict",
     "item",
+    "len",
     "parse",
     "parse_components",
     "q",

{sdmxlib-0.63.0/src/sdmxlib/storage → sdmxlib-0.65.0/src/sdmxlib}/_time.py

@@ -35,12 +35,19 @@ breaking.
 from __future__ import annotations
 
 from datetime import UTC
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, overload
 
 if TYPE_CHECKING:
     from datetime import datetime
 
 
+# Overloaded so a value known to be present stays `datetime` on the way out —
+# a query literal (#415) is never None, and widening it to `datetime | None`
+# would push a pointless check onto every caller.
+@overload
+def to_storage(value: datetime) -> datetime: ...
+@overload
+def to_storage(value: None) -> None: ...
 def to_storage(value: datetime | None) -> datetime | None:
     """A datetime as this schema stores it: UTC, with `tzinfo` stripped.
 
@@ -60,6 +67,10 @@ def to_storage(value: datetime | None) -> datetime | None:
     return value.astimezone(UTC).replace(tzinfo=None)
 
 
+@overload
+def from_storage(value: datetime) -> datetime: ...
+@overload
+def from_storage(value: None) -> None: ...
 def from_storage(value: datetime | None) -> datetime | None:
     """A stored datetime as the model sees it: the same instant, tagged UTC.
 

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/api/_freshness.py

@@ -56,7 +56,7 @@ class Freshness:
     "has the pointer moved?", which validators answer without one.
     """
 
-    #: When the response was received. Naive UTC, per `storage._time`'s
+    #: When the response was received. Naive UTC, per `sdmxlib._time`'s
     #: convention, so comparisons cannot shift with a reader's zone.
     stored_at: datetime
     #: `ETag`, for `If-None-Match`. The cheapest revalidation there is.

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/api/client.py

@@ -223,7 +223,7 @@ def _raise_for_status(response: httpx.Response, *, url: str) -> None:
 def _now() -> datetime:
     """Now, as naive UTC.
 
-    `storage._time`'s convention, so a stored instant means the same thing to
+    `sdmxlib._time`'s convention, so a stored instant means the same thing to
     every reader regardless of their machine's zone.
     """
     return datetime.now(UTC).replace(tzinfo=None)

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_csv/writer.py

@@ -23,7 +23,7 @@ column maps in their dict-insertion order, which matches the order in
 which they were declared on the structure.
 
 Performance: rows are serialised via Polars' Rust CSV writer
-(``pl.from_arrow(batch).write_csv``) rather than Python's `csv`,
+(``pl.DataFrame(batch).write_csv``) rather than Python's `csv`,
 which makes the writer fast enough that bulk SDMX-CSV downloads are
 network-bound rather than writer-bound (#61).
 
@@ -43,7 +43,7 @@ Example:
 from __future__ import annotations
 
 import io
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 import polars as pl
 
@@ -143,10 +143,11 @@ def write_sdmx_csv(
             )
             raise KeyError(msg)
 
-        # pl.from_arrow returns DataFrame | Series; a RecordBatch always
-        # gives back a DataFrame, but the union is sticky in the stubs.
+        # The constructor rather than `pl.from_arrow`, which is typed
+        # `DataFrame | Series` and since polars 2.0 really does return a Series
+        # for stream-only inputs — a `cast` here would have hidden that.
         df = (
-            cast("pl.DataFrame", pl.from_arrow(batch))
+            pl.DataFrame(batch)
             .select(data_cols)
             .with_columns(
                 pl.lit(structure).alias("STRUCTURE"),

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/_data_common.py

@@ -14,7 +14,7 @@ from __future__ import annotations
 import calendar
 import re
 from datetime import datetime, timedelta
-from typing import TYPE_CHECKING, Literal, NamedTuple, TypedDict, cast
+from typing import TYPE_CHECKING, Literal, NamedTuple, TypedDict
 
 import msgspec
 import polars as pl
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
     import pyarrow as pa
 
     from sdmxlib.model.annotations import Annotations
-    from sdmxlib.model.codelist import Codelist, CodelistLike
+    from sdmxlib.model.codelist import Code, CodelistLike
     from sdmxlib.model.dataflow import Dataflow
     from sdmxlib.model.datastructure import Component, DataStructure, Dimension
     from sdmxlib.model.istring import InternationalString
@@ -194,28 +194,16 @@ def _frequency_dimension(dsd: DataStructure) -> Dimension | None:
     return None
 
 
-def frequency_durations(dsd: DataStructure, lang: str) -> tuple[str | None, dict[str, _Duration]]:
-    """Find the frequency dimension and its codes' annotation-derived durations.
+def frequency_dimension_id(dsd: DataStructure) -> str | None:
+    """The id of the DSD's frequency dimension, or None if it has none.
 
-    Returns ``(freq_dim_id, overrides)``. ``overrides`` only carries codes that
-    declare an explicit ``duration`` annotation — the standard/``A{n}``
-    convention is applied lazily at lookup time, so codes outside the codelist
-    (or an unresolved codelist) still resolve by convention.
+    Only the id. The FREQ codes' ``duration`` annotations are read by
+    `ComponentTable` from the codes a message actually observed, in the same
+    batched lookup that labels them — this used to materialise the whole FREQ
+    codelist up front, one of the per-message full reads #410 removed.
     """
     freq = _frequency_dimension(dsd)
-    if freq is None:
-        return None, {}
-    overrides: dict[str, _Duration] = {}
-    codelist: CodelistLike | None = getattr(freq, "resolved_codelist", None)
-    if codelist is not None:
-        # `materialize()` because this reads each code's *annotations*, which
-        # is past what `CodelistLike` offers; free when the codelist is already
-        # eager, and a frequency codelist is a handful of codes either way.
-        for code in codelist.materialize().codes:
-            duration = _annotation_duration(code.annotations, lang)
-            if duration is not None:
-                overrides[code.id] = duration
-    return freq.id, overrides
+    return freq.id if freq is not None else None
 
 
 def _add_duration(start: datetime, duration: _Duration) -> datetime:
@@ -316,6 +304,16 @@ class ComponentTable:
     SDMX-JSON spec explicitly allows this) rather than the full
     enumeration. The StatCan census geography codelist (579k codes)
     would otherwise blow up a ``limit=2`` data response to >30 MB.
+
+    **The cost follows the response too, not just its size.** A value's
+    name, ``parent`` and annotations are filled in at `to_json`, by one
+    `CodelistLike.codes_for` per component over exactly the ids observed.
+    Calling `materialize` here instead read every code of every enumerated
+    component on every message once the DSD came from a `LocalRegistry` —
+    1.4 s to write one observation against a 200k-code geography, where the
+    same DSD with eager codelists took 3 ms (#410). Looking each code up as it
+    arrived would be one round trip per distinct value, which is the same
+    mistake at the other end of the scale.
     """
 
     def __init__(
@@ -328,7 +326,6 @@ class ComponentTable:
         annotations: AnnotationPool,
         time_dim_id: str | None = None,
         freq_dim_id: str | None = None,
-        freq_overrides: dict[str, _Duration] | None = None,
     ) -> None:
         self._ids = component_ids
         # Shared with the sibling table — see `AnnotationPool`. Required rather
@@ -342,11 +339,12 @@ class ComponentTable:
         self._kind = kind
         self._time_dim_id = time_dim_id
         # FREQ-driven period bounds (issue #149): the frequency dimension id,
-        # its codes' annotation-derived duration overrides, and the
-        # accumulated TIME_PERIOD id → set of co-occurring FREQ ids seen while
-        # streaming. Bounds are resolved at to_json() time from this.
+        # its observed codes' annotation-derived duration overrides (filled
+        # by `_resolve_codes`, alongside their labels), and the accumulated
+        # TIME_PERIOD id → set of co-occurring FREQ ids seen while streaming.
+        # Bounds are resolved at to_json() time from these.
         self._freq_dim_id = freq_dim_id
-        self._freq_overrides = freq_overrides or {}
+        self._freq_overrides: dict[str, _Duration] = {}
         self._time_freqs: dict[str, set[str]] = {}
         # ``dsd.dimensions`` is ItemList[Dimension] and does NOT include
         # the time dimension (which lives at ``dsd.time_dimension``).
@@ -370,14 +368,12 @@ class ComponentTable:
         # so position N maps to the Nth-inserted entry. ``to_json``
         # walks this directly.
         self._codes: dict[str, dict[str, dict[str, object]]] = {cid: {} for cid in component_ids}
-        # Per-component resolved codelists, stored by reference. Labels are
-        # resolved on-demand in ``position()`` via ``ItemList.get`` (O(1));
-        # we deliberately do NOT walk the full codelist here. For census-scale
-        # codelists (CL_GEOGRAPHY: 579k codes) the eager walk dominated wall
-        # time on small data responses — a single-row /data query spent
-        # multiple seconds populating a label cache it would never read more
-        # than once from.
-        self._codelists: dict[str, Codelist] = {}
+        # Per-component resolved codelists, held as they came — eager or lazy
+        # — and never walked: `_resolve_codes` asks each for the observed ids
+        # only. For census-scale codelists (CL_GEOGRAPHY: 579k codes) the walk
+        # dominated wall time on small data responses, first as a label cache
+        # and again as a per-message `materialize()` (#410).
+        self._codelists: dict[str, CodelistLike] = {}
 
         for cid in component_ids:
             comp = components.get(cid)
@@ -395,10 +391,7 @@ class ComponentTable:
             # getattr returns Any; the annotation kills it on this line.
             cl: CodelistLike | None = getattr(comp, "resolved_codelist", None)
             if cl is not None:
-                # Materialised because `value_entry` looks codes up by id and
-                # reads name, parent_id and annotations off each — the full
-                # `Code`, not the id-and-label surface `CodelistLike` shares.
-                self._codelists[cid] = cl.materialize()
+                self._codelists[cid] = cl
             comp_anns = self._annotations.register(comp.annotations)
             if comp_anns:
                 self._component_annotations[cid] = comp_anns
@@ -412,22 +405,39 @@ class ComponentTable:
         pos = len(table)
         table[code_id] = pos
         self._position_str[component_id][code_id] = str(pos)
-        # Both SDMX-JSON 1.0 and 2.0 schemas require every value entry
-        # to carry id + name. Look the localized name up on-demand from
-        # the resolved codelist's id-indexed ItemList (O(1)); fall back
-        # to the id when no codelist is registered or the code is
-        # unknown to it (e.g. observed values outside the enumerated set).
-        label = code_id
-        parent_id: str | None = None
-        value_anns: list[int] = []
-        cl = self._codelists.get(component_id)
-        if cl is not None:
-            code = cl.codes.get(code_id)
-            if code is not None:
-                label = code.name.get(self._lang) or code_id
-                parent_id = code.parent_id
-                value_anns = self._annotations.register(code.annotations)
-        entry: dict[str, object] = {"id": code_id, "name": label}
+        # Both SDMX-JSON 1.0 and 2.0 schemas require every value entry to carry
+        # id + name. The id stands in for the name until `_resolve_codes`
+        # replaces it, and stays for a value the codelist does not hold (or a
+        # component with no codelist).
+        self._codes[component_id][code_id] = {"id": code_id, "name": code_id}
+        return pos
+
+    def _resolve_codes(self) -> None:
+        """Fill each observed value's name, ``parent`` and annotations.
+
+        One `CodelistLike.codes_for` per enumerated component, over the ids
+        this message observed — the whole point of deferring it here (#410).
+        Annotations are registered in position order, so the pool's indices
+        are deterministic for a given input.
+        """
+        for component_id, cl in self._codelists.items():
+            entries = self._codes[component_id]
+            if not entries:
+                continue
+            found = cl.codes_for(entries.keys())
+            for code_id, entry in entries.items():
+                code = found.get(code_id)
+                if code is None:
+                    continue
+                if component_id == self._freq_dim_id:
+                    duration = _annotation_duration(code.annotations, self._lang)
+                    if duration is not None:
+                        self._freq_overrides[code_id] = duration
+                self._fill_entry(entry, code)
+
+    def _fill_entry(self, entry: dict[str, object], code: Code) -> None:
+        """Write one resolved *code*'s name, ``parent`` and annotations into *entry*."""
+        entry["name"] = code.name.get(self._lang) or code.id
         # A value's hierarchy edge (#365). `parent` is a first-class property of
         # this object in the 1.0, 2.0 and 2.1 data schemas, and omitting it made
         # every hierarchical codelist arrive flat: the .Stat Data Explorer groups
@@ -446,20 +456,17 @@ class ComponentTable:
         # `IDType`, so an id it rejects was already invalid SDMX upstream. Adding
         # a filter here would turn someone else's malformed codelist into a
         # clean-looking document with a silently missing hierarchy.
-        if parent_id is not None:
-            entry["parent"] = parent_id
+        if code.parent_id is not None:
+            entry["parent"] = code.parent_id
         # The code's annotations, as indices into the shared pool (#366). This
         # is how a consumer gets a value's ORDER, NOT_DISPLAYED and the rest —
         # the *type* vocabulary is the annotation creator's, not SDMX's ("the
         # types are not enumerated, as these can be specified by the user or
         # creator", per the AnnotationType documentation in every SDMX-ML
         # version), so the writer emits all of them and filters none.
+        value_anns = self._annotations.register(code.annotations)
         if value_anns:
             entry["annotations"] = value_anns
-        # TIME_PERIOD start/end is resolved in to_json() once the full set of
-        # co-occurring FREQ values is known (see _resolve_time_bounds).
-        self._codes[component_id][code_id] = entry
-        return pos
 
     def tracks_freq(self) -> bool:
         """True if both a time dimension and a frequency dimension are known."""
@@ -532,7 +539,12 @@ class ComponentTable:
         return {cid: pos_str[cid] for cid in code_ids}
 
     def to_json(self) -> list[dict[str, object]]:
-        """Render the per-position metadata array for the structures block."""
+        """Render the per-position metadata array for the structures block.
+
+        Call once, after the last batch: this is where observed values are
+        resolved against their codelists.
+        """
+        self._resolve_codes()
         out: list[dict[str, object]] = []
         for key_pos, cid in enumerate(self._ids):
             code_entries = self._codes[cid]
@@ -628,7 +640,11 @@ def encode_batch(
         )
         raise KeyError(msg)
 
-    df = cast("pl.DataFrame", pl.from_arrow(batch))
+    # The constructor, not `pl.from_arrow`: the same frame for a RecordBatch, but
+    # typed `DataFrame` rather than `DataFrame | Series` — and polars 2.0 made
+    # `from_arrow` return a Series for stream-only inputs, which a `cast` would
+    # have passed through unchecked.
+    df = pl.DataFrame(batch)
     n = df.height
     if n == 0:
         return b""

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/data_v1.py

@@ -49,7 +49,7 @@ from sdmxlib.formats.sdmx_json._data_common import (
     decode_observations,
     display_names,
     encode_batch,
-    frequency_durations,
+    frequency_dimension_id,
     resolve_columns,
     rows_to_lazyframe,
 )
@@ -121,7 +121,7 @@ def write_sdmx_json_data(
     dim_cols, dim_phys, attr_cols, attr_phys, _measure_id, measure_col = resolve_columns(dsd)
 
     time_dim_id = dsd.time_dimension.id if dsd.time_dimension is not None else "TIME_PERIOD"
-    freq_dim_id, freq_overrides = frequency_durations(dsd, lang)
+    freq_dim_id = frequency_dimension_id(dsd)
     # One pool for the whole structure block — the two tables emit integer
     # indices into it, and an index is meaningless without knowing which array
     # it indexes (#366).
@@ -134,7 +134,6 @@ def write_sdmx_json_data(
         annotations=annotations,
         time_dim_id=time_dim_id,
         freq_dim_id=freq_dim_id,
-        freq_overrides=freq_overrides,
     )
     attr_lookup = ComponentTable(attr_cols, dsd, kind="attribute", lang=lang, annotations=annotations)
 

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/formats/sdmx_json/data_v2.py

@@ -63,7 +63,7 @@ from sdmxlib.formats.sdmx_json._data_common import (
     decode_observations,
     display_names,
     encode_batch,
-    frequency_durations,
+    frequency_dimension_id,
     resolve_columns,
     rows_to_lazyframe,
 )
@@ -142,7 +142,7 @@ def write_sdmx_json_data(
     dim_cols, dim_phys, attr_cols, attr_phys, _measure_id, measure_col = resolve_columns(dsd)
 
     time_dim_id = dsd.time_dimension.id if dsd.time_dimension is not None else "TIME_PERIOD"
-    freq_dim_id, freq_overrides = frequency_durations(dsd, lang)
+    freq_dim_id = frequency_dimension_id(dsd)
     # One pool for the whole structure block — the two tables emit integer
     # indices into it, and an index is meaningless without knowing which array
     # it indexes (#366).
@@ -155,7 +155,6 @@ def write_sdmx_json_data(
         annotations=annotations,
         time_dim_id=time_dim_id,
         freq_dim_id=freq_dim_id,
-        freq_overrides=freq_overrides,
     )
     attr_lookup = ComponentTable(attr_cols, dsd, kind="attribute", lang=lang, annotations=annotations)
 

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/local/sql.py

@@ -59,6 +59,20 @@ if TYPE_CHECKING:
     from sdmxlib.model.binding import DataflowBinding
 
 
+class SidecarNotUniqueError(ValueError):
+    """A series-attributes sidecar holds more than one row per join key (#412).
+
+    The sidecar is joined as `LEFT JOIN … USING (<keys>)`, so every extra row
+    per key multiplies each observation it matches. Nothing errors downstream:
+    `count()`, `collect()` and every aggregate over a sidecar column come back
+    inflated by the fan-out factor, and every row looks valid.
+
+    A `ValueError` because it is the same kind of refusal `plan` already makes
+    for a sidecar sharing no columns — the binding names a table that cannot
+    serve as its series attributes — so a caller catching that catches this.
+    """
+
+
 @frozen(kw_only=True)
 class QueryPlan:
     """The resolved ``FROM`` side of a `DataflowBinding`'s query.
@@ -109,6 +123,10 @@ class SchemaInspector:
     def __init__(self, conn: duckdb.DuckDBPyConnection) -> None:
         self._conn = conn
         self._columns_cache: dict[str, tuple[str, ...]] = {}
+        #: (table, keys) -> (rows, distinct keys), for `key_counts`. Shares
+        #: `_columns_cache`'s lock and lifecycle: both describe a table as it
+        #: stands, and `invalidate` is the one signal that it changed.
+        self._key_counts_cache: dict[tuple[str, tuple[str, ...]], tuple[int, int]] = {}
         self._lock = threading.Lock()
 
     def columns(self, table: str) -> tuple[str, ...]:
@@ -139,15 +157,50 @@ class SchemaInspector:
         cols_b = set(self.columns(table_b))
         return tuple(c for c in self.columns(table_a) if c in cols_b)
 
+    def key_counts(self, table: str, keys: tuple[str, ...]) -> tuple[int, int]:
+        """``(rows, distinct key combinations)`` of ``table`` over ``keys`` (cached).
+
+        Rows with a NULL in any key are left out of both counts: `USING` never
+        matches NULL, so repeated NULL-keyed rows cannot multiply a join, and
+        refusing a table for them would be refusing a harmless one.
+
+        A scan of the table — ~5 ms on cenprof-2021-csip's 619-row geography
+        sidecar, ~37 ms cold — so memoised single-flight beside `columns`, and
+        dropped by `invalidate` with it.
+
+        Args:
+            table: The table to count, as `columns` takes it.
+            keys: The columns whose combinations are counted.
+        """
+        cache_key = (table, keys)
+        cached = self._key_counts_cache.get(cache_key)
+        if cached is not None:
+            return cached
+        with self._lock:
+            cached = self._key_counts_cache.get(cache_key)
+            if cached is not None:
+                return cached
+            tuple_sql = ", ".join(keys)
+            not_null = " AND ".join(f"{k} IS NOT NULL" for k in keys)
+            row = self._conn.execute(
+                f"SELECT count(*), count(DISTINCT ({tuple_sql})) FROM {table} WHERE {not_null}"
+            ).fetchone()
+            counts = (int(row[0]), int(row[1])) if row else (0, 0)
+            self._key_counts_cache[cache_key] = counts
+            return counts
+
     def invalidate(self, table: str) -> None:
-        """Drop the cached column list for ``table`` (e.g. after a rewrite)."""
+        """Drop everything cached about ``table`` (e.g. after a rewrite)."""
         with self._lock:
             self._columns_cache.pop(table, None)
+            for cache_key in [k for k in self._key_counts_cache if k[0] == table]:
+                del self._key_counts_cache[cache_key]
 
     def invalidate_all(self) -> None:
-        """Drop all cached column lists."""
+        """Drop everything cached about every table."""
         with self._lock:
             self._columns_cache.clear()
+            self._key_counts_cache.clear()
 
     def plan(self, binding: DataflowBinding) -> QueryPlan:
         """Resolve ``binding``'s main/sidecar tables into a `QueryPlan`.
@@ -163,6 +216,9 @@ class SchemaInspector:
         Raises:
             ValueError: If the binding names a sidecar that shares no columns
                 with the main table (no derivable join keys).
+            SidecarNotUniqueError: If the sidecar holds more than one row for
+                some combination of those keys, which would multiply every
+                observation the join matches (#412).
         """
         main = f"{binding.database_role}.{binding.table_location}"
         sidecar_loc = binding.series_attributes_location
@@ -176,4 +232,15 @@ class SchemaInspector:
                 f"shares no columns with {main!r}; cannot derive JOIN keys"
             )
             raise ValueError(msg)
+        rows, distinct = self.key_counts(sidecar, join_keys)
+        if rows != distinct:
+            keys = ", ".join(join_keys)
+            msg = (
+                f"series-attributes sidecar {sidecar!r} holds {rows:,} rows but only {distinct:,} distinct "
+                f"({keys}) — joined on those keys, every observation of a repeated series would be counted "
+                f"once per sidecar row, so counts and aggregates over its columns come back inflated (#412). "
+                f"The keys are the columns the two tables share; make the sidecar unique on them, or drop the "
+                f"shared columns that are not part of the series key."
+            )
+            raise SidecarNotUniqueError(msg)
         return QueryPlan(main=main, sidecar=sidecar, join_keys=join_keys)

{sdmxlib-0.63.0 → sdmxlib-0.65.0}/src/sdmxlib/model/codelist.py

@@ -1,5 +1,6 @@
 """Codelist and Code — enumerated value sets."""
 
+from collections.abc import Iterable
 from datetime import datetime
 from enum import Enum
 from typing import TYPE_CHECKING, ClassVar, Protocol, runtime_checkable
@@ -186,14 +187,28 @@ class CodelistLike(Protocol):
         """``{code_id: label}`` in ``lang``, falling back to the id."""
         ...
 
+    def codes_for(self, ids: Iterable[str]) -> dict[str, Code]:
+        """``{code_id: Code}`` for exactly *ids*, each fully built.
+
+        Ids the codelist does not hold are absent from the result rather than
+        an error — a caller resolving observed values meets values outside the
+        enumeration, and the id is then its own label.
+
+        The targeted counterpart to `materialize`: one batched read on a lazy
+        codelist, whose cost follows ``len(ids)`` rather than the codelist. The
+        SDMX-JSON data writer called `materialize` per component per message
+        and paid for every code in a 579,951-code geography to label the
+        handful a response observed (#410).
+        """
+        ...
+
     def materialize(self) -> "Codelist":
         """This codelist with every `Code` built, for callers needing more.
 
         Free on an eager `Codelist`, which returns itself; a full read on a
-        lazy one. Declared here so a caller that genuinely needs each code's
-        annotations or ``parent_id`` — the SDMX-JSON data writer does — asks
-        for that in one visible call instead of iterating something that might
-        be a `Query`.
+        lazy one, whose cost is the whole codelist. A caller that needs some
+        codes' annotations or ``parent_id`` rather than all of them wants
+        `codes_for` instead — that was the SDMX-JSON data writer (#410).
         """
         ...
 
@@ -385,6 +400,13 @@ class Codelist(MaintainableArtefact):
         """
         return self.codes.ids()
 
+    def codes_for(self, ids: Iterable[str]) -> dict[str, Code]:
+        """``{code_id: Code}`` for exactly *ids*; unknown ids are absent.
+
+        The eager half of `CodelistLike.codes_for` — an `ItemList.get` per id.
+        """
+        return {cid: code for cid in ids if (code := self.codes.get(cid)) is not None}
+
     def materialize(self) -> "Codelist":
         """This codelist — it is already materialised.