sdmxlib 0.41.0__tar.gz → 0.41.2__tar.gz

{sdmxlib-0.41.0 → sdmxlib-0.41.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.41.0
+Version: 0.41.2
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner

{sdmxlib-0.41.0 → sdmxlib-0.41.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sdmxlib"
-version = "0.41.0"
+version = "0.41.2"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }

{sdmxlib-0.41.0 → sdmxlib-0.41.2}/src/sdmxlib/formats/sdmx_json/_data_common.py

@@ -13,7 +13,8 @@ from __future__ import annotations
 
 import calendar
 import re
-from typing import TYPE_CHECKING, Any, Literal, cast
+from datetime import datetime, timedelta
+from typing import TYPE_CHECKING, Any, Literal, NamedTuple, cast
 
 import orjson
 import polars as pl
@@ -21,8 +22,10 @@ import polars as pl
 if TYPE_CHECKING:
     import pyarrow as pa
 
+    from sdmxlib.model.annotations import Annotations
     from sdmxlib.model.binding import DataflowBinding
     from sdmxlib.model.datastructure import DataStructure
+    from sdmxlib.model.istring import InternationalString
 
 from sdmxlib.model.ref import Ref
 
@@ -96,6 +99,142 @@ def _period_bounds(period_id: str) -> tuple[str, str] | None:
     return None
 
 
+# ── FREQ-driven multi-period bounds (issue #149) ──────────────────────────────
+#
+# A bare TIME_PERIOD id like ``"2021"`` is only a *start anchor*; the period's
+# length comes from the FREQ dimension's code (``A`` = 1 year, ``A5`` = 5 years,
+# ``Q`` = 1 quarter, ...). ``_period_bounds`` alone treats ``"2021"`` as a single
+# calendar year, which under-reports coverage for multi-year frequencies. The
+# writer resolves each FREQ code to a duration — an explicit ``duration``
+# annotation on the code overrides the standard/``A{n}`` convention — and uses it
+# to extend the end of the period. We only ever *extend* past the id-shape bound,
+# never shrink below it, so standard single-period frequencies and inconsistent
+# data stay byte-identical to the id-shape default.
+
+
+class _Duration(NamedTuple):
+    """A calendar duration decomposed into addable parts (weeks fold into days)."""
+
+    years: int = 0
+    months: int = 0
+    days: int = 0
+    hours: int = 0
+
+
+# Implied duration of the standard single-letter SDMX frequency codes.
+_FREQ_UNIT: dict[str, _Duration] = {
+    "A": _Duration(years=1),
+    "Y": _Duration(years=1),
+    "S": _Duration(months=6),
+    "Q": _Duration(months=3),
+    "M": _Duration(months=1),
+    "W": _Duration(days=7),
+    "D": _Duration(days=1),
+    "H": _Duration(hours=1),
+}
+# ``A5`` / ``A10`` / ``S2`` / ... — a standard unit times a positive multiple.
+_FREQ_MULTIPLE = re.compile(r"^([AYSQMWDH])(\d+)$")
+# ISO-8601 duration subset we accept on a ``duration`` annotation.
+_ISO_DURATION = re.compile(r"^P(?:(\d+)Y)?(?:(\d+)M)?(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?)?$")
+
+
+def _parse_iso_duration(text: str) -> _Duration | None:
+    """Parse the ISO-8601 duration subset (``P5Y``, ``P1M``, ``PT1H``, ...)."""
+    m = _ISO_DURATION.match(text.strip())
+    if not m or not any(m.groups()):
+        return None
+    y, mo, w, d, h = (int(g) if g else 0 for g in m.groups())
+    return _Duration(years=y, months=mo, days=w * 7 + d, hours=h)
+
+
+def _convention_duration(freq_id: str) -> _Duration | None:
+    """Duration implied by a FREQ code id via the standard/``A{n}`` convention."""
+    unit = _FREQ_UNIT.get(freq_id)
+    if unit is not None:
+        return unit
+    m = _FREQ_MULTIPLE.match(freq_id)
+    if m is None:
+        return None
+    base = _FREQ_UNIT[m.group(1)]
+    n = int(m.group(2))
+    return _Duration(years=base.years * n, months=base.months * n, days=base.days * n, hours=base.hours * n)
+
+
+def _first_text(istr: InternationalString) -> str | None:
+    """First localized value of an InternationalString, or None if empty."""
+    first = next(iter(istr.items()), None)
+    return first[1] if first is not None else None
+
+
+def _annotation_duration(annotations: Annotations, lang: str) -> _Duration | None:
+    """Duration from a ``duration`` annotation (case-insensitive type), if any."""
+    for ann in annotations:
+        if ann.type and ann.type.strip().lower() == "duration":
+            text = ann.text.get(lang) or _first_text(ann.text)
+            if text:
+                return _parse_iso_duration(text)
+    return None
+
+
+def _frequency_dimension(dsd: DataStructure) -> Any:
+    """The DSD's frequency dimension, by well-known id then concept role."""
+    for dim in dsd.dimensions:
+        if dim.id in ("FREQ", "FREQUENCY"):
+            return dim
+    for dim in dsd.dimensions:
+        roles = getattr(dim, "concept_role", ()) or ()
+        if any(str(r).upper() in ("FREQ", "FREQUENCY") for r in roles):
+            return dim
+    return None
+
+
+def frequency_durations(dsd: DataStructure, lang: str) -> tuple[str | None, dict[str, _Duration]]:
+    """Find the frequency dimension and its codes' annotation-derived durations.
+
+    Returns ``(freq_dim_id, overrides)``. ``overrides`` only carries codes that
+    declare an explicit ``duration`` annotation — the standard/``A{n}``
+    convention is applied lazily at lookup time, so codes outside the codelist
+    (or an unresolved codelist) still resolve by convention.
+    """
+    freq = _frequency_dimension(dsd)
+    if freq is None:
+        return None, {}
+    overrides: dict[str, _Duration] = {}
+    codelist = getattr(freq, "resolved_codelist", None)
+    if codelist is not None:
+        for code in codelist.codes:
+            duration = _annotation_duration(code.annotations, lang)
+            if duration is not None:
+                overrides[code.id] = duration
+    return freq.id, overrides
+
+
+def _add_duration(start: datetime, duration: _Duration) -> datetime:
+    """Add a `_Duration` to *start*, clamping the day to the target month."""
+    total_months = start.year * 12 + (start.month - 1) + duration.years * 12 + duration.months
+    year, month0 = divmod(total_months, 12)
+    month = month0 + 1
+    day = min(start.day, calendar.monthrange(year, month)[1])
+    shifted = start.replace(year=year, month=month, day=day)
+    return shifted + timedelta(days=duration.days, hours=duration.hours)
+
+
+def _bounds_from_anchor(period_id: str, duration: _Duration | None) -> tuple[str, str] | None:
+    """``(start, end)`` for *period_id* extended by *duration*.
+
+    ``start`` is the id's anchor (reusing `_period_bounds`); ``end`` is
+    ``start + duration - 1s``. Returns the plain id-shape bounds when *duration*
+    is None, and None when the id shape itself is unrecognised (e.g. the
+    ``<start>/<duration>`` form the consumer parses itself).
+    """
+    base = _period_bounds(period_id)
+    if base is None or duration is None:
+        return base
+    start_dt = datetime.fromisoformat(base[0])
+    end_dt = _add_duration(start_dt, duration) - timedelta(seconds=1)
+    return base[0], end_dt.strftime("%Y-%m-%dT%H:%M:%S")
+
+
 class ComponentTable:
     """Lazy id → position table for one component kind (dim or attr).
 
@@ -119,11 +258,20 @@ class ComponentTable:
         kind: ComponentKind,
         lang: str,
         time_dim_id: str | None = None,
+        freq_dim_id: str | None = None,
+        freq_overrides: dict[str, _Duration] | None = None,
     ) -> None:
         self._ids = component_ids
         self._lang = lang
         self._kind = kind
         self._time_dim_id = time_dim_id
+        # FREQ-driven period bounds (issue #149): the frequency dimension id,
+        # its codes' annotation-derived duration overrides, and the
+        # accumulated TIME_PERIOD id → set of co-occurring FREQ ids seen while
+        # streaming. Bounds are resolved at to_json() time from this.
+        self._freq_dim_id = freq_dim_id
+        self._freq_overrides = freq_overrides or {}
+        self._time_freqs: dict[str, set[str]] = {}
         # ``dsd.dimensions`` is ItemList[Dimension] and does NOT include
         # the time dimension (which lives at ``dsd.time_dimension``).
         # Use ``key_dimensions`` so TIME_PERIOD's concept-derived name
@@ -146,11 +294,14 @@ class ComponentTable:
         # so position N maps to the Nth-inserted entry. ``to_json``
         # walks this directly.
         self._codes: dict[str, dict[str, dict[str, object]]] = {cid: {} for cid in component_ids}
-        # static label cache (codelist code id → localized name). Read
-        # at code-registration time to fill the ``name`` field; never
-        # iterated for emission, so unobserved codes stay out of the
-        # serialised values array.
-        self._code_labels: dict[str, dict[str, str]] = {cid: {} for cid in component_ids}
+        # Per-component resolved codelists, stored by reference. Labels are
+        # resolved on-demand in ``position()`` via ``ItemList.get`` (O(1));
+        # we deliberately do NOT walk the full codelist here. For census-scale
+        # codelists (CL_GEOGRAPHY: 579k codes) the eager walk dominated wall
+        # time on small data responses — a single-row /data query spent
+        # multiple seconds populating a label cache it would never read more
+        # than once from.
+        self._codelists: dict[str, Any] = {}
 
         for cid in component_ids:
             comp = components.get(cid)
@@ -164,12 +315,10 @@ class ComponentTable:
             self._names[cid] = name
             # TimeDimension carries a Facet representation, not a codelist
             # ref, so it has no ``resolved_codelist`` property — skip the
-            # label-cache step for components without one.
+            # codelist registration for components without one.
             cl = getattr(comp, "resolved_codelist", None)
-            if cl is None:
-                continue
-            for code in cl.codes:
-                self._code_labels[cid][code.id] = code.name.get(lang) or code.id
+            if cl is not None:
+                self._codelists[cid] = cl
 
     def position(self, component_id: str, code_id: str) -> int:
         """Look up ``code_id`` under ``component_id``, assigning on first sight."""
@@ -181,17 +330,71 @@ class ComponentTable:
         table[code_id] = pos
         self._position_str[component_id][code_id] = str(pos)
         # Both SDMX-JSON 1.0 and 2.0 schemas require every value entry
-        # to carry id + name. Pull the localized name from the codelist
-        # label cache if known, otherwise fall back to the id.
-        label = self._code_labels[component_id].get(code_id, code_id)
+        # to carry id + name. Look the localized name up on-demand from
+        # the resolved codelist's id-indexed ItemList (O(1)); fall back
+        # to the id when no codelist is registered or the code is
+        # unknown to it (e.g. observed values outside the enumerated set).
+        label = code_id
+        cl = self._codelists.get(component_id)
+        if cl is not None:
+            code = cl.codes.get(code_id)
+            if code is not None:
+                label = code.name.get(self._lang) or code_id
         entry: dict[str, object] = {"id": code_id, "name": label}
-        if component_id == self._time_dim_id:
-            bounds = _period_bounds(code_id)
-            if bounds is not None:
-                entry["start"], entry["end"] = bounds
+        # TIME_PERIOD start/end is resolved in to_json() once the full set of
+        # co-occurring FREQ values is known (see _resolve_time_bounds).
         self._codes[component_id][code_id] = entry
         return pos
 
+    def tracks_freq(self) -> bool:
+        """True if both a time dimension and a frequency dimension are known."""
+        return self._time_dim_id is not None and self._freq_dim_id is not None
+
+    def observe_batch_time_freq(self, df: pl.DataFrame, dim_cols: list[str], dim_phys: list[str]) -> None:
+        """Record the distinct ``(TIME_PERIOD, FREQ)`` pairs in *df*.
+
+        Cheap per-batch vectorised ``unique()`` — accumulated across batches so
+        `_resolve_time_bounds` can pick each period's duration from the
+        frequencies it actually co-occurs with.
+        """
+        if not self.tracks_freq() or self._time_dim_id not in dim_cols or self._freq_dim_id not in dim_cols:
+            return
+        time_col = dim_phys[dim_cols.index(self._time_dim_id)]
+        freq_col = dim_phys[dim_cols.index(self._freq_dim_id)]
+        pairs = df.select(
+            pl.col(time_col).cast(pl.String).fill_null("").alias("__t"),
+            pl.col(freq_col).cast(pl.String).fill_null("").alias("__f"),
+        ).unique()
+        times = pairs.get_column("__t").to_list()
+        freqs = pairs.get_column("__f").to_list()
+        for time_id, freq_id in zip(times, freqs, strict=True):
+            self._time_freqs.setdefault(time_id, set()).add(freq_id)
+
+    def _duration_for_freq(self, freq_id: str) -> _Duration | None:
+        """Duration for a FREQ code: annotation override else convention."""
+        return self._freq_overrides.get(freq_id) or _convention_duration(freq_id)
+
+    def _resolve_time_bounds(self, code_id: str) -> tuple[str, str] | None:
+        """Period bounds for a TIME_PERIOD id, extended by FREQ where unambiguous.
+
+        Only extends past the plain id-shape bound — never shrinks below it — so
+        standard single-period frequencies and contradictory data stay identical
+        to the id-shape default. A period that co-occurs with conflicting
+        durations (mixed FREQ) falls back to the id-shape default.
+        """
+        base = _period_bounds(code_id)
+        freqs = self._time_freqs.get(code_id)
+        if base is None or not freqs:
+            return base
+        durations = {self._duration_for_freq(f) for f in freqs}
+        durations.discard(None)
+        if len(durations) != 1:
+            return base
+        extended = _bounds_from_anchor(code_id, next(iter(durations)))
+        if extended is None:
+            return base
+        return extended if extended[1] > base[1] else base
+
     def register_many(self, component_id: str, code_ids: list[str]) -> dict[str, str]:
         """Ensure every *code_id* has a position; return a focused str-map.
 
@@ -213,10 +416,16 @@ class ComponentTable:
         """Render the per-position metadata array for the structures block."""
         out: list[dict[str, object]] = []
         for key_pos, cid in enumerate(self._ids):
+            code_entries = self._codes[cid]
+            if cid == self._time_dim_id:
+                for code_id, value_entry in code_entries.items():
+                    bounds = self._resolve_time_bounds(code_id)
+                    if bounds is not None:
+                        value_entry["start"], value_entry["end"] = bounds
             entry: dict[str, object] = {
                 "id": cid,
                 "name": self._names.get(cid, cid),
-                "values": list(self._codes[cid].values()),
+                "values": list(code_entries.values()),
             }
             if self._kind == "dimension":
                 entry["keyPosition"] = key_pos
@@ -298,6 +507,9 @@ def encode_batch(
         uniq = df.get_column(dim_phys[col_idx]).cast(pl.String).fill_null("").unique(maintain_order=True).to_list()
         dim_str_tables.append(dim_lookup.register_many(cid, uniq))
 
+    # Accumulate (TIME_PERIOD, FREQ) co-occurrence for duration-aware bounds.
+    dim_lookup.observe_batch_time_freq(df, dim_cols, dim_phys)
+
     attr_int_tables: list[dict[str, int]] = []
     for col_idx, cid in enumerate(attr_cols):
         uniq = df.get_column(attr_phys[col_idx]).cast(pl.String).drop_nulls().unique(maintain_order=True).to_list()

{sdmxlib-0.41.0 → sdmxlib-0.41.2}/src/sdmxlib/formats/sdmx_json/data_v1.py

@@ -46,6 +46,7 @@ from sdmxlib.formats.sdmx_json._data_common import (
     decode_observations,
     display_names,
     encode_batch,
+    frequency_durations,
     resolve_columns,
     rows_to_lazyframe,
 )
@@ -116,7 +117,16 @@ def write_sdmx_json_data(
     dim_cols, dim_phys, attr_cols, attr_phys, _measure_id, measure_col = resolve_columns(binding)
 
     time_dim_id = dsd.time_dimension.id if dsd.time_dimension is not None else "TIME_PERIOD"
-    dim_lookup = ComponentTable(dim_cols, dsd, kind="dimension", lang=lang, time_dim_id=time_dim_id)
+    freq_dim_id, freq_overrides = frequency_durations(dsd, lang)
+    dim_lookup = ComponentTable(
+        dim_cols,
+        dsd,
+        kind="dimension",
+        lang=lang,
+        time_dim_id=time_dim_id,
+        freq_dim_id=freq_dim_id,
+        freq_overrides=freq_overrides,
+    )
     attr_lookup = ComponentTable(attr_cols, dsd, kind="attribute", lang=lang)
 
     names = display_names(binding, dsd, lang)

{sdmxlib-0.41.0 → sdmxlib-0.41.2}/src/sdmxlib/formats/sdmx_json/data_v2.py

@@ -60,6 +60,7 @@ from sdmxlib.formats.sdmx_json._data_common import (
     decode_observations,
     display_names,
     encode_batch,
+    frequency_durations,
     resolve_columns,
     rows_to_lazyframe,
 )
@@ -137,7 +138,16 @@ def write_sdmx_json_data(
     dim_cols, dim_phys, attr_cols, attr_phys, _measure_id, measure_col = resolve_columns(binding)
 
     time_dim_id = dsd.time_dimension.id if dsd.time_dimension is not None else "TIME_PERIOD"
-    dim_lookup = ComponentTable(dim_cols, dsd, kind="dimension", lang=lang, time_dim_id=time_dim_id)
+    freq_dim_id, freq_overrides = frequency_durations(dsd, lang)
+    dim_lookup = ComponentTable(
+        dim_cols,
+        dsd,
+        kind="dimension",
+        lang=lang,
+        time_dim_id=time_dim_id,
+        freq_dim_id=freq_dim_id,
+        freq_overrides=freq_overrides,
+    )
     attr_lookup = ComponentTable(attr_cols, dsd, kind="attribute", lang=lang)
 
     names = display_names(binding, dsd, lang)