sdmxlib 0.8.0__py3-none-any.whl

sdmxlib/api/session.py

@@ -0,0 +1,124 @@
+"""Session — identity map and artefact construction."""
+
+from typing import Any
+
+from sdmxlib.model.message import StructureMessage
+from sdmxlib.model.urn import SdmxUrn
+
+
+def _version_key(version: str) -> tuple[int, ...]:
+    """Parse a version string into a tuple of ints for correct ordering.
+
+    Falls back to ``(0,)`` if the string is not a valid dotted-integer version.
+    """
+    try:
+        return tuple(int(x) for x in version.split("."))
+    except (ValueError, AttributeError):
+        return (0,)
+
+
+class Session:
+    """Session-scoped identity map for SDMX artefacts.
+
+    Interns artefacts by URN so the same object is returned for the
+    same URN within a session. Also merges successive StructureMessages
+    into the identity map.
+
+    Four separate indices:
+    - ``_map``: versioned URN string → artefact (all ingested)
+    - ``_latest``: ``cls:agency:id`` → artefact (latest version, all)
+    - ``_resolved_map``: versioned URN string → artefact (resolved-only)
+    - ``_resolved_latest``: ``cls:agency:id`` → artefact (latest resolved)
+
+    The resolved indices are populated only when ``ingest(..., resolved=True)``
+    is called. This lets ``get_resolved()`` return a cached object only when
+    it was fetched with ``references=all``, preventing unresolved objects
+    from being returned by named methods or ``resolve()``.
+    """
+
+    def __init__(self) -> None:
+        self._map: dict[str, Any] = {}
+        self._latest: dict[str, Any] = {}
+        self._resolved_map: dict[str, Any] = {}
+        self._resolved_latest: dict[str, Any] = {}
+        self._class_tomsg_field: dict[str, str] = {}
+
+    def ingest(self, msg: StructureMessage, *, resolved: bool = False) -> None:
+        """Add all artefacts from a StructureMessage to the identity map.
+
+        Pass ``resolved=True`` when the message was fetched with
+        ``references=all`` so that ``get_resolved()`` can return the cached
+        object on subsequent calls.
+        """
+        for artefact in msg.all_artefacts():
+            cls_name = type(artefact).__name__
+            package = getattr(type(artefact), "sdmx_package", None)
+            if package is None:
+                continue
+            urn = SdmxUrn.make(
+                type(artefact),
+                package=package,
+                artefact_class=cls_name,
+                agency=artefact.agency_id,
+                id=artefact.id,
+                version=artefact.version,
+            )
+            msg_field = getattr(type(artefact), "msg_field", None)
+            if msg_field and cls_name not in self._class_tomsg_field:
+                self._class_tomsg_field[cls_name] = msg_field
+            key = str(urn)
+            self._map[key] = artefact
+            latest_key = f"{cls_name}:{artefact.agency_id}:{artefact.id}"
+            existing = self._latest.get(latest_key)
+            if existing is None or _version_key(artefact.version) > _version_key(existing.version):
+                self._latest[latest_key] = artefact
+            if resolved:
+                self._resolved_map[key] = artefact
+                existing_r = self._resolved_latest.get(latest_key)
+                if existing_r is None or artefact.version > existing_r.version:
+                    self._resolved_latest[latest_key] = artefact
+
+    def get[T](self, urn: SdmxUrn[T]) -> T | None:
+        """Return a cached artefact by URN, or None if not cached."""
+        hit = self._map.get(str(urn))
+        if hit is not None:
+            return hit  # type: ignore[return-value]
+        if urn.version == "latest":
+            latest_key = f"{urn.artefact_class}:{urn.agency}:{urn.id}"
+            return self._latest.get(latest_key)  # type: ignore[return-value]
+        return None
+
+    def get_resolved[T](self, urn: SdmxUrn[T]) -> T | None:
+        """Return a cached artefact only if it was ingested as resolved."""
+        hit = self._resolved_map.get(str(urn))
+        if hit is not None:
+            return hit  # type: ignore[return-value]
+        if urn.version == "latest":
+            latest_key = f"{urn.artefact_class}:{urn.agency}:{urn.id}"
+            return self._resolved_latest.get(latest_key)  # type: ignore[return-value]
+        return None
+
+    def first_of_class[T](self, artefact_type: type[T]) -> T | None:
+        """Return the first cached artefact whose type matches artefact_type."""
+        for v in self._map.values():
+            if isinstance(v, artefact_type):
+                return v
+        return None
+
+    def all_of_class[T](self, artefact_type: type[T]) -> list[T]:
+        """Return all cached artefacts whose type matches artefact_type."""
+        return [v for v in self._map.values() if isinstance(v, artefact_type)]
+
+    def extract[T](self, msg: StructureMessage, urn: SdmxUrn[T]) -> T | None:
+        """Pull a specific artefact out of a freshly-parsed message by URN.
+
+        Looks in the StructureMessage collection named by the artefact
+        type's ``msg_field`` class attribute.
+        """
+        field = self._class_tomsg_field.get(urn.artefact_class)
+        if field is None:
+            return None
+        collection = getattr(msg, field, None)
+        if collection is None:
+            return None
+        return collection.get(urn)  # type: ignore[return-value]

sdmxlib/formats/__init__.py

@@ -0,0 +1,166 @@
+"""sdmxlib.formats — SDMX wire format readers and writers.
+
+Entry points:
+
+    from sdmxlib.formats import read, parse, to_xml, to_json
+
+    msg = read("path/to/file.xml")
+    msg = read(xml_bytes)
+
+    reg = sl.Registry()
+    read("codelists.xml", registry=reg)
+    read("datastructure.xml", registry=reg)   # cross-file Refs resolve automatically
+
+    xml_bytes = to_xml(msg)
+    json_bytes = to_json(msg)
+"""
+
+from pathlib import Path
+
+from sdmxlib.formats.sdmx_json.reader import read as _read_json
+from sdmxlib.formats.sdmx_json.writer import write as _write_json
+from sdmxlib.formats.sdmx_ml21.namespaces import MES as _MES_21
+from sdmxlib.formats.sdmx_ml21.reader import read as _read_21
+from sdmxlib.formats.sdmx_ml21.writer import write as _write_21
+from sdmxlib.formats.sdmx_ml30.namespaces import MES as _MES_30
+from sdmxlib.formats.sdmx_ml30.reader import read as _read_30
+from sdmxlib.formats.sdmx_ml30.writer import write as _write_30
+from sdmxlib.model.message import StructureMessage
+from sdmxlib.model.registry import Registry
+
+
+def read(
+    source: "str | Path | bytes",
+    *,
+    version: str | None = None,
+    registry: Registry | None = None,
+) -> StructureMessage:
+    """Read an SDMX file (or raw bytes) into a StructureMessage.
+
+    A convenience wrapper over :func:`parse` that accepts a file path in
+    addition to raw bytes. Format is auto-detected when *version* is omitted.
+
+    If *registry* is provided all parsed artefacts are added to it, enabling
+    cross-file Ref resolution when multiple files are loaded into the same
+    registry.
+
+    Args:
+        source: File path (``str`` or ``Path``) or raw bytes (XML or JSON).
+        version: Optional format hint: ``"json"``, ``"2.1"``, or ``"3.0"``.
+            Auto-detected if omitted.
+        registry: Optional :class:`~sdmxlib.model.registry.Registry` to intern
+            artefacts into after parsing.
+
+    Example::
+
+        msg = sl.read("path/to/datastructure.xml")
+
+        reg = sl.Registry()
+        sl.read("codelists.xml", registry=reg)
+        sl.read("datastructure.xml", registry=reg)
+        dsd = reg.get(sl.DataStructure.urn(agency="ESTAT", id="CPI", version="1.0"))
+    """
+    data = Path(source).read_bytes() if isinstance(source, (str, Path)) else source
+    return parse(data, version=version, registry=registry)
+
+
+def parse(
+    data: bytes,
+    *,
+    version: str | None = None,
+    registry: Registry | None = None,
+) -> StructureMessage:
+    """Parse SDMX bytes into a StructureMessage.
+
+    Auto-detects the format (SDMX-JSON 2.0 or SDMX-ML 2.1/3.0) when
+    *version* is not supplied. Cross-references within the message are
+    auto-resolved.
+
+    If *registry* is provided all parsed artefacts are added to it after
+    parsing, interning their Refs and enabling cross-message resolution.
+
+    Args:
+        data: Raw bytes — JSON or XML.
+        version: Optional hint: ``"json"``, ``"2.1"``, or ``"3.0"``.
+            Auto-detected if omitted.
+        registry: Optional Registry to intern artefacts into after parsing.
+
+    Raises:
+        ValueError: If the format cannot be detected or is not supported.
+    """
+    detected = version or _detect_version(data)
+    if detected == "json":
+        return _read_json(data, registry=registry)
+    if detected == "2.1":
+        return _read_21(data, registry=registry)
+    if detected == "3.0":
+        return _read_30(data, registry=registry)
+    msg = f"Unsupported format: {detected!r}"
+    raise ValueError(msg)
+
+
+def serialize(msg: StructureMessage, *, version: str = "2.1", pretty_print: bool = False) -> bytes:
+    """Serialise a StructureMessage to bytes.
+
+    Args:
+        msg: The StructureMessage to serialise.
+        version: Format to write — ``"2.1"``, ``"3.0"`` (SDMX-ML XML) or
+            ``"json"`` (SDMX-JSON 2.0).
+        pretty_print: If True, indent the output.
+
+    Raises:
+        ValueError: If the requested version is not supported.
+    """
+    if version == "json":
+        return _write_json(msg, pretty_print=pretty_print)
+    if version == "2.1":
+        return _write_21(msg, pretty_print=pretty_print)
+    if version == "3.0":
+        return _write_30(msg, pretty_print=pretty_print)
+    err = f"Unsupported format for writing: {version!r}"
+    raise ValueError(err)
+
+
+def to_xml(msg: StructureMessage, *, version: str = "2.1", pretty_print: bool = False) -> bytes:
+    """Serialise a StructureMessage to SDMX-ML XML bytes.
+
+    Args:
+        msg: The StructureMessage to serialise.
+        version: SDMX-ML version — ``"2.1"`` (default) or ``"3.0"``.
+        pretty_print: If True, indent the output.
+
+    Example:
+        xml_bytes = to_xml(msg)
+        xml_bytes = to_xml(msg, version="3.0", pretty_print=True)
+    """
+    return serialize(msg, version=version, pretty_print=pretty_print)
+
+
+def to_json(msg: StructureMessage, *, pretty_print: bool = False) -> bytes:
+    """Serialise a StructureMessage to SDMX-JSON 2.0 bytes.
+
+    Args:
+        msg: The StructureMessage to serialise.
+        pretty_print: If True, indent the output.
+
+    Example:
+        json_bytes = to_json(msg)
+        json_bytes = to_json(msg, pretty_print=True)
+    """
+    return serialize(msg, version="json", pretty_print=pretty_print)
+
+
+def _detect_version(data: bytes) -> str:
+    """Detect the wire format from the first bytes of the payload."""
+    # JSON messages start with '{' (possibly after whitespace)
+    chunk = data[:2048]
+    stripped = chunk.lstrip()
+    if stripped and stripped[0:1] == b"{":
+        return "json"
+    text = chunk.decode("utf-8", errors="replace")
+    if _MES_30 in text:
+        return "3.0"
+    if _MES_21 in text:
+        return "2.1"
+    msg = "Cannot detect format: not JSON and no recognised SDMX-ML namespace"
+    raise ValueError(msg)

sdmxlib/formats/sdmx_csv/__init__.py

@@ -0,0 +1,5 @@
+"""SDMX-CSV data message reader."""
+
+from sdmxlib.formats.sdmx_csv.reader import scan_csv
+
+__all__ = ["scan_csv"]

sdmxlib/formats/sdmx_csv/reader.py

@@ -0,0 +1,147 @@
+"""SDMX-CSV data reader.
+
+Parses SDMX-CSV messages into :class:`~sdmxlib.model.dataset.Dataset` objects.
+
+SDMX-CSV format::
+
+    DATAFLOW,FREQ,REF_AREA,INDICATOR,TIME_PERIOD,OBS_VALUE,OBS_STATUS
+    ESTAT:STS_INPR_M(1.0),M,AT,INDPRO,2024-01,98.5,A
+
+The ``DATAFLOW`` column identifies the dataflow but is not a DSD component.
+It is extracted and then dropped; the remaining columns are parsed using the
+DSD schema.
+"""
+
+import csv
+import io
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import polars as pl
+
+import sdmxlib.polars as slpl
+from sdmxlib.model.dataflow import Dataflow
+from sdmxlib.model.dataset import Dataset
+from sdmxlib.model.ref import Ref
+from sdmxlib.model.urn import SdmxUrn
+
+if TYPE_CHECKING:
+    from sdmxlib.model.datastructure import DataStructure
+
+_DATAFLOW_COL = "DATAFLOW"
+_KEY_COL = "KEY"
+_DATAFLOW_RE = re.compile(r"^([^:]+):([^(]+)\(([^)]+)\)$")
+
+
+def parse_dataflow_id(dataflow_id: str) -> tuple[str, str, str]:
+    """Parse an SDMX-CSV DATAFLOW value into ``(agency, id, version)``.
+
+    Example:
+        parse_dataflow_id("ESTAT:STS_INPR_M(1.0)")  # ("ESTAT", "STS_INPR_M", "1.0")
+    """
+    m = _DATAFLOW_RE.match(dataflow_id)
+    if m is None:
+        msg = f"Cannot parse DATAFLOW value {dataflow_id!r} — expected 'AGENCY:ID(VERSION)'"
+        raise ValueError(msg)
+    return m.group(1), m.group(2), m.group(3)
+
+
+def _peek_dataflow_id(source: "str | Path | bytes") -> str | None:
+    """Extract the DATAFLOW value from the first data row without reading the whole file."""
+    if isinstance(source, (str, Path)):
+        with Path(source).open(newline="", encoding="utf-8") as f:
+            head = f.read(4096)
+    else:
+        head = source[:4096].decode("utf-8", errors="replace")
+
+    reader = csv.reader(io.StringIO(head))
+    try:
+        header = next(reader)
+    except StopIteration:
+        return None
+    try:
+        df_idx = header.index(_DATAFLOW_COL)
+    except ValueError:
+        return None
+    try:
+        first_row = next(reader)
+    except StopIteration:
+        return None
+    return first_row[df_idx] if df_idx < len(first_row) else None
+
+
+def _dataflow_ref(dataflow_str: str) -> "Ref[Dataflow]":
+    """Build an unresolved Ref[Dataflow] from an SDMX-CSV DATAFLOW column value."""
+    agency, id_, version = parse_dataflow_id(dataflow_str)
+    urn = SdmxUrn.make(
+        Dataflow,
+        package="datastructure",
+        artefact_class="Dataflow",
+        agency=agency,
+        id=id_,
+        version=version,
+    )
+    return Ref[Dataflow].unresolved(urn)
+
+
+def scan_csv(
+    source: "str | Path | bytes",
+    structure: "DataStructure",
+    *,
+    infer_schema_length: "int | None" = 0,
+    dataflow: "Ref[Dataflow] | None" = None,
+) -> Dataset:
+    """Parse an SDMX-CSV message into a :class:`~sdmxlib.model.dataset.Dataset`.
+
+    File paths are scanned lazily via ``pl.scan_csv``; bytes (e.g. from an
+    HTTP response) are read eagerly and wrapped in a ``LazyFrame``. Either
+    way the returned ``Dataset.data`` is always a ``pl.LazyFrame``.
+
+    Args:
+        source: File path or raw bytes from an SDMX-CSV response.
+        structure: Fully resolved :class:`~sdmxlib.model.datastructure.DataStructure`.
+                   Codelists should be resolved for ``pl.Enum`` typing.
+        infer_schema_length: Passed to Polars. ``0`` (default) means
+                             schema is fully DSD-driven with no row sampling,
+                             which is required for true streaming on paths.
+        dataflow: Optional resolved :class:`~sdmxlib.model.dataflow.Dataflow` ref.
+                  When ``None`` and a ``DATAFLOW`` column is present, an
+                  unresolved ref is built from the column value.
+
+    Returns:
+        A :class:`~sdmxlib.model.dataset.Dataset` with a lazy ``data`` frame.
+
+    Example:
+        import sdmxlib.polars as slpl
+
+        ds = slpl.scan_csv("data.csv", dsd)
+        df = ds.label().collect()
+    """
+    dataflow_str = _peek_dataflow_id(source)
+    dataflow_ref = dataflow
+    if dataflow_ref is None and dataflow_str is not None:
+        dataflow_ref = _dataflow_ref(dataflow_str)
+
+    schema_overrides = slpl.schema(structure)
+
+    if isinstance(source, (str, Path)):
+        lf = pl.scan_csv(
+            source,
+            schema_overrides=schema_overrides,
+            infer_schema_length=infer_schema_length,
+        )
+    else:
+        lf = pl.read_csv(
+            io.BytesIO(source),
+            schema_overrides=schema_overrides,
+            infer_schema_length=infer_schema_length,
+        ).lazy()
+
+    # Drop metadata columns that are not DSD components.
+    col_names = lf.collect_schema().names()
+    to_drop = [c for c in (_DATAFLOW_COL, _KEY_COL) if c in col_names]
+    if to_drop:
+        lf = lf.drop(to_drop)
+
+    return Dataset(structure=structure, data=lf, dataflow=dataflow_ref)

sdmxlib/formats/sdmx_json/__init__.py

@@ -0,0 +1 @@
+"""sdmxlib.formats.sdmx_json — SDMX-JSON 2.0 structure reader."""