sdmxlib 0.8.0__py3-none-any.whl

sdmxlib/api/registry.py

@@ -0,0 +1,666 @@
+"""RestRegistry — HTTP client for an SDMX REST endpoint."""
+
+from pathlib import Path
+from typing import Self
+
+from sdmxlib.api.client import SdmxClient
+from sdmxlib.api.providers import ProviderInfo
+from sdmxlib.api.query import DataQuery, GetQuery, Query, _KeySpec
+from sdmxlib.api.session import Session
+from sdmxlib.formats import parse as _parse
+from sdmxlib.model.dataflow import Dataflow
+from sdmxlib.model.dataset import Dataset
+from sdmxlib.model.datastructure import DataStructure
+from sdmxlib.model.message import StructureMessage
+from sdmxlib.model.ref import Ref
+from sdmxlib.model.registry import Registry
+from sdmxlib.model.urn import SdmxUrn
+
+SDMX_GLOBAL_REGISTRY = "https://registry.sdmx.org/sdmx/v2"
+
+
+def _serialise_key(structure: DataStructure, spec: _KeySpec) -> str:  # noqa: C901, PLR0912
+    """Serialise a _KeySpec to an SDMX REST positional key string.
+
+    Returns ``spec.raw_key`` unchanged when no slices or measures are set.
+    Otherwise validates dimension ids and builds ``DIM1+DIM2.DIM3....`` format.
+    Unspecified dimensions become empty strings (wildcard). Codes are sorted
+    for stable, deterministic URLs.
+
+    When ``spec.measure_dim_hint`` is set and the DSD has no formal
+    ``MeasureDimension``, ``.measures()`` codes are injected as a regular
+    dimension slice at the hint dimension's positional slot.
+    """
+    if not spec.slices and spec.measures is None:
+        return spec.raw_key
+
+    slice_map: dict[str, list[str]] = {f.id: f.codes for f in spec.slices}
+
+    # Validate all slice dim ids exist in the DSD
+    valid_ids = {d.id for d in structure.dimensions}
+    md = structure.measure_dimension
+    if md is not None:
+        valid_ids.add(md.id)
+    unknown = set(slice_map) - valid_ids
+    if unknown:
+        names = ", ".join(sorted(unknown))
+        msg = f"Dimension(s) not found in DSD: {names}"
+        raise ValueError(msg)
+
+    # Validate hint dim exists when provided
+    if spec.measure_dim_hint is not None and spec.measure_dim_hint not in valid_ids:
+        msg = f"measure_dim {spec.measure_dim_hint!r} not found in DSD"
+        raise ValueError(msg)
+
+    if md is not None:
+        # ── Formal MeasureDimension path ──────────────────────────────────────
+        # Conflict: same dim targeted by both .slice() and .measures()
+        if spec.measures is not None and md.id in slice_map:
+            msg = f"Cannot filter {md.id!r} via both .slice() and .measures() — use one or the other"
+            raise ValueError(msg)
+
+        parts: list[str] = []
+        for dim in structure.dimensions:
+            codes = slice_map.get(dim.id)
+            parts.append("+".join(sorted(codes)) if codes else "")
+
+        # MeasureDimension appended after all regular dims
+        if spec.measures is not None:
+            parts.append("+".join(sorted(spec.measures)))
+        else:
+            codes = slice_map.get(md.id)
+            parts.append("+".join(sorted(codes)) if codes else "")
+
+    else:
+        # ── No formal MeasureDimension — hint path ────────────────────────────
+        # .measures() with a hint injects codes at the hint dim's regular slot
+        effective_slice_map = dict(slice_map)
+        if spec.measures is not None and spec.measure_dim_hint is not None:
+            if spec.measure_dim_hint in effective_slice_map:
+                msg = (
+                    f"Cannot filter {spec.measure_dim_hint!r} via both .slice() and .measures() — use one or the other"
+                )
+                raise ValueError(msg)
+            effective_slice_map[spec.measure_dim_hint] = sorted(spec.measures)
+
+        parts = []
+        for dim in structure.dimensions:
+            codes = effective_slice_map.get(dim.id)
+            parts.append("+".join(sorted(codes)) if codes else "")
+
+    key = ".".join(parts)
+    return key if any(p for p in parts) else "all"
+
+
+class RestRegistry:
+    """Connection to an SDMX REST endpoint.
+
+    Use as a context manager. All artefact access happens within the scope.
+
+    Args:
+        endpoint: Base URL or :class:`~sdmxlib.api.providers.ProviderInfo`.
+                  Pass a ``ProviderInfo`` from :class:`~sdmxlib.api.providers.Provider`
+                  to have the API version set automatically.
+        api: SDMX REST API version — ``"2.1"`` (default) or ``"3.0"``.
+             Ignored when *endpoint* is a ``ProviderInfo``.
+
+    Example:
+        with sl.RestRegistry(sl.Provider.ECB) as reg:
+            dsd = reg.get(sl.DataStructure, agency="ECB", id="ECB_EXR1").resolve()
+
+        with sl.RestRegistry("https://data-api.ecb.europa.eu/service") as reg:
+            for df in reg.query(sl.Dataflow, agency="ECB").browse():
+                print(df.id)
+    """
+
+    def __init__(
+        self,
+        endpoint: "str | ProviderInfo",
+        *,
+        api: str = "2.1",
+        timeout: float = 30.0,
+        headers: dict[str, str] | None = None,
+        **httpx_kwargs: object,
+    ) -> None:
+        if isinstance(endpoint, ProviderInfo):
+            url = endpoint.url
+            api = endpoint.api
+        else:
+            url = endpoint
+        self._client = SdmxClient(
+            url,
+            api=api,
+            timeout=timeout,
+            headers=headers,
+            **httpx_kwargs,
+        )
+        self._session = Session()
+        self._registry = Registry()
+
+    @property
+    def registry(self) -> Registry:
+        """The domain Registry populated by every artefact fetched so far.
+
+        Artefacts from all ``get()``, ``query()``, ``fetch()``, and
+        ``resolve()`` calls accumulate here with interned Refs. Cross-request
+        Ref resolution happens automatically — a codelist fetched in one
+        request resolves dimension Refs in a DSD fetched in another.
+
+        Example:
+            with sl.RestRegistry(...) as rest:
+                dsd = rest.get(sl.DataStructure, agency="ECB", id="ECB_EXR1").resolve()
+                codelists = rest.registry.get_all(sl.Codelist)
+        """
+        return self._registry
+
+    def load(self, source: "str | Path | bytes") -> StructureMessage:
+        """Load artefacts from a local SDMX file into this registry.
+
+        Useful for pre-populating the registry with local files before making
+        network requests, or for mixing local and remote artefacts. All loaded
+        artefacts are treated as fully resolved — no follow-up HTTP requests
+        are made.
+
+        Ref interning applies: artefacts loaded from file resolve Refs in
+        artefacts already present (from earlier ``load()`` or HTTP calls), and
+        vice-versa.
+
+        Args:
+            source: File path (``str`` or ``Path``) or raw bytes.
+
+        Returns:
+            The parsed :class:`~sdmxlib.model.message.StructureMessage`.
+
+        Example::
+
+            with sl.RestRegistry("https://...") as reg:
+                reg.load("local_constraints.xml")  # pre-populate
+                dsd = reg.get(sl.DataStructure, agency="ESTAT", id="CPI").resolve()
+        """
+        data = Path(source).read_bytes() if isinstance(source, (str, Path)) else source
+        msg = _parse(data, registry=self._registry)
+        self._session.ingest(msg, resolved=True)
+        return msg
+
+    def close(self) -> None:
+        """Close the underlying HTTP client."""
+        self._client.close()
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+    # ── Internal helpers ──────────────────────────────────────────────────────
+
+    def _fetch_one[T](
+        self,
+        artefact_type: type[T],
+        agency: str,
+        id: str,  # noqa: A002
+        version: str = "latest",
+        *,
+        detail: str,
+        references: str,
+    ) -> T:
+        """Fetch a single artefact with explicit wire parameters."""
+        resource = getattr(artefact_type, "sdmx_class", artefact_type.__name__)
+        package = getattr(artefact_type, "sdmx_package", resource.lower())
+        urn = SdmxUrn.make(
+            artefact_type,
+            package=package,
+            artefact_class=resource,
+            agency=agency,
+            id=id,
+            version=version,
+        )
+        resolved = references == "descendants"
+        if resolved:
+            cached = self._session.get_resolved(urn)
+            if cached is not None:
+                return cached
+        elif references == "none":
+            cached = self._session.get(urn)
+            if cached is not None:
+                return cached
+        msg = self._client.get_artefact(resource, agency, id, version, detail=detail, references=references)
+        self._session.ingest(msg, resolved=resolved)
+        self._registry.add_all(msg.all_artefacts())
+        result = self._session.extract(msg, urn)
+        if result is None:
+            err = f"{resource} {agency}:{id}({version}) not found in response"
+            raise KeyError(err)
+        return result
+
+    def _fetch_all_with[T](
+        self,
+        artefact_type: type[T],
+        agency: str | None,
+        *,
+        detail: str,
+        references: str,
+    ) -> list[T]:
+        """Fetch all artefacts of a type with explicit wire parameters."""
+        resource = getattr(artefact_type, "sdmx_class", artefact_type.__name__)
+        resolved = references == "descendants"
+        msg = self._client.get_all(resource, agency, detail=detail, references=references)
+        self._session.ingest(msg, resolved=resolved)
+        self._registry.add_all(msg.all_artefacts())
+        return self._session.all_of_class(artefact_type)
+
+    # ── Generic entry points ──────────────────────────────────────────────────
+
+    def get[T](
+        self,
+        artefact_type: type[T],
+        *,
+        agency: str,
+        id: str,  # noqa: A002
+        version: str = "latest",
+    ) -> GetQuery[T]:
+        """Point lookup for a single artefact. Returns a GetQuery.
+
+        Call ``.fetch()`` for a skeleton or ``.resolve()`` to follow references.
+
+        Example:
+            reg.get(sl.DataStructure, agency="ECB", id="ECB_EXR1").fetch()
+            reg.get(sl.DataStructure, agency="ECB", id="ECB_EXR1").resolve()
+            reg.get(sl.DataStructure, agency="ECB", id="ECB_EXR1").resolve(sl.Codelist)
+            reg.get(sl.DataStructure, agency="ECB", id="ECB_EXR1").resolve(partial=False)
+        """
+        return GetQuery(
+            fetcher=lambda: self._fetch_one(
+                artefact_type,
+                agency,
+                id,
+                version,
+                detail="full",
+                references="none",
+            ),
+            resolver=lambda refs, partial: self._fetch_one(
+                artefact_type,
+                agency,
+                id,
+                version,
+                detail="referencepartial" if partial else "full",
+                references=refs,
+            ),
+        )
+
+    def query[T](self, artefact_type: type[T], *, agency: str | None = None) -> Query[T]:
+        """Browse all artefacts of a type. Returns a chainable Query.
+
+        Example:
+            reg.query(sl.Dataflow, agency="ECB").browse()
+            reg.query(sl.Dataflow, agency="ECB").filter(...).fetch()
+            reg.query(sl.Codelist).sort_by("id").resolve()
+            reg.query(sl.Dataflow, agency="ECB").resolve(sl.DataStructure)
+        """
+        return Query(
+            fetcher=lambda: self._fetch_all_with(
+                artefact_type,
+                agency,
+                detail="full",
+                references="none",
+            ),
+            resolver=lambda refs, partial: self._fetch_all_with(
+                artefact_type,
+                agency,
+                detail="referencepartial" if partial else "full",
+                references=refs,
+            ),
+            browser=lambda: self._fetch_all_with(
+                artefact_type,
+                agency,
+                detail="allstubs",
+                references="none",
+            ),
+        )
+
+    # ── URN / Ref-based access ────────────────────────────────────────────────
+
+    def fetch[T](self, urn: SdmxUrn[T]) -> T:
+        """Fetch an artefact by URN with unresolved Refs."""
+        cached = self._session.get(urn)
+        if cached is not None:
+            return cached
+        msg = self._client.get_artefact(
+            urn.artefact_class,
+            urn.agency,
+            urn.id,
+            urn.version,
+            detail="full",
+            references="none",
+        )
+        self._session.ingest(msg)
+        self._registry.add_all(msg.all_artefacts())
+        result = self._session.extract(msg, urn)
+        if result is None:
+            err = f"{urn} not found in response"
+            raise KeyError(err)
+        return result
+
+    def resolve[T](self, target: SdmxUrn[T] | Ref[T], *, partial: bool = True) -> T:
+        """Fetch an artefact by URN or Ref with all Refs resolved.
+
+        Args:
+            target: A ``SdmxUrn[T]`` or ``Ref[T]`` from graph traversal.
+            partial: If ``True`` (default), referenced ItemSchemes are trimmed
+                     by ContentConstraints. Set to ``False`` for full audits.
+
+        Example:
+            reg.resolve(sl.Codelist.urn(agency="ECB", id="CL_CURRENCY"))
+            reg.resolve(dsd.dimensions["FREQ"].representation)
+            reg.resolve(dsd.dimensions["FREQ"].representation, partial=False)
+        """
+        urn: SdmxUrn[T] = target.urn if isinstance(target, Ref) else target
+        cached = self._session.get_resolved(urn)
+        if cached is not None:
+            return cached
+        detail = "referencepartial" if partial else "full"
+        msg = self._client.get_artefact(
+            urn.artefact_class,
+            urn.agency,
+            urn.id,
+            urn.version,
+            detail=detail,
+            references="descendants",
+        )
+        self._session.ingest(msg, resolved=True)
+        self._registry.add_all(msg.all_artefacts())
+        result = self._session.extract(msg, urn)
+        if result is None:
+            err = f"{urn} not found in response"
+            raise KeyError(err)
+        return result
+
+    # ── Data fetching ─────────────────────────────────────────────────────────
+
+    def data(  # noqa: C901
+        self,
+        dataflow: "type[Dataflow] | Dataflow",
+        *,
+        agency: str | None = None,
+        id: str | None = None,  # noqa: A002
+        version: str = "latest",
+        key: str = "all",
+        start_period: str | None = None,
+        end_period: str | None = None,
+        measure_dim: str | None = None,
+    ) -> DataQuery:
+        """Start a data query for observation data. Returns a :class:`DataQuery`.
+
+        Pass the :class:`~sdmxlib.model.dataflow.Dataflow` *type* with
+        ``agency`` and ``id`` to mirror :meth:`get` — the registry resolves
+        the structure internally on :meth:`~DataQuery.fetch`:
+
+        .. code-block:: python
+
+            reg.data(sl.Dataflow, agency="ECB", id="EXR", key="M.USD.EUR.SP00.A").fetch()
+
+        Or pass an already-resolved :class:`~sdmxlib.model.dataflow.Dataflow`
+        instance (e.g. from a previous :meth:`query`) to skip the extra fetch:
+
+        .. code-block:: python
+
+            reg.data(flow, key="M.USD.EUR.SP00.A").fetch()
+
+        Args:
+            dataflow: The :class:`~sdmxlib.model.dataflow.Dataflow` class or a
+                      resolved instance.
+            agency: Maintainer agency id. Required when passing the type.
+            id: Dataflow id. Required when passing the type.
+            version: Dataflow version. Defaults to ``"latest"``.
+            key: SDMX key filter string, e.g. ``"M.USD.EUR.SP00.A"``.
+            start_period: ISO period start, e.g. ``"2020-01"``.
+            end_period: ISO period end.
+            measure_dim: Dimension id to treat as the measure axis when the DSD
+                has no formal ``<MeasureDimension>``. Enables :meth:`~DataQuery.measures`
+                filtering and :meth:`~sdmxlib.model.dataset.Dataset.pivot` on
+                providers that use a regular dimension (e.g. ``MEASURE``,
+                ``INDICATOR``, ``SUBJECT``) instead of the SDMX 2.1 construct.
+                If absent and no ``<MeasureDimension>`` exists, ``pivot()``
+                raises with a list of candidate enumerated dimensions.
+        """
+        if isinstance(dataflow, type):
+            if agency is None or id is None:
+                msg = "agency and id are required when passing the Dataflow type"
+                raise ValueError(msg)
+            _agency, _id, _version = agency, id, version
+
+            def _resolve_flow() -> "tuple[DataStructure, Dataflow]":
+                flow = self._fetch_one(
+                    Dataflow,
+                    _agency,
+                    _id,
+                    _version,
+                    detail="referencepartial",
+                    references="descendants",
+                )
+                if flow.structure is None:
+                    msg = f"Dataflow {flow.id!r} has no structure reference"
+                    raise ValueError(msg)
+                return flow.structure(), flow
+
+            def _fetcher(spec: _KeySpec) -> Dataset:
+                structure, flow = _resolve_flow()
+                ds = self._fetch_data(
+                    _agency,
+                    _id,
+                    flow.version,  # use resolved version — providers may reject "latest" for data
+                    structure,
+                    key=_serialise_key(structure, spec),
+                    start_period=start_period,
+                    end_period=end_period,
+                    dataflow=flow,
+                )
+                if measure_dim is not None:
+                    ds.measure_dim_hint = measure_dim
+                return ds
+
+            def _streamer(path: "Path", spec: _KeySpec) -> Dataset:
+                structure, flow = _resolve_flow()
+                ds = self._fetch_data_to(
+                    path,
+                    _agency,
+                    _id,
+                    flow.version,  # use resolved version — providers may reject "latest" for data
+                    structure,
+                    key=_serialise_key(structure, spec),
+                    start_period=start_period,
+                    end_period=end_period,
+                    dataflow=flow,
+                )
+                if measure_dim is not None:
+                    ds.measure_dim_hint = measure_dim
+                return ds
+        else:
+            if dataflow.structure is None:
+                msg = f"Dataflow {dataflow.id!r} has no structure reference"
+                raise ValueError(msg)
+            _agency = dataflow.agency_id
+            _id = dataflow.id
+            _version = dataflow.version
+            structure: DataStructure = dataflow.structure()
+
+            def _fetcher(spec: _KeySpec) -> Dataset:  # type: ignore[misc]
+                ds = self._fetch_data(
+                    _agency,
+                    _id,
+                    _version,
+                    structure,
+                    key=_serialise_key(structure, spec),
+                    start_period=start_period,
+                    end_period=end_period,
+                    dataflow=dataflow,
+                )
+                if measure_dim is not None:
+                    ds.measure_dim_hint = measure_dim
+                return ds
+
+            def _streamer(path: "Path", spec: _KeySpec) -> Dataset:  # type: ignore[misc]
+                ds = self._fetch_data_to(
+                    path,
+                    _agency,
+                    _id,
+                    _version,
+                    structure,
+                    key=_serialise_key(structure, spec),
+                    start_period=start_period,
+                    end_period=end_period,
+                    dataflow=dataflow,
+                )
+                if measure_dim is not None:
+                    ds.measure_dim_hint = measure_dim
+                return ds
+
+        return DataQuery(fetcher=_fetcher, streamer=_streamer, raw_key=key, measure_dim=measure_dim)
+
+    def _fetch_data(
+        self,
+        agency: str,
+        id: str,  # noqa: A002
+        version: str,
+        structure: DataStructure,
+        *,
+        key: str,
+        start_period: str | None,
+        end_period: str | None,
+        dataflow: "Dataflow | None" = None,
+    ) -> Dataset:
+        """Fetch observation data and return a Dataset."""
+        from sdmxlib.formats.sdmx_csv.reader import scan_csv  # noqa: PLC0415
+
+        csv_bytes = self._client.get_data(
+            agency,
+            id,
+            version,
+            key=key,
+            start_period=start_period,
+            end_period=end_period,
+        )
+        dataflow_ref = Ref.to(dataflow) if dataflow is not None else None
+        return scan_csv(csv_bytes, structure, dataflow=dataflow_ref)
+
+    def _fetch_data_to(
+        self,
+        path: Path,
+        agency: str,
+        id: str,  # noqa: A002
+        version: str,
+        structure: DataStructure,
+        *,
+        key: str,
+        start_period: str | None,
+        end_period: str | None,
+        dataflow: "Dataflow | None" = None,
+    ) -> Dataset:
+        """Stream observation data to a file and return a file-backed Dataset."""
+        from sdmxlib.formats.sdmx_csv.reader import scan_csv  # noqa: PLC0415
+
+        self._client.stream_data_to(
+            agency,
+            id,
+            version,
+            path=path,
+            key=key,
+            start_period=start_period,
+            end_period=end_period,
+        )
+        dataflow_ref = Ref.to(dataflow) if dataflow is not None else None
+        return scan_csv(path, structure, dataflow=dataflow_ref)
+
+    def scan_csv(self, source: "str | Path | bytes") -> Dataset:
+        """Load an SDMX-CSV file or bytes, auto-resolving the structure via this registry.
+
+        Reads the ``DATAFLOW`` column to identify the dataflow, fetches and
+        resolves the :class:`~sdmxlib.model.datastructure.DataStructure` if it
+        is not already cached, then returns a lazy
+        :class:`~sdmxlib.model.dataset.Dataset`.
+
+        This is the preferred entry point when you have a local SDMX-CSV file:
+        the registry owns resolution, so you do not need to fetch the DSD
+        separately.
+
+        Args:
+            source: File path or raw SDMX-CSV bytes.
+
+        Raises:
+            ValueError: If the ``DATAFLOW`` column is absent or unparseable.
+
+        Example:
+            with sl.RestRegistry(sl.Provider.ECB) as reg:
+                ds = reg.scan_csv("ecb_exr.csv")
+                df = ds.label().collect()
+        """
+        from sdmxlib.formats.sdmx_csv.reader import _peek_dataflow_id, parse_dataflow_id, scan_csv  # noqa: PLC0415
+
+        dataflow_str = _peek_dataflow_id(source)
+        if dataflow_str is None:
+            msg = (
+                "DATAFLOW column not found in CSV — cannot auto-resolve structure. "
+                "Pass the DataStructure explicitly to slpl.scan_csv()."
+            )
+            raise ValueError(msg)
+
+        agency, id_, version = parse_dataflow_id(dataflow_str)
+        flow = self._fetch_one(
+            Dataflow,
+            agency,
+            id_,
+            version,
+            detail="referencepartial",
+            references="descendants",
+        )
+
+        if flow.structure is None:
+            msg = f"Dataflow {flow.id!r} has no structure reference"
+            raise ValueError(msg)
+
+        structure: DataStructure = flow.structure()
+        return scan_csv(source, structure, dataflow=Ref.to(flow))
+
+
+# ── FmrRegistry ───────────────────────────────────────────────────────────────
+
+
+class BearerToken:
+    """Bearer token for FMR authentication."""
+
+    def __init__(self, token: str) -> None:
+        self.token = token
+
+
+class FmrRegistry(RestRegistry):
+    """Registry configured for Fusion Metadata Registry endpoints.
+
+    Defaults to the SDMX Global Registry.
+
+    Example:
+        with sl.FmrRegistry() as reg:
+            ...
+        with sl.FmrRegistry("https://my-fmr.example.com/sdmxapi/rest") as reg:
+            ...
+        with sl.FmrRegistry(auth=sl.BearerToken("eyJ...")) as reg:
+            ...
+        with sl.FmrRegistry(auth=("user", "pass")) as reg:
+            ...
+    """
+
+    def __init__(
+        self,
+        endpoint: str = SDMX_GLOBAL_REGISTRY,
+        *,
+        auth: "tuple[str, str] | BearerToken | None" = None,
+        timeout: float = 30.0,
+        **kwargs: object,
+    ) -> None:
+        headers: dict[str, str] = {}
+        httpx_kwargs: dict[str, object] = dict(kwargs)
+        if isinstance(auth, BearerToken):
+            headers["Authorization"] = f"Bearer {auth.token}"
+        elif isinstance(auth, tuple):
+            httpx_kwargs["auth"] = auth
+        # The SDMX Global Registry currently serves 2.1 format responses.
+        super().__init__(endpoint, api="2.1", timeout=timeout, headers=headers, **httpx_kwargs)