pysdmx 1.3.0__py3-none-any.whl → 1.4.0rc1__py3-none-any.whl

pysdmx/__extras_check.py

@@ -50,7 +50,8 @@ def __check_vtl_extra() -> None:
     except ImportError:
         raise ImportError(
             ERROR_MESSAGE.format(
-                extra_name="vtlengine",
-                extra_desc="validations for VTL engine",
+                extra_name="vtl",
+                extra_desc="VTL Scripts, SDMX-VTL model validations"
+                " and prettify",
             )
         ) from None

pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.3.0"
+__version__ = "1.4.0rc1"

pysdmx/api/fmr/__init__.py

@@ -642,7 +642,7 @@ class RegistryClient(__BaseRegistryClient):
         """
         query = super()._report_q(provider, id, version)
         out = self.__fetch(query)
-        return super()._out(out, self.deser.report)[0]
+        return super()._out(out, self.deser.report).reports[0]
 
     def get_reports(
         self,
@@ -667,7 +667,7 @@ class RegistryClient(__BaseRegistryClient):
         """
         query = super()._reports_q(artefact_type, agency, id, version)
         out = self.__fetch(query)
-        return super()._out(out, self.deser.report, True)
+        return super()._out(out, self.deser.report).reports
 
     def get_mapping(
         self,
@@ -1104,7 +1104,7 @@ class AsyncRegistryClient(__BaseRegistryClient):
         """
         query = super()._report_q(provider, id, version)
         out = await self.__fetch(query)
-        return super()._out(out, self.deser.report)[0]
+        return super()._out(out, self.deser.report).reports[0]
 
     async def get_reports(
         self,
@@ -1129,7 +1129,7 @@ class AsyncRegistryClient(__BaseRegistryClient):
         """
         query = super()._reports_q(artefact_type, agency, id, version)
         out = await self.__fetch(query)
-        return super()._out(out, self.deser.report, True)
+        return super()._out(out, self.deser.report).reports
 
     async def get_mapping(
         self,

pysdmx/api/gds/__init__.py

@@ -0,0 +1,328 @@
+"""API client for interacting with the GDS (Global Discovery Service) service.
+
+This module provides classes and utilities to interact with the GDS service,
+allowing retrieval of metadata such as agency information in SDMX-JSON format.
+
+Exports: GdsClient: A synchronous client for retrieving metadata from
+the GDS.
+"""
+
+from typing import Any, Literal, Optional, Sequence
+
+from msgspec.json import decode
+
+from pysdmx.api.qb.gds import GdsQuery, GdsType
+from pysdmx.api.qb.service import GdsAsyncRestService, GdsRestService
+from pysdmx.api.qb.util import REST_ALL
+from pysdmx.io.json.gds.reader import deserializers as gds_readers
+from pysdmx.io.serde import Deserializer
+from pysdmx.model import Agency
+from pysdmx.model.gds import (
+    GdsCatalog,
+    GdsSdmxApi,
+    GdsService,
+    GdsUrnResolver,
+)
+
+GDS_BASE_ENDPOINT = "https://gds.sdmx.io/"
+
+READERS = gds_readers
+
+
+class __BaseGdsClient:
+    def __init__(
+        self,
+        api_endpoint: str = GDS_BASE_ENDPOINT,
+        pem: Optional[str] = None,
+    ):
+        """Instantiate a new client against the target endpoint."""
+        if api_endpoint.endswith("/"):
+            api_endpoint = api_endpoint[0:-1]
+        self.api_endpoint = api_endpoint
+        self.reader = READERS
+
+    def _out(self, response: bytes, typ: Deserializer, *params: Any) -> Any:
+        return decode(response, type=typ).to_model(*params)
+
+    def _agencies_q(self, agency: str) -> GdsQuery:
+        return GdsQuery(artefact_type=GdsType.GDS_AGENCY, resource_id=agency)
+
+    def _catalogs_q(
+        self,
+        agency: str,
+        resource: str = REST_ALL,
+        version: str = REST_ALL,
+        resource_type: Optional[Literal["data", "metadata"]] = None,
+        message_format: Optional[Literal["json", "csv", "xml"]] = None,
+        api_version: Optional[str] = None,
+        detail: Optional[Literal["full", "raw"]] = None,
+        references: Optional[Literal["none", "children"]] = None,
+    ) -> GdsQuery:
+        return GdsQuery(
+            artefact_type=GdsType.GDS_CATALOG,
+            agency=agency,
+            resource_id=resource,
+            version=version,
+            resource_type=resource_type,
+            message_format=message_format,
+            api_version=api_version,
+            detail=detail,
+            references=references,
+        )
+
+    def _sdmx_api_q(self, id: str = REST_ALL) -> GdsQuery:
+        return GdsQuery(artefact_type=GdsType.GDS_SDMX_API, resource_id=id)
+
+    def _services_q(
+        self, agency: str, resource: str = REST_ALL, version: str = REST_ALL
+    ) -> GdsQuery:
+        return GdsQuery(
+            artefact_type=GdsType.GDS_SERVICE,
+            agency=agency,
+            resource_id=resource,
+            version=version,
+        )
+
+    def _urn_resolver_q(self, urn: str) -> GdsQuery:
+        return GdsQuery(
+            artefact_type=GdsType.GDS_URN_RESOLVER, resource_id=urn
+        )
+
+
+class GdsClient(__BaseGdsClient):
+    """A client to be used to retrieve metadata from the GDS.
+
+    With this client, metadata will be retrieved in a synchronous fashion.
+    """
+
+    def __init__(
+        self,
+        api_endpoint: str = GDS_BASE_ENDPOINT,
+        pem: Optional[str] = None,
+    ) -> None:
+        """Instantiate a new client against the target endpoint.
+
+        Args:
+            api_endpoint: The endpoint of the targeted service.
+            pem: In case the service exposed a certificate created
+                by an unknown certificate authority, you can pass
+                a pem file for this authority using this parameter.
+        """
+        super().__init__(api_endpoint, pem)
+        self.__service = GdsRestService(
+            self.api_endpoint,
+            pem=pem,
+            timeout=10.0,
+        )
+
+    def __fetch(
+        self,
+        query: GdsQuery,
+    ) -> bytes:
+        """Fetch the requested metadata from the GDS service."""
+        return self.__service.gds(query)
+
+    def get_agencies(self, agency: str) -> Sequence[Agency]:
+        """Get the list of agencies for the supplied name.
+
+        Args:
+            agency: The agency maintaining the agency scheme from
+                which sub-agencies must be returned.
+
+        Returns:
+            The requested list of agencies.
+        """
+        query = super()._agencies_q(agency)
+        out = self.__fetch(query)
+        agencies = super()._out(out, self.reader.agencies)
+        return agencies
+
+    def get_catalogs(
+        self,
+        catalog: str,
+        resource: str = REST_ALL,
+        version: str = REST_ALL,
+        resource_type: Optional[Literal["data", "metadata"]] = None,
+        message_format: Optional[Literal["json", "csv", "xml"]] = None,
+        api_version: Optional[str] = None,
+        detail: Optional[Literal["full", "raw"]] = None,
+        references: Optional[Literal["none", "children"]] = None,
+    ) -> Sequence[GdsCatalog]:
+        """Get the list of catalogs for the supplied parameters.
+
+        Args:
+            catalog: The agency maintaining the catalog.
+            resource: The resource ID(s) to query. Defaults to '*'.
+            version: The version(s) of the resource. Defaults to '*'.
+            resource_type: Filters the endpoints that support
+             the requested resource type (eg, 'data', 'metadata')
+            message_format: Filters the endpoints that support any
+              of the requested message formats.
+            api_version: Filters the endpoints that is in a
+              specific SDMX API version.
+              Multiple values separated by commas are possible.
+              By default (if nothing is sent) it returns everything.
+            detail: The amount of information to be returned.
+              If detail=full: All available information for all artefacts
+              should be returned.
+              If detail=raw: Any nested service will be referenced.
+            references: Instructs the web service to return
+              (or not) the artefacts referenced by the
+              artefact to be returned.
+              If references=none: No referenced artefacts will be returned.
+              If references=children: Returns the artefacts
+              referenced by the artefact to be returned.
+
+        Returns:
+            A list of GdsCatalog objects.
+        """
+        query = super()._catalogs_q(
+            catalog,
+            resource,
+            version,
+            resource_type,
+            message_format,
+            api_version,
+            detail,
+            references,
+        )
+        response = self.__fetch(query)
+        catalogs = super()._out(response, self.reader.catalogs)
+        return catalogs
+
+    def get_sdmx_apis(
+        self, api_version: str = REST_ALL
+    ) -> Sequence[GdsSdmxApi]:
+        """Get the list of SDMX API versions.
+
+        Args:
+            api_version: The version of the SDMX API to be returned.
+              Defaults to '*'.
+        """
+        query = super()._sdmx_api_q(api_version)
+        response = self.__fetch(query)
+        sdmx_api = super()._out(response, self.reader.sdmx_api)
+        return sdmx_api
+
+    def get_services(
+        self, service: str, resource: str = REST_ALL, version: str = REST_ALL
+    ) -> Sequence[GdsService]:
+        """Get the list of services for the supplied parameters.
+
+        Args:
+            service: The agency maintaining the service.
+            resource: The resource ID(s) to query. Defaults to '*'.
+            version: The version(s) of the resource. Defaults to '*'.
+
+        Returns:
+            A list of GdsService objects.
+        """
+        query = super()._services_q(service, resource, version)
+        response = self.__fetch(query)
+        services = super()._out(response, self.reader.services)
+        return services
+
+    def get_urn_resolver(self, urn: str) -> GdsUrnResolver:
+        """Resolve a URN to its corresponding resource.
+
+        Args:
+            urn: The URN to resolve.
+
+        Returns:
+            A GdsUrnResolver object with the resolved information.
+        """
+        query = super()._urn_resolver_q(urn)
+        response = self.__fetch(query)
+        urn_resolution = super()._out(response, self.reader.urn_resolver)
+        return urn_resolution
+
+
+class AsyncGdsClient(__BaseGdsClient):
+    """A client to be used to retrieve metadata from the GDS asynchronously."""
+
+    def __init__(
+        self,
+        api_endpoint: str = GDS_BASE_ENDPOINT,
+        pem: Optional[str] = None,
+    ) -> None:
+        """Instantiate a new client against the target endpoint.
+
+        Args:
+            api_endpoint: The endpoint of the targeted service.
+            pem: PEM file for unknown certificate authorities.
+        """
+        super().__init__(api_endpoint, pem)
+        self.__service = GdsAsyncRestService(
+            self.api_endpoint,
+            pem=pem,
+            timeout=10.0,
+        )
+
+    async def __fetch(self, query: GdsQuery) -> bytes:
+        """Fetch the requested metadata from the GDS service asynchronously."""
+        return await self.__service.gds(query)
+
+    async def get_agencies(self, agency: str) -> Sequence[Agency]:
+        """Get the list of agencies for the supplied name asynchronously.
+
+        Args:
+            agency: The agency maintaining the agency scheme.
+
+        Returns:
+            The requested list of agencies.
+        """
+        query = super(AsyncGdsClient, self)._agencies_q(agency)
+        out = await self.__fetch(query)
+        agencies = super(AsyncGdsClient, self)._out(out, self.reader.agencies)
+        return agencies
+
+    async def get_catalogs(
+        self,
+        catalog: str,
+        resource: str = REST_ALL,
+        version: str = REST_ALL,
+        resource_type: Optional[Literal["data", "metadata"]] = None,
+        message_format: Optional[Literal["json", "csv", "xml"]] = None,
+        api_version: Optional[str] = None,
+        detail: Optional[Literal["full", "raw"]] = None,
+        references: Optional[Literal["none", "children"]] = None,
+    ) -> Sequence[GdsCatalog]:
+        """Get the list of catalogs for the supplied params asynchronously."""
+        query = super()._catalogs_q(
+            catalog,
+            resource,
+            version,
+            resource_type,
+            message_format,
+            api_version,
+            detail,
+            references,
+        )
+        response = await self.__fetch(query)
+        catalogs = super()._out(response, self.reader.catalogs)
+        return catalogs
+
+    async def get_sdmx_apis(
+        self, api_version: str = REST_ALL
+    ) -> Sequence[GdsSdmxApi]:
+        """Get the list of SDMX API versions asynchronously."""
+        query = super()._sdmx_api_q(api_version)
+        response = await self.__fetch(query)
+        sdmx_api = super()._out(response, self.reader.sdmx_api)
+        return sdmx_api
+
+    async def get_services(
+        self, service: str, resource: str = REST_ALL, version: str = REST_ALL
+    ) -> Sequence[GdsService]:
+        """Get a list of services for the supplied params asynchronously."""
+        query = super()._services_q(service, resource, version)
+        response = await self.__fetch(query)
+        services = super()._out(response, self.reader.services)
+        return services
+
+    async def get_urn_resolver(self, urn: str) -> GdsUrnResolver:
+        """Resolve a URN to its corresponding resource asynchronously."""
+        query = super()._urn_resolver_q(urn)
+        response = await self.__fetch(query)
+        urn_resolution = super()._out(response, self.reader.urn_resolver)
+        return urn_resolution

pysdmx/api/qb/gds.py

@@ -0,0 +1,153 @@
+"""Build GDS-REST structure queries."""
+
+from enum import Enum
+from typing import Literal, Optional, Sequence, Union
+
+import msgspec
+
+from pysdmx.api.qb.util import (
+    REST_ALL,
+    REST_LATEST,
+)
+from pysdmx.errors import Invalid
+
+
+class GdsType(Enum):
+    """The type of GDS metadata to be returned."""
+
+    GDS_AGENCY = "agency"
+    GDS_CATALOG = "catalog"
+    GDS_SDMX_API = "sdmxapi"
+    GDS_SERVICE = "service"
+    GDS_URN_RESOLVER = "urn_resolver"
+    ALL = REST_ALL
+    LATEST = REST_LATEST
+
+
+_RESOURCES = {
+    GdsType.GDS_AGENCY,
+    GdsType.GDS_CATALOG,
+    GdsType.GDS_SDMX_API,
+    GdsType.GDS_SERVICE,
+    GdsType.GDS_URN_RESOLVER,
+}
+
+
+class GdsQuery(msgspec.Struct, frozen=True, omit_defaults=True):
+    """A query for base GDS metadata.
+
+    Attributes:
+        artefact_type: The type of GDS metadata to be returned.
+        agency: The agency (or agencies) maintaining the artefact(s)
+            to be returned.
+        resource_id: The resource ID(s) to query. Defaults to '*'.
+        version: The version(s) of the resource. Defaults to '*'.
+        resource_type: Filters the endpoints that support
+          the requested resource type (eg, 'data', 'metadata').
+        message_format: Filters the endpoints that support any
+          of the requested message formats.
+
+        api_version: Filters the endpoints that is in a
+          specific SDMX API version.
+          Multiple values separated by commas are possible.
+          By default (if nothing is sent) it returns everything.
+
+        detail: The amount of information to be returned.
+            Option full: All available information for all artefacts
+              should be returned.
+            Option raw: Any nested service will be referenced.
+
+        references: Instructs the web service to return
+          (or not) the artefacts referenced by the
+          artefact to be returned.
+
+            Option none: No referenced artefacts will be returned.
+            Option children: Returns the artefacts
+              referenced by the artefact to be returned.
+    """
+
+    artefact_type: GdsType
+    agency: Union[str, Sequence[str]] = REST_ALL
+    resource_id: Union[str, Sequence[str]] = REST_ALL
+    version: Optional[str] = None
+    resource_type: Optional[Literal["data", "metadata"]] = None
+    message_format: Optional[Literal["json", "csv", "xml"]] = None
+    api_version: Optional[str] = None
+    detail: Optional[Literal["full", "raw"]] = None
+    references: Optional[Literal["none", "children"]] = None
+
+    def validate(self) -> None:
+        """Validate the query."""
+        try:
+            decoder.decode(encoder.encode(self))
+        except msgspec.DecodeError as err:
+            raise Invalid("Invalid Structure Query", str(err)) from err
+
+    def _get_base_url(self) -> str:
+        """The URL for the query in the GDS-REST API."""
+        self.__validate_query()
+        return self.__create_query()
+
+    def get_url(self) -> str:
+        """The URL for the query in the GDS-REST API."""
+        base_url = self._get_base_url()
+        params = []
+        if self.resource_type:
+            params.append(f"resource_type={self.resource_type}")
+        if self.message_format:
+            params.append(f"message_format={self.message_format}")
+        if self.api_version:
+            params.append(f"api_version={self.api_version}")
+        if self.detail:
+            params.append(f"detail={self.detail}")
+        if self.references:
+            params.append(f"references={self.references}")
+        query_string = f"/?{'&'.join(params)}" if params else ""
+        return f"{base_url}{query_string}"
+
+    def __validate_query(self) -> None:
+        self.validate()
+        self.__check_artefact_type(self.artefact_type)
+
+    def __check_artefact_type(self, atyp: GdsType) -> None:
+        if atyp not in _RESOURCES:
+            raise Invalid(
+                "Validation Error",
+                f"{atyp} is not valid for GDS-REST.",
+            )
+
+    def __to_type_kw(self, val: GdsType) -> str:
+        return val.value
+
+    def __to_kw(self, val: str) -> str:
+        return val
+
+    def __to_kws(self, vals: Union[str, Sequence[str]]) -> str:
+        vals = [vals] if isinstance(vals, str) else vals
+        mapped = [self.__to_kw(v) for v in vals]
+        sep = ","
+        return sep.join(mapped)
+
+    def __create_query(self) -> str:
+        t = self.__to_type_kw(self.artefact_type)
+        a = self.__to_kws(self.agency)
+        r = self.__to_kws(self.resource_id)
+        v = self.__to_kws(self.version) if self.version else REST_ALL
+
+        if t in ("sdmxapi", "agency", "urn_resolver"):
+            return f"/{t}/{r}"
+
+        vu = f"/{v}" if v != REST_ALL else ""
+        ru = f"/{r}{vu}" if vu or self.resource_id != REST_ALL else ""
+        au = f"/{a}{ru}" if ru or self.agency != REST_ALL else ""
+
+        return f"/{t}{au}"
+
+
+decoder = msgspec.json.Decoder(GdsQuery)
+encoder = msgspec.json.Encoder()
+
+
+__all__ = [
+    "GdsQuery",
+]

pysdmx/api/qb/service.py

@@ -1,4 +1,4 @@
-"""Connector to SDMX-REST services."""
+"""Connector to SDMX-REST and GDS-REST services."""
 
 from typing import NoReturn, Optional, Union
 
@@ -7,6 +7,7 @@ import httpx
 from pysdmx import errors
 from pysdmx.api.qb.availability import AvailabilityFormat, AvailabilityQuery
 from pysdmx.api.qb.data import DataFormat, DataQuery
+from pysdmx.api.qb.gds import GdsQuery
 from pysdmx.api.qb.refmeta import (
     RefMetaByMetadataflowQuery,
     RefMetaByMetadatasetQuery,
@@ -22,6 +23,7 @@ from pysdmx.api.qb.registration import (
 from pysdmx.api.qb.schema import SchemaFormat, SchemaQuery
 from pysdmx.api.qb.structure import StructureFormat, StructureQuery
 from pysdmx.api.qb.util import ApiVersion
+from pysdmx.io.format import GDS_FORMAT
 
 
 class _CoreRestService:
@@ -100,7 +102,7 @@ class _CoreRestService:
 class RestService(_CoreRestService):
     """Synchronous connector to SDMX-REST services.
 
-    Attributes:
+    Args:
         api_endpoint: The entry point (URL) of the SDMX-REST service.
         api_version: The most recent version of the SDMX-REST specification
             supported by the service.
@@ -210,7 +212,7 @@ class RestService(_CoreRestService):
 class AsyncRestService(_CoreRestService):
     """Asynchronous connector to SDMX-REST services.
 
-    Attributes:
+    Args:
         api_endpoint: The entry point (URL) of the SDMX-REST service.
         api_version: The most recent version of the SDMX-REST specification
             supported by the service.
@@ -321,3 +323,89 @@ class AsyncRestService(_CoreRestService):
                 return r.content
             except (httpx.RequestError, httpx.HTTPStatusError) as e:
                 self._map_error(e)
+
+
+class _CoreGdsRestService:
+    """Base class for GDS-REST services."""
+
+    def __init__(
+        self,
+        api_endpoint: str,
+        pem: Optional[str] = None,
+        timeout: float = 5.0,
+    ):
+        self._api_endpoint = api_endpoint.rstrip("/")
+        self._ssl_context = (
+            httpx.create_ssl_context(verify=pem)
+            if pem
+            else httpx.create_ssl_context()
+        )
+        self._headers = {"Accept-Encoding": "gzip, deflate"}
+        self._timeout = timeout
+
+    def _map_error(
+        self, e: Union[httpx.RequestError, httpx.HTTPStatusError]
+    ) -> NoReturn:
+        q = e.request.url
+        if isinstance(e, httpx.HTTPStatusError):
+            s = e.response.status_code
+            t = e.response.text
+            if s == 404:
+                msg = (
+                    f"The requested resource(s) could "
+                    f"not be found. Query: `{q}`"
+                )
+                raise errors.NotFound("Not found", msg) from e
+            elif s < 500:
+                msg = f"Client error {s}. Query: `{q}`. Error: `{t}`."
+                raise errors.Invalid(f"Client error {s}", msg) from e
+            else:
+                msg = f"Service error {s}. Query: `{q}`. Error: `{t}`."
+                raise errors.InternalError(f"Service error {s}", msg) from e
+        else:
+            msg = f"Connection error. Query: `{q}`. Error: `{e}`."
+            raise errors.Unavailable("Connection error", msg) from e
+
+
+class GdsRestService(_CoreGdsRestService):
+    """Synchronous GDS-REST service."""
+
+    def _fetch(self, query: str, format_: str) -> bytes:
+        with httpx.Client(verify=self._ssl_context) as client:
+            try:
+                url = f"{self._api_endpoint}{query}"
+                headers = {**self._headers, "Accept": format_}
+                response = client.get(
+                    url, headers=headers, timeout=self._timeout
+                )
+                response.raise_for_status()
+                return response.content
+            except (httpx.RequestError, httpx.HTTPStatusError) as e:
+                self._map_error(e)
+
+    def gds(self, query: GdsQuery) -> bytes:
+        """Execute a GDS query against the service."""
+        q = query.get_url()
+        return self._fetch(q, GDS_FORMAT)
+
+
+class GdsAsyncRestService(_CoreGdsRestService):
+    """Asynchronous GDS-REST service."""
+
+    async def _fetch(self, query: str, format_: str) -> bytes:
+        async with httpx.AsyncClient(verify=self._ssl_context) as client:
+            try:
+                url = f"{self._api_endpoint}{query}"
+                headers = {**self._headers, "Accept": format_}
+                response = await client.get(
+                    url, headers=headers, timeout=self._timeout
+                )
+                response.raise_for_status()
+                return response.content
+            except (httpx.RequestError, httpx.HTTPStatusError) as e:
+                self._map_error(e)
+
+    async def gds(self, query: GdsQuery) -> bytes:
+        """Execute a GDS query against the service."""
+        q = query.get_url()
+        return await self._fetch(q, GDS_FORMAT)

pysdmx/api/qb/structure.py

@@ -272,6 +272,7 @@ _API_RESOURCES = {
     "V2.0.0": _V2_0_RESOURCES,
     "V2.1.0": _V2_0_RESOURCES,
     "V2.2.0": _V2_0_RESOURCES,
+    "V2.2.1": _V2_0_RESOURCES,
     "LATEST": _V2_0_RESOURCES,
 }
 

pysdmx/api/qb/util.py

@@ -24,6 +24,7 @@ class ApiVersion(IntEnum):
     V2_0_0 = 8
     V2_1_0 = 9
     V2_2_0 = 10
+    V2_2_1 = 11
 
 
 MULT_SEP = re.compile(r"\+")

pysdmx/io/__init__.py

@@ -1,5 +1,6 @@
 """IO module for SDMX data."""
 
 from pysdmx.io.reader import get_datasets, read_sdmx
+from pysdmx.io.writer import write_sdmx
 
-__all__ = ["read_sdmx", "get_datasets"]
+__all__ = ["read_sdmx", "get_datasets", "write_sdmx"]