pysdmx 1.7.0__py3-none-any.whl → 1.8.1__py3-none-any.whl

pysdmx/io/json/sdmxjson2/messages/msd.py

@@ -0,0 +1,245 @@
+"""Collection of SDMX-JSON schemas for SDMX-REST MSD queries."""
+
+from typing import List, Literal, Optional, Sequence, Union
+
+from msgspec import Struct
+
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages.code import JsonCodelist, JsonValuelist
+from pysdmx.io.json.sdmxjson2.messages.concept import JsonConceptScheme
+from pysdmx.io.json.sdmxjson2.messages.core import (
+    JsonAnnotation,
+    JsonRepresentation,
+    MaintainableType,
+)
+from pysdmx.io.json.sdmxjson2.messages.dsd import (
+    _find_concept,
+    _get_concept_reference,
+    _get_json_representation,
+    _get_representation,
+)
+from pysdmx.model import (
+    Agency,
+    ArrayBoundaries,
+    Codelist,
+    MetadataComponent,
+    MetadataStructure,
+)
+from pysdmx.util import parse_item_urn
+
+
+class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for an attribute."""
+
+    id: str
+    conceptIdentity: str
+    minOccurs: int
+    maxOccurs: Union[int, Literal["unbounded"]]
+    isPresentational: bool
+    localRepresentation: Optional[JsonRepresentation] = None
+    metadataAttributes: Sequence["JsonMetadataAttribute"] = ()
+
+    def to_model(
+        self, cs: Sequence[JsonConceptScheme], cls: Sequence[Codelist]
+    ) -> MetadataComponent:
+        """Returns a metadata component."""
+        c = (
+            _find_concept(cs, self.conceptIdentity).to_model(cls)
+            if cs
+            else parse_item_urn(self.conceptIdentity)
+        )
+        dt, facets, codes, _ = _get_representation(
+            self.id, self.localRepresentation, cls, {}
+        )
+
+        if self.localRepresentation and self.localRepresentation.enumeration:
+            local_enum_ref = self.localRepresentation.enumeration
+        else:
+            local_enum_ref = None
+
+        if self.maxOccurs == "unbounded":
+            ab = ArrayBoundaries(self.minOccurs)
+        elif self.maxOccurs > 1:
+            ab = ArrayBoundaries(self.minOccurs, self.maxOccurs)
+        else:
+            ab = None
+
+        return MetadataComponent(
+            id=self.id,
+            is_presentational=self.isPresentational,
+            concept=c,
+            local_dtype=dt,
+            local_facets=facets,
+            local_codes=codes,
+            array_def=ab,
+            local_enum_ref=local_enum_ref,
+            components=[a.to_model(cs, cls) for a in self.metadataAttributes],
+        )
+
+    @classmethod
+    def from_model(self, cmp: MetadataComponent) -> "JsonMetadataAttribute":
+        """Converts a pysdmx metadata attribute to an SDMX-JSON one."""
+        concept = _get_concept_reference(cmp)
+        repr = _get_json_representation(cmp)
+
+        min_occurs = cmp.array_def.min_size if cmp.array_def else 0
+        if cmp.array_def is None or cmp.array_def.max_size is None:
+            max_occurs: Union[int, Literal["unbounded"]] = "unbounded"
+        else:
+            max_occurs = cmp.array_def.max_size
+
+        return JsonMetadataAttribute(
+            id=cmp.id,
+            conceptIdentity=concept,
+            localRepresentation=repr,
+            minOccurs=min_occurs,
+            maxOccurs=max_occurs,
+            isPresentational=cmp.is_presentational,
+            metadataAttributes=[
+                JsonMetadataAttribute.from_model(c) for c in cmp.components
+            ],
+        )
+
+
+class JsonMetadataAttributes(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for the list of metadata attributes."""
+
+    id: Literal["MetadataAttributeDescriptor"] = "MetadataAttributeDescriptor"
+    metadataAttributes: Sequence[JsonMetadataAttribute] = ()
+
+    def to_model(
+        self, cs: Sequence[JsonConceptScheme], cls: Sequence[Codelist]
+    ) -> List[MetadataComponent]:
+        """Returns the list of metadata attributes."""
+        return [a.to_model(cs, cls) for a in self.metadataAttributes]
+
+    @classmethod
+    def from_model(
+        self, attributes: Sequence[MetadataComponent]
+    ) -> "JsonMetadataAttributes":
+        """Converts a pysdmx list of metadata attributes to SDMX-JSON."""
+        return JsonMetadataAttributes(
+            metadataAttributes=[
+                JsonMetadataAttribute.from_model(a) for a in attributes
+            ]
+        )
+
+
+class JsonMetadataComponents(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for the list of DSD components."""
+
+    metadataAttributeList: Optional[JsonMetadataAttributes] = None
+
+    def to_model(
+        self,
+        cs: Sequence[JsonConceptScheme],
+        cls: Sequence[JsonCodelist],
+        vls: Sequence[JsonValuelist],
+    ) -> Sequence[MetadataComponent]:
+        """Returns the components for this DSD."""
+        enums = [cl.to_model() for cl in cls]
+        enums.extend([vl.to_model() for vl in vls])
+        comps = (
+            self.metadataAttributeList.to_model(cs, enums)
+            if self.metadataAttributeList
+            else []
+        )
+        return comps
+
+    @classmethod
+    def from_model(
+        self, components: Sequence[MetadataComponent]
+    ) -> "JsonMetadataComponents":
+        """Converts a pysdmx components list to an SDMX-JSON one."""
+        attributes = JsonMetadataAttributes.from_model(components)
+        return JsonMetadataComponents(metadataAttributeList=attributes)
+
+
+class JsonMetadataStructure(MaintainableType, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for a DSD."""
+
+    metadataStructureComponents: Optional[JsonMetadataComponents] = None
+
+    def to_model(
+        self,
+        cs: Sequence[JsonConceptScheme],
+        cls: Sequence[JsonCodelist],
+        vls: Sequence[JsonValuelist],
+    ) -> MetadataStructure:
+        """Map to pysdmx model class."""
+        c = self.metadataStructureComponents.to_model(  # type: ignore[union-attr]
+            cs,
+            cls,
+            vls,
+        )
+        return MetadataStructure(
+            id=self.id,
+            name=self.name,
+            agency=self.agency,
+            description=self.description,
+            version=self.version,
+            annotations=[a.to_model() for a in self.annotations],
+            is_external_reference=self.isExternalReference,
+            valid_from=self.validFrom,
+            valid_to=self.validTo,
+            components=c,
+        )
+
+    @classmethod
+    def from_model(self, msd: MetadataStructure) -> "JsonMetadataStructure":
+        """Converts a pysdmx MSD to an SDMX-JSON one."""
+        if not msd.name:
+            raise errors.Invalid(
+                "Invalid input",
+                "SDMX-JSON metadata structures must have a name",
+                {"metadata_structure": msd.id},
+            )
+
+        return JsonMetadataStructure(
+            agency=(
+                msd.agency.id if isinstance(msd.agency, Agency) else msd.agency
+            ),
+            id=msd.id,
+            name=msd.name,
+            version=msd.version,
+            isExternalReference=msd.is_external_reference,
+            validFrom=msd.valid_from,
+            validTo=msd.valid_to,
+            description=msd.description,
+            annotations=tuple(
+                [JsonAnnotation.from_model(a) for a in msd.annotations]
+            ),
+            metadataStructureComponents=JsonMetadataComponents.from_model(
+                msd.components
+            ),
+        )
+
+
+class JsonMetadataStructures(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for metadata structures."""
+
+    metadataStructures: Sequence[JsonMetadataStructure]
+    conceptSchemes: Sequence[JsonConceptScheme] = ()
+    valuelists: Sequence[JsonValuelist] = ()
+    codelists: Sequence[JsonCodelist] = ()
+
+    def to_model(self) -> Sequence[MetadataStructure]:
+        """Returns the requested msds."""
+        return [
+            msd.to_model(
+                self.conceptSchemes,
+                self.codelists,
+                self.valuelists,
+            )
+            for msd in self.metadataStructures
+        ]
+
+
+class JsonMetadataStructuresMessage(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for /metadatastructure queries."""
+
+    data: JsonMetadataStructures
+
+    def to_model(self) -> Sequence[MetadataStructure]:
+        """Returns the requested metadata structures."""
+        return self.data.to_model()

pysdmx/io/json/sdmxjson2/messages/provider.py

@@ -9,8 +9,18 @@ from pysdmx.io.json.sdmxjson2.messages.core import (
     ItemSchemeType,
     JsonAnnotation,
 )
+from pysdmx.io.json.sdmxjson2.messages.mpa import (
+    JsonMetadataProvisionAgreement,
+)
 from pysdmx.io.json.sdmxjson2.messages.pa import JsonProvisionAgreement
-from pysdmx.model import Agency, DataflowRef, DataProvider, DataProviderScheme
+from pysdmx.model import (
+    Agency,
+    DataflowRef,
+    DataProvider,
+    DataProviderScheme,
+    MetadataProvider,
+    MetadataProviderScheme,
+)
 from pysdmx.util import parse_item_urn, parse_urn
 
 
@@ -114,3 +124,109 @@ class JsonProviderMessage(Struct, frozen=True, omit_defaults=True):
     def to_model(self) -> Sequence[DataProviderScheme]:
         """Returns the requested list of data provider schemes."""
         return self.data.to_model()
+
+
+class JsonMetadataProviderScheme(
+    ItemSchemeType, frozen=True, omit_defaults=True
+):
+    """SDMX-JSON payload for a metadata provider scheme."""
+
+    metadataProviders: Sequence[MetadataProvider] = ()
+
+    def __get_df_ref(self, ref: str) -> DataflowRef:
+        a = parse_urn(ref)
+        return DataflowRef(id=a.id, agency=a.agency, version=a.version)
+
+    def to_model(
+        self, pas: Sequence[JsonMetadataProvisionAgreement]
+    ) -> MetadataProviderScheme:
+        """Converts a JsonMetadataProviderScheme to a pysdmx one."""
+        if pas:
+            paprs: Dict[str, Set[DataflowRef]] = defaultdict(set)
+            for pa in pas:
+                df = self.__get_df_ref(pa.metadataflow)
+                ref = parse_item_urn(pa.metadataProvider)
+                paprs[f"{ref.agency}:{ref.item_id}"].add(df)
+            provs = [
+                MetadataProvider(
+                    id=p.id,
+                    name=p.name,
+                    description=p.description,
+                    contacts=p.contacts,
+                    dataflows=list(paprs[f"{self.agency}:{p.id}"]),
+                    annotations=tuple(
+                        [a.to_model() for a in self.annotations]
+                    ),
+                )
+                for p in self.metadataProviders
+            ]
+        else:
+            provs = [
+                MetadataProvider(
+                    id=p.id,
+                    name=p.name,
+                    description=p.description,
+                    contacts=p.contacts,
+                    annotations=tuple(
+                        [a.to_model() for a in self.annotations]
+                    ),
+                )
+                for p in self.metadataProviders
+            ]
+        return MetadataProviderScheme(
+            agency=self.agency,
+            description=self.description,
+            items=provs,
+            annotations=[a.to_model() for a in self.annotations],
+            is_external_reference=self.isExternalReference,
+            is_partial=self.isPartial,
+            valid_from=self.validFrom,
+            valid_to=self.validTo,
+        )
+
+    @classmethod
+    def from_model(
+        self, dps: MetadataProviderScheme
+    ) -> "JsonMetadataProviderScheme":
+        """Converts a pysdmx metadata provider scheme to an SDMX-JSON one."""
+        return JsonMetadataProviderScheme(
+            id="METADATA_PROVIDERS",
+            name="METADATA_PROVIDERS",
+            agency=(
+                dps.agency.id if isinstance(dps.agency, Agency) else dps.agency
+            ),
+            description=dps.description,
+            version="1.0",
+            metadataProviders=dps.items,
+            annotations=tuple(
+                [JsonAnnotation.from_model(a) for a in dps.annotations]
+            ),
+            isExternalReference=dps.is_external_reference,
+            isPartial=dps.is_partial,
+            validFrom=dps.valid_from,
+            validTo=dps.valid_to,
+        )
+
+
+class JsonMetadataProviderSchemes(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for the list of metadata provider schemes."""
+
+    metadataProviderSchemes: Sequence[JsonMetadataProviderScheme]
+    metadataProvisionAgreements: Sequence[JsonMetadataProvisionAgreement] = ()
+
+    def to_model(self) -> Sequence[MetadataProviderScheme]:
+        """Converts a JsonMetadataProviderSchemes to the pysdmx model."""
+        return [
+            s.to_model(self.metadataProvisionAgreements)
+            for s in self.metadataProviderSchemes
+        ]
+
+
+class JsonMetadataProviderMessage(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for /metadataproviderscheme queries."""
+
+    data: JsonMetadataProviderSchemes
+
+    def to_model(self) -> Sequence[MetadataProviderScheme]:
+        """Returns the requested list of metadata provider schemes."""
+        return self.data.to_model()

pysdmx/io/json/sdmxjson2/messages/structure.py

@@ -24,8 +24,16 @@ from pysdmx.io.json.sdmxjson2.messages.map import (
     JsonRepresentationMap,
     JsonStructureMap,
 )
+from pysdmx.io.json.sdmxjson2.messages.metadataflow import JsonMetadataflow
+from pysdmx.io.json.sdmxjson2.messages.mpa import (
+    JsonMetadataProvisionAgreement,
+)
+from pysdmx.io.json.sdmxjson2.messages.msd import JsonMetadataStructure
 from pysdmx.io.json.sdmxjson2.messages.pa import JsonProvisionAgreement
-from pysdmx.io.json.sdmxjson2.messages.provider import JsonDataProviderScheme
+from pysdmx.io.json.sdmxjson2.messages.provider import (
+    JsonDataProviderScheme,
+    JsonMetadataProviderScheme,
+)
 from pysdmx.io.json.sdmxjson2.messages.vtl import (
     JsonCustomTypeScheme,
     JsonNamePersonalisationScheme,
@@ -50,8 +58,12 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
     hierarchyAssociations: Sequence[JsonHierarchyAssociation] = ()
     agencySchemes: Sequence[JsonAgencyScheme] = ()
     dataProviderSchemes: Sequence[JsonDataProviderScheme] = ()
+    metadataProviderSchemes: Sequence[JsonMetadataProviderScheme] = ()
     dataflows: Sequence[JsonDataflow] = ()
     provisionAgreements: Sequence[JsonProvisionAgreement] = ()
+    metadataflows: Sequence[JsonMetadataflow] = ()
+    metadataProvisionAgreements: Sequence[JsonMetadataProvisionAgreement] = ()
+    metadataStructures: Sequence[JsonMetadataStructure] = ()
     structureMaps: Sequence[JsonStructureMap] = ()
     representationMaps: Sequence[JsonRepresentationMap] = ()
     categorisations: Sequence[JsonCategorisation] = ()
@@ -89,6 +101,14 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
             i.to_model(self.provisionAgreements)
             for i in self.dataProviderSchemes
         )
+        structures.extend(
+            i.to_model(self.metadataProvisionAgreements)
+            for i in self.metadataProviderSchemes
+        )
+        structures.extend(
+            i.to_model(self.conceptSchemes, self.codelists, self.valueLists)
+            for i in self.metadataStructures
+        )
         structures.extend(
             i.to_model(
                 self.dataStructures,
@@ -99,6 +119,10 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
             for i in self.dataflows
         )
         structures.extend(i.to_model() for i in self.provisionAgreements)
+        structures.extend(i.to_model() for i in self.metadataflows)
+        structures.extend(
+            i.to_model() for i in self.metadataProvisionAgreements
+        )
         structures.extend(
             i.to_model(self.representationMaps) for i in self.structureMaps
         )

pysdmx/io/json/sdmxjson2/reader/__init__.py

@@ -20,4 +20,9 @@ deserializers = Deserializers(
     mapping=msg.JsonMappingMessage,  # type: ignore[arg-type]
     code_map=msg.JsonRepresentationMapMessage,  # type: ignore[arg-type]
     transformation_scheme=msg.JsonTransfoMsg,  # type: ignore[arg-type]
+    metadataflows=msg.JsonMdfsMsg,  # type: ignore[arg-type]
+    metadata_provision_agreement=msg.JsonMPAMsg,  # type: ignore[arg-type]
+    metadata_providers=msg.JsonMetadataProviderMessage,  # type: ignore[arg-type]
+    msds=msg.JsonMetadataStructuresMessage,  # type: ignore[arg-type]
+    data_structures=msg.JsonDataStructuresMessage,  # type: ignore[arg-type]
 )

pysdmx/io/serde.py

@@ -41,6 +41,11 @@ class Deserializers:
     mapping: Deserializer
     code_map: Deserializer
     transformation_scheme: Deserializer
+    metadataflows: Deserializer
+    metadata_provision_agreement: Deserializer
+    metadata_providers: Deserializer
+    msds: Deserializer
+    data_structures: Deserializer
 
 
 @dataclass

pysdmx/io/writer.py

@@ -19,11 +19,9 @@ WRITERS = {
     Format.DATA_SDMX_ML_2_1_STR: "pysdmx.io.xml.sdmx21.writer."
     "structure_specific",
     Format.STRUCTURE_SDMX_ML_2_1: "pysdmx.io.xml.sdmx21.writer.structure",
-    Format.DATA_SDMX_ML_3_0: "pysdmx.io.xml.sdmx30.writer."
-    "structure_specific",
+    Format.DATA_SDMX_ML_3_0: "pysdmx.io.xml.sdmx30.writer.structure_specific",
     Format.STRUCTURE_SDMX_ML_3_0: "pysdmx.io.xml.sdmx30.writer.structure",
-    Format.DATA_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer."
-    "structure_specific",
+    Format.DATA_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer.structure_specific",
     Format.STRUCTURE_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer.structure",
     Format.STRUCTURE_SDMX_JSON_2_0_0: (
         "pysdmx.io.json.sdmxjson2.writer.structure"

pysdmx/io/xml/__structure_aux_writer.py

@@ -310,9 +310,9 @@ def __write_maintainable(
         f"{str(maintainable.is_external_reference).lower()!r}"
     )
     if not references_30 and not (isinstance(maintainable, AgencyScheme)):
-        outfile[
-            "Attributes"
-        ] += f" isFinal={str(maintainable.is_final).lower()!r}"
+        outfile["Attributes"] += (
+            f" isFinal={str(maintainable.is_final).lower()!r}"
+        )
 
     if isinstance(maintainable.agency, str):
         outfile["Attributes"] += f" agencyID={maintainable.agency!r}"
@@ -809,9 +809,9 @@ def __write_scheme(  # noqa: C901
         DSD,
         DFW,
     ]:
-        data[
-            "Attributes"
-        ] += f" isPartial={str(item_scheme.is_partial).lower()!r}"
+        data["Attributes"] += (
+            f" isPartial={str(item_scheme.is_partial).lower()!r}"
+        )
     if scheme in [
         RULE_SCHEME,
         UDO_SCHEME,
@@ -820,9 +820,9 @@ def __write_scheme(  # noqa: C901
         CUSTOM_TYPE_SCHEME,
         NAME_PER_SCHEME,
     ]:
-        data[
-            "Attributes"
-        ] += f" {_write_vtl(item_scheme, indent, references_30)}"
+        data["Attributes"] += (
+            f" {_write_vtl(item_scheme, indent, references_30)}"
+        )
 
     outfile = ""
 

pysdmx/io/xml/__write_data_aux.py

@@ -33,8 +33,7 @@ def check_dimension_at_observation(
         dimension_codes = [dim.id for dim in components.dimensions]
         if value not in dimension_codes:
             raise Invalid(
-                f"Dimension at observation {value} "
-                f"not found in dataset {key}."
+                f"Dimension at observation {value} not found in dataset {key}."
             )
     # Add the missing datasets on mapping with ALL_DIM
     for key in datasets:

pysdmx/model/__init__.py

@@ -53,7 +53,14 @@ from pysdmx.model.map import (
     StructureMap,
     ValueMap,
 )
-from pysdmx.model.metadata import MetadataAttribute, MetadataReport
+from pysdmx.model.metadata import (
+    MetadataAttribute,
+    MetadataComponent,
+    Metadataflow,
+    MetadataProvisionAgreement,
+    MetadataReport,
+    MetadataStructure,
+)
 from pysdmx.model.organisation import (
     AgencyScheme,
     DataConsumerScheme,
@@ -174,9 +181,13 @@ __all__ = [
     "ImplicitComponentMap",
     "ItemReference",
     "MetadataAttribute",
+    "MetadataComponent",
+    "Metadataflow",
     "MetadataProvider",
     "MetadataProviderScheme",
+    "MetadataProvisionAgreement",
     "MetadataReport",
+    "MetadataStructure",
     "MultiComponentMap",
     "MultiRepresentationMap",
     "MultiValueMap",

pysdmx/model/dataflow.py

@@ -116,6 +116,7 @@ class Component(
         id: A unique identifier for the component (e.g. FREQ).
         required: Whether the component must have a value.
         role: The role played by the component.
+        concept: The concept giving its identity to the component.
         local_dtype: The component's local data type (string, number, etc.).
         local_facets: Additional local details such as the component's minimum
             length.

pysdmx/model/map.py

@@ -486,7 +486,9 @@ class StructureMap(MaintainableArtefact, frozen=True, omit_defaults=True):
         """Return the number of mapping rules in the structure map."""
         return len(self.maps)
 
-    def __getitem__(self, id_: str) -> Optional[
+    def __getitem__(
+        self, id_: str
+    ) -> Optional[
         Sequence[
             Union[
                 ComponentMap,

pysdmx/model/message.py

@@ -34,8 +34,17 @@ from pysdmx.model.map import (
     RepresentationMap,
     StructureMap,
 )
-from pysdmx.model.metadata import MetadataReport
-from pysdmx.model.organisation import AgencyScheme, DataProviderScheme
+from pysdmx.model.metadata import (
+    Metadataflow,
+    MetadataProvisionAgreement,
+    MetadataReport,
+    MetadataStructure,
+)
+from pysdmx.model.organisation import (
+    AgencyScheme,
+    DataProviderScheme,
+    MetadataProviderScheme,
+)
 from pysdmx.model.submission import SubmissionResult
 from pysdmx.model.vtl import (
     CustomTypeScheme,
@@ -221,6 +230,10 @@ class StructureMessage(Struct, repr_omit_defaults=True, frozen=True):
         """Returns the Dataflows."""
         return self.__get_elements(Dataflow)
 
+    def get_metadataflows(self) -> List[Metadataflow]:
+        """Returns the MetadataProvisionAgreements."""
+        return self.__get_elements(Metadataflow)
+
     def get_organisation_scheme(self, short_urn: str) -> AgencyScheme:
         """Returns a specific OrganisationScheme."""
         return self.__get_single_structure(AgencyScheme, short_urn)
@@ -281,6 +294,20 @@ class StructureMessage(Struct, repr_omit_defaults=True, frozen=True):
         """Returns the ProvisionAgreements."""
         return self.__get_elements(ProvisionAgreement)
 
+    def get_metadata_provider_schemes(self) -> List[MetadataProviderScheme]:
+        """Returns the MetadataProviderSchemes."""
+        return self.__get_elements(MetadataProviderScheme)
+
+    def get_metadata_provision_agreements(
+        self,
+    ) -> List[MetadataProvisionAgreement]:
+        """Returns the MetadataProvisionAgreements."""
+        return self.__get_elements(MetadataProvisionAgreement)
+
+    def get_metadata_structures(self) -> List[MetadataStructure]:
+        """Returns the MetadataStructures."""
+        return self.__get_elements(MetadataStructure)
+
     def get_structure_maps(self) -> List[StructureMap]:
         """Returns the StructureMaps."""
         return self.__get_elements(StructureMap)