pysdmx 1.6.0__py3-none-any.whl → 1.8.0__py3-none-any.whl

pysdmx/io/json/fusion/messages/metadataflow.py

@@ -0,0 +1,44 @@
+"""Collection of Fusion-JSON schemas for dataflow queries."""
+
+from typing import Optional, Sequence
+
+from msgspec import Struct
+
+from pysdmx.io.json.fusion.messages.core import FusionString
+from pysdmx.model import Metadataflow as MDF
+
+
+class FusionMetadataflow(Struct, frozen=True, rename={"agency": "agencyId"}):
+    """Fusion-JSON payload for a metadataflow."""
+
+    id: str
+    agency: str
+    names: Sequence[FusionString]
+    metadataStructureRef: str
+    targets: Sequence[str]
+    descriptions: Optional[Sequence[FusionString]] = None
+    version: str = "1.0"
+
+    def to_model(self) -> MDF:
+        """Converts a FusionMetadataflow to a standard metadataflow."""
+        return MDF(
+            id=self.id,
+            agency=self.agency,
+            name=self.names[0].value if self.names else None,
+            description=(
+                self.descriptions[0].value if self.descriptions else None
+            ),
+            version=self.version,
+            structure=self.metadataStructureRef,
+            targets=self.targets,
+        )
+
+
+class FusionMetadataflowsMessage(Struct, frozen=True):
+    """Fusion-JSON payload for /metadataflow queries."""
+
+    Metadataflow: Sequence[FusionMetadataflow]
+
+    def to_model(self) -> Sequence[MDF]:
+        """Returns the requested metadataflow details."""
+        return [df.to_model() for df in self.Metadataflow]

pysdmx/io/json/fusion/messages/mpa.py

@@ -0,0 +1,45 @@
+"""Collection of Fusion-JSON schemas for provision agreements."""
+
+from typing import Optional, Sequence
+
+from msgspec import Struct
+
+from pysdmx.io.json.fusion.messages.core import FusionString
+from pysdmx.model import MetadataProvisionAgreement as MPA
+
+
+class FusionMetadataProvisionAgreement(
+    Struct, frozen=True, rename={"agency": "agencyId"}
+):
+    """Fusion-JSON payload for a metadata provision agreement."""
+
+    id: str
+    names: Sequence[FusionString]
+    agency: str
+    metadataflowRef: str
+    metadataproviderRef: str
+    descriptions: Optional[Sequence[FusionString]] = None
+    version: str = "1.0"
+
+    def to_model(self) -> MPA:
+        """Converts a JsonPA to a standard provision agreement."""
+        description = self.descriptions[0].value if self.descriptions else None
+        return MPA(
+            id=self.id,
+            name=self.names[0].value,
+            agency=self.agency,
+            description=description,
+            version=self.version,
+            metadata_provider=self.metadataproviderRef,
+            metadataflow=self.metadataflowRef,
+        )
+
+
+class FusionMetadataProvisionAgreementMessage(Struct, frozen=True):
+    """Fusion-JSON payload for /metadataprovisionagreement queries."""
+
+    MetadataProvisionAgreement: Sequence[FusionMetadataProvisionAgreement]
+
+    def to_model(self) -> Sequence[MPA]:
+        """Returns the requested metadata provision agreements."""
+        return [c.to_model() for c in self.MetadataProvisionAgreement]

pysdmx/io/json/fusion/messages/msd.py

@@ -0,0 +1,121 @@
+"""Collection of Fusion-JSON schemas for SDMX-REST schema queries."""
+
+from typing import Literal, Optional, Sequence, Union
+
+from msgspec import Struct
+
+from pysdmx.io.json.fusion.messages.code import FusionCodelist
+from pysdmx.io.json.fusion.messages.concept import FusionConceptScheme
+from pysdmx.io.json.fusion.messages.core import (
+    FusionRepresentation,
+    FusionString,
+)
+from pysdmx.io.json.fusion.messages.dsd import (
+    _find_concept,
+    _get_representation,
+)
+from pysdmx.model import (
+    ArrayBoundaries,
+    MetadataComponent,
+)
+from pysdmx.model import (
+    MetadataStructure as MSD,
+)
+
+
+class FusionMetadataAttribute(Struct, frozen=True):
+    """Fusion-JSON payload for a metadata attribute."""
+
+    id: str
+    concept: str
+    minOccurs: int
+    maxOccurs: Union[int, Literal["unbounded"]]
+    presentational: Optional[bool] = False
+    representation: Optional[FusionRepresentation] = None
+    metadataAttributes: Sequence["FusionMetadataAttribute"] = ()
+
+    def to_model(
+        self,
+        cs: Sequence[FusionConceptScheme],
+        cls: Sequence[FusionCodelist],
+    ) -> MetadataComponent:
+        """Returns an attribute."""
+        c = _find_concept(cs, self.concept)
+        dt, facets, codes, _ = _get_representation(
+            self.id, self.representation, cls, {}
+        )
+
+        if self.representation and self.representation.representation:
+            local_enum_ref = self.representation.representation
+        else:
+            local_enum_ref = None
+
+        if self.maxOccurs == "unbounded":
+            ab = ArrayBoundaries(self.minOccurs)
+        elif self.maxOccurs > 1:
+            ab = ArrayBoundaries(self.minOccurs, self.maxOccurs)
+        else:
+            ab = None
+
+        return MetadataComponent(
+            self.id,
+            is_presentational=self.presentational,  # type: ignore[arg-type]
+            concept=c.to_model(cls),
+            local_dtype=dt,
+            local_facets=facets,
+            local_codes=codes,
+            array_def=ab,
+            local_enum_ref=local_enum_ref,
+            components=[
+                ma.to_model(cs, cls) for ma in self.metadataAttributes
+            ],
+        )
+
+
+class FusionMetadataStructure(
+    Struct, frozen=True, rename={"agency": "agencyId"}
+):
+    """Fusion-JSON payload for an MSD."""
+
+    id: str
+    names: Sequence[FusionString]
+    agency: str
+    descriptions: Optional[Sequence[FusionString]] = None
+    version: str = "1.0"
+    metadataAttributes: Sequence[FusionMetadataAttribute] = ()
+
+    def to_model(
+        self,
+        cs: Sequence[FusionConceptScheme],
+        cls: Sequence[FusionCodelist],
+    ) -> MSD:
+        """Returns the schema for this DSD."""
+        return MSD(
+            id=self.id,
+            agency=self.agency,
+            name=self.names[0].value if self.names else None,
+            description=(
+                self.descriptions[0].value if self.descriptions else None
+            ),
+            version=self.version,
+            components=[a.to_model(cs, cls) for a in self.metadataAttributes],
+        )
+
+
+class FusionMetadataStructuresMessage(Struct, frozen=True):
+    """Fusion-JSON payload for /metadatastructure queries."""
+
+    ConceptScheme: Sequence[FusionConceptScheme]
+    MetadataStructure: Sequence[FusionMetadataStructure]
+    ValueList: Sequence[FusionCodelist] = ()
+    Codelist: Sequence[FusionCodelist] = ()
+
+    def to_model(self) -> Sequence[MSD]:
+        """Returns the requested metadata structures."""
+        all_mds = []
+        for msd in self.MetadataStructure:
+            cls: list[FusionCodelist] = []
+            cls.extend(self.Codelist)
+            cls.extend(self.ValueList)
+            all_mds.append(msd.to_model(self.ConceptScheme, cls))
+        return all_mds

pysdmx/io/json/fusion/messages/org.py

@@ -6,12 +6,14 @@ from typing import Dict, Optional, Sequence, Set
 from msgspec import Struct
 
 from pysdmx.io.json.fusion.messages.core import FusionString
+from pysdmx.io.json.fusion.messages.mpa import FusionMetadataProvisionAgreement
 from pysdmx.io.json.fusion.messages.pa import FusionProvisionAgreement
 from pysdmx.model import (
     Agency,
     Contact,
     DataflowRef,
     DataProvider,
+    MetadataProvider,
 )
 from pysdmx.model import (
     AgencyScheme as AS,
@@ -19,6 +21,9 @@ from pysdmx.model import (
 from pysdmx.model import (
     DataProviderScheme as DPS,
 )
+from pysdmx.model import (
+    MetadataProviderScheme as MDPS,
+)
 from pysdmx.util import parse_item_urn, parse_urn
 
 
@@ -188,3 +193,88 @@ class FusionProviderMessage(Struct, frozen=True):
             p.to_model(self.ProvisionAgreement)
             for p in self.DataProviderScheme
         ]
+
+
+class FusionMetadataProvider(Struct, frozen=True):
+    """Fusion-JSON payload for a metadata provider."""
+
+    id: str
+    names: Sequence[FusionString]
+    descriptions: Optional[Sequence[FusionString]] = None
+    contacts: Sequence[FusionContact] = ()
+
+    def to_model(self, owner: Optional[str] = None) -> MetadataProvider:
+        """Converts a FusionMetadataProvider to a pysdmx one."""
+        d = self.descriptions[0].value if self.descriptions else None
+        c = tuple([c.to_model() for c in self.contacts])
+        oid = f"{owner}.{self.id}" if owner and owner != "SDMX" else self.id
+        return MetadataProvider(
+            id=oid, name=self.names[0].value, description=d, contacts=c
+        )
+
+
+class FusionMetadataProviderScheme(Struct, frozen=True):
+    """Fusion-JSON payload for a metadata provider scheme."""
+
+    id: str
+    agencyId: str
+    names: Sequence[FusionString] = ()
+    descriptions: Sequence[FusionString] = ()
+    items: Sequence[FusionMetadataProvider] = ()
+
+    def __get_df_ref(self, ref: str) -> DataflowRef:
+        a = parse_urn(ref)
+        return DataflowRef(id=a.id, agency=a.agency, version=a.version)
+
+    def to_model(
+        self, pas: Sequence[FusionMetadataProvisionAgreement]
+    ) -> MDPS:
+        """Converts a FusionProviderScheme to a DataProviderScheme."""
+        if pas:
+            paprs: Dict[str, Set[DataflowRef]] = defaultdict(set)
+            for pa in pas:
+                df = self.__get_df_ref(pa.metadataflowRef)
+                ref = parse_item_urn(pa.metadataproviderRef)
+                paprs[f"{ref.agency}:{ref.item_id}"].add(df)
+            prvs = [o.to_model() for o in self.items]
+            prvs = [
+                MetadataProvider(
+                    id=p.id,
+                    name=p.name,
+                    description=p.description,
+                    contacts=p.contacts,
+                    dataflows=list(paprs[f"{self.agencyId}:{p.id}"]),
+                )
+                for p in prvs
+            ]
+            return MDPS(
+                agency=self.agencyId,
+                name=self.names[0].value,
+                description=(
+                    self.descriptions[0].value if self.descriptions else None
+                ),
+                items=prvs,
+            )
+        else:
+            return MDPS(
+                agency=self.agencyId,
+                name=self.names[0].value,
+                description=(
+                    self.descriptions[0].value if self.descriptions else None
+                ),
+                items=[o.to_model() for o in self.items],
+            )
+
+
+class FusionMetadataProviderMessage(Struct, frozen=True):
+    """Fusion-JSON payload for /metadataproviderscheme queries."""
+
+    MetadataProviderScheme: Sequence[FusionMetadataProviderScheme]
+    MetadataProvisionAgreement: Sequence[FusionMetadataProvisionAgreement] = ()
+
+    def to_model(self) -> Sequence[MDPS]:
+        """Returns the requested list of metadata providers."""
+        return [
+            p.to_model(self.MetadataProvisionAgreement)
+            for p in self.MetadataProviderScheme
+        ]

pysdmx/io/json/fusion/reader/__init__.py

@@ -20,4 +20,9 @@ deserializers = Deserializers(
     mapping=msg.FusionMappingMessage,  # type: ignore[arg-type]
     code_map=msg.FusionRepresentationMapMessage,  # type: ignore[arg-type]
     transformation_scheme=msg.FusionTransfoMsg,  # type: ignore[arg-type]
+    metadataflows=msg.FusionMetadataflowsMessage,  # type: ignore[arg-type]
+    metadata_provision_agreement=msg.FusionMetadataProvisionAgreementMessage,  # type: ignore[arg-type]
+    metadata_providers=msg.FusionMetadataProviderMessage,  # type: ignore[arg-type]
+    msds=msg.FusionMetadataStructuresMessage,  # type: ignore[arg-type]
+    data_structures=msg.FusionDataStructuresMessage,  # type: ignore[arg-type]
 )

pysdmx/io/json/sdmxjson2/messages/__init__.py

@@ -23,10 +23,20 @@ from pysdmx.io.json.sdmxjson2.messages.map import (
     JsonRepresentationMapsMessage,
     JsonStructureMapsMessage,
 )
+from pysdmx.io.json.sdmxjson2.messages.metadataflow import (
+    JsonMetadataflowsMessage as JsonMdfsMsg,
+)
+from pysdmx.io.json.sdmxjson2.messages.mpa import (
+    JsonMetadataProvisionAgreementsMessage as JsonMPAMsg,
+)
+from pysdmx.io.json.sdmxjson2.messages.msd import JsonMetadataStructuresMessage
 from pysdmx.io.json.sdmxjson2.messages.pa import (
     JsonProvisionAgreementsMessage as JsonPAMessage,
 )
-from pysdmx.io.json.sdmxjson2.messages.provider import JsonProviderMessage
+from pysdmx.io.json.sdmxjson2.messages.provider import (
+    JsonMetadataProviderMessage,
+    JsonProviderMessage,
+)
 from pysdmx.io.json.sdmxjson2.messages.report import JsonMetadataMessage
 from pysdmx.io.json.sdmxjson2.messages.schema import JsonSchemaMessage
 from pysdmx.io.json.sdmxjson2.messages.structure import JsonStructureMessage
@@ -43,14 +53,18 @@ __all__ = [
     "JsonDataflowMessage",
     "JsonDataflowsMessage",
     "JsonDataStructuresMessage",
+    "JsonMetadataProviderMessage",
     "JsonProviderMessage",
     "JsonPAMessage",
     "JsonSchemaMessage",
     "JsonHierarchyAssociationMessage",
     "JsonHierarchiesMessage",
     "JsonHierarchyMessage",
+    "JsonMdfsMsg",
     "JsonMetadataMessage",
+    "JsonMPAMsg",
     "JsonMappingMessage",
+    "JsonMetadataStructuresMessage",
     "JsonRepresentationMapMessage",
     "JsonRepresentationMapsMessage",
     "JsonStructureMapsMessage",

pysdmx/io/json/sdmxjson2/messages/code.py

@@ -34,7 +34,7 @@ class JsonCode(NameableType, frozen=True, omit_defaults=True):
     parent: Optional[str] = None
 
     def __handle_date(self, datestr: str) -> datetime:
-        return datetime.strptime(datestr, _VAL_FMT)
+        return datetime.strptime(datestr, _VAL_FMT)  # noqa
 
     def __get_val(
         self, a: JsonAnnotation
@@ -49,26 +49,38 @@ class JsonCode(NameableType, frozen=True, omit_defaults=True):
 
     def to_model(self) -> Code:
         """Converts a JsonCode to a standard code."""
+        # Pre-filter annotations once outside the tuple creation
+        vf, vt = None, None
+
         if self.annotations:
-            vp = [
-                a for a in self.annotations if a.type == "FR_VALIDITY_PERIOD"
+            # Get validity period info
+            vp = next(
+                (
+                    a
+                    for a in self.annotations
+                    if a.type == "FR_VALIDITY_PERIOD"
+                ),
+                None,
+            )
+            if vp:
+                vf, vt = self.__get_val(vp)
+
+            # Pre-filter non-validity period annotations
+            filtered_annotations = [
+                a.to_model()
+                for a in self.annotations
+                if a.type != "FR_VALIDITY_PERIOD"
             ]
         else:
-            vp = None
-        vf, vt = self.__get_val(vp[0]) if vp else (None, None)
+            filtered_annotations = []
+
         return Code(
             id=self.id,
             name=self.name,
             description=self.description,
             valid_from=vf,
             valid_to=vt,
-            annotations=tuple(
-                [
-                    a.to_model()
-                    for a in self.annotations
-                    if a.type != "FR_VALIDITY_PERIOD"
-                ]
-            ),
+            annotations=tuple(filtered_annotations),
         )
 
     @classmethod
@@ -113,13 +125,23 @@ class JsonCodelist(ItemSchemeType, frozen=True, omit_defaults=True):
 
     def to_model(self) -> Codelist:
         """Converts a JsonCodelist to a standard codelist."""
+        # Process codes in batches to reduce memory pressure
+        batch_size = 10000
+        all_codes = []
+
+        # Process in batches
+        for i in range(0, len(self.codes), batch_size):
+            batch = self.codes[i : i + batch_size]
+            batch_models = [code.to_model() for code in batch]
+            all_codes.extend(batch_models)
+
         return Codelist(
             id=self.id,
             name=self.name,
             agency=self.agency,
             description=self.description,
             version=self.version,
-            items=tuple([i.to_model() for i in self.codes]),
+            items=tuple(all_codes),
             annotations=tuple([a.to_model() for a in self.annotations]),
             is_external_reference=self.isExternalReference,
             is_partial=self.isPartial,

pysdmx/io/json/sdmxjson2/messages/concept.py

@@ -9,6 +9,7 @@ from pysdmx.io.json.sdmxjson2.messages.code import JsonCodelist
 from pysdmx.io.json.sdmxjson2.messages.core import (
     ItemSchemeType,
     JsonAnnotation,
+    JsonLink,
     JsonRepresentation,
     NameableType,
 )
@@ -29,6 +30,7 @@ class JsonConcept(NameableType, frozen=True, omit_defaults=True):
     coreRepresentation: Optional[JsonRepresentation] = None
     parent: Optional[str] = None
     isoConceptReference: Optional[IsoConceptReference] = None
+    links: Sequence[JsonLink] = ()
 
     def to_model(self, codelists: Sequence[Codelist]) -> Concept:
         """Converts a JsonConcept to a standard concept."""
@@ -48,6 +50,8 @@ class JsonConcept(NameableType, frozen=True, omit_defaults=True):
             facets = None
             codes = None
             cl_ref = None
+        urn_lnk = [lnk for lnk in self.links if lnk.rel == "self"]
+        c_urn = urn_lnk[0].urn if len(urn_lnk) > 0 else None
         return Concept(
             id=self.id,
             dtype=dt,
@@ -56,6 +60,7 @@ class JsonConcept(NameableType, frozen=True, omit_defaults=True):
             description=self.description,
             codes=codes,
             enum_ref=cl_ref,
+            urn=c_urn,
         )
 
     @classmethod
@@ -95,18 +100,10 @@ class JsonConceptScheme(ItemSchemeType, frozen=True, omit_defaults=True):
 
     concepts: Sequence[JsonConcept] = ()
 
-    def __set_urn(self, concept: Concept) -> Concept:
-        urn = (
-            "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept="
-            f"{self.agency}:{self.id}({self.version}).{concept.id}"
-        )
-        return msgspec.structs.replace(concept, urn=urn)
-
     def to_model(self, codelists: Sequence[JsonCodelist]) -> ConceptScheme:
         """Converts a JsonConceptScheme to a standard concept scheme."""
         cls = [cl.to_model() for cl in codelists]
         concepts = [c.to_model(cls) for c in self.concepts]
-        concepts = [self.__set_urn(c) for c in concepts]
         return ConceptScheme(
             id=self.id,
             name=self.name,

pysdmx/io/json/sdmxjson2/messages/dsd.py

@@ -1,6 +1,6 @@
-"""Collection of SDMX-JSON schemas for SDMX-REST schema queries."""
+"""Collection of SDMX-JSON schemas for SDMX-REST DSD queries."""
 
-from typing import Dict, List, Literal, Optional, Sequence, Tuple
+from typing import Dict, List, Literal, Optional, Sequence, Tuple, Union
 
 from msgspec import Struct
 
@@ -27,6 +27,7 @@ from pysdmx.model import (
     DataType,
     Facets,
     ItemReference,
+    MetadataComponent,
     Role,
 )
 from pysdmx.model.dataflow import Group
@@ -43,7 +44,7 @@ def _find_concept(cs: Sequence[JsonConceptScheme], urn: str) -> JsonConcept:
     return [c for c in f[0].concepts if c.id == r.item_id][0]
 
 
-def __get_type(repr_: JsonRepresentation) -> Optional[str]:
+def _get_type(repr_: JsonRepresentation) -> Optional[str]:
     t: Optional[str] = None
     if repr_.enumerationFormat:
         t = repr_.enumerationFormat.dataType
@@ -67,13 +68,15 @@ def _get_representation(
 ]:
     valid = cons.get(id_, [])
     codes = local.to_enumeration(cls, valid) if local else None
-    dt = DataType(__get_type(local)) if local else None
+    dt = DataType(_get_type(local)) if local else None
     facets = local.to_facets() if local else None
     ab = local.to_array_def() if local else None
     return (dt, facets, codes, ab)
 
 
-def _get_concept_reference(component: Component) -> str:
+def _get_concept_reference(
+    component: Union[Component, MetadataComponent],
+) -> str:
     if isinstance(component.concept, ItemReference):
         concept = (
             "urn:sdmx:org.sdmx.infomodel.conceptscheme."
@@ -95,7 +98,7 @@ def _get_concept_reference(component: Component) -> str:
 
 
 def _get_json_representation(
-    comp: Component,
+    comp: Union[Component, MetadataComponent],
 ) -> Optional[JsonRepresentation]:
     enum = comp.local_enum_ref if comp.local_enum_ref else None
     return JsonRepresentation.from_model(

pysdmx/io/json/sdmxjson2/messages/metadataflow.py

@@ -0,0 +1,88 @@
+"""Collection of SDMX-JSON schemas for dataflow queries."""
+
+from typing import Sequence
+
+from msgspec import Struct
+
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages.core import (
+    JsonAnnotation,
+    MaintainableType,
+)
+from pysdmx.model import Agency, Metadataflow, MetadataStructure
+
+
+class JsonMetadataflow(MaintainableType, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for a metadataflow."""
+
+    structure: str = ""
+    targets: Sequence[str] = ()
+
+    def to_model(self) -> Metadataflow:
+        """Converts a FusionMetadataflow to a standard one."""
+        return Metadataflow(
+            id=self.id,
+            agency=self.agency,
+            name=self.name,
+            description=self.description,
+            version=self.version,
+            structure=self.structure,
+            targets=self.targets,
+            annotations=[a.to_model() for a in self.annotations],
+            is_external_reference=self.isExternalReference,
+            valid_from=self.validFrom,
+            valid_to=self.validTo,
+        )
+
+    @classmethod
+    def from_model(self, df: Metadataflow) -> "JsonMetadataflow":
+        """Converts a pysdmx metadataflow to an SDMX-JSON one."""
+        if not df.name:
+            raise errors.Invalid(
+                "Invalid input",
+                "SDMX-JSON metadataflows must have a name",
+                {"metadataflow": df.id},
+            )
+        if isinstance(df.structure, MetadataStructure):
+            dsdref = (
+                "urn:sdmx:org.sdmx.infomodel.metadatastructure.MetadataStructure="
+                f"{df.structure.agency}:{df.structure.id}({df.structure.version})"
+            )
+        else:
+            dsdref = df.structure  # type: ignore[assignment]
+        return JsonMetadataflow(
+            agency=(
+                df.agency.id if isinstance(df.agency, Agency) else df.agency
+            ),
+            id=df.id,
+            name=df.name,
+            version=df.version,
+            isExternalReference=df.is_external_reference,
+            validFrom=df.valid_from,
+            validTo=df.valid_to,
+            description=df.description,
+            annotations=tuple(
+                [JsonAnnotation.from_model(a) for a in df.annotations]
+            ),
+            structure=dsdref,
+        )
+
+
+class JsonMetadataflows(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for the list of metadataflows."""
+
+    metadataflows: Sequence[JsonMetadataflow]
+
+    def to_model(self) -> Sequence[Metadataflow]:
+        """Returns the requested metadataflows."""
+        return [mdf.to_model() for mdf in self.metadataflows]
+
+
+class JsonMetadataflowsMessage(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for /metadataflow queries."""
+
+    data: JsonMetadataflows
+
+    def to_model(self) -> Sequence[Metadataflow]:
+        """Returns the requested dataflows."""
+        return self.data.to_model()