pysdmx 1.9.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

pysdmx/io/json/sdmxjson2/messages/structure.py

@@ -17,6 +17,7 @@ from pysdmx.io.json.sdmxjson2.messages.code import (
     JsonValuelist,
 )
 from pysdmx.io.json.sdmxjson2.messages.concept import JsonConceptScheme
+from pysdmx.io.json.sdmxjson2.messages.constraint import JsonDataConstraint
 from pysdmx.io.json.sdmxjson2.messages.core import JsonHeader
 from pysdmx.io.json.sdmxjson2.messages.dataflow import JsonDataflow
 from pysdmx.io.json.sdmxjson2.messages.dsd import JsonDataStructure
@@ -49,90 +50,95 @@ from pysdmx.model.message import StructureMessage
 class JsonStructures(Struct, frozen=True, omit_defaults=True):
     """The allowed strutures."""
 
-    dataStructures: Sequence[JsonDataStructure] = ()
+    agencySchemes: Sequence[JsonAgencyScheme] = ()
+    categorisations: Sequence[JsonCategorisation] = ()
     categorySchemes: Sequence[JsonCategoryScheme] = ()
-    conceptSchemes: Sequence[JsonConceptScheme] = ()
     codelists: Sequence[JsonCodelist] = ()
-    valueLists: Sequence[JsonValuelist] = ()
+    conceptSchemes: Sequence[JsonConceptScheme] = ()
+    customTypeSchemes: Sequence[JsonCustomTypeScheme] = ()
+    dataConstraints: Sequence[JsonDataConstraint] = ()
+    dataflows: Sequence[JsonDataflow] = ()
+    dataProviderSchemes: Sequence[JsonDataProviderScheme] = ()
+    dataStructures: Sequence[JsonDataStructure] = ()
     hierarchies: Sequence[JsonHierarchy] = ()
     hierarchyAssociations: Sequence[JsonHierarchyAssociation] = ()
-    agencySchemes: Sequence[JsonAgencyScheme] = ()
-    dataProviderSchemes: Sequence[JsonDataProviderScheme] = ()
-    metadataProviderSchemes: Sequence[JsonMetadataProviderScheme] = ()
-    dataflows: Sequence[JsonDataflow] = ()
-    provisionAgreements: Sequence[JsonProvisionAgreement] = ()
     metadataflows: Sequence[JsonMetadataflow] = ()
+    metadataProviderSchemes: Sequence[JsonMetadataProviderScheme] = ()
     metadataProvisionAgreements: Sequence[JsonMetadataProvisionAgreement] = ()
     metadataStructures: Sequence[JsonMetadataStructure] = ()
-    structureMaps: Sequence[JsonStructureMap] = ()
-    representationMaps: Sequence[JsonRepresentationMap] = ()
-    categorisations: Sequence[JsonCategorisation] = ()
-    customTypeSchemes: Sequence[JsonCustomTypeScheme] = ()
-    vtlMappingSchemes: Sequence[JsonVtlMappingScheme] = ()
     namePersonalisationSchemes: Sequence[JsonNamePersonalisationScheme] = ()
+    provisionAgreements: Sequence[JsonProvisionAgreement] = ()
+    representationMaps: Sequence[JsonRepresentationMap] = ()
     rulesetSchemes: Sequence[JsonRulesetScheme] = ()
+    structureMaps: Sequence[JsonStructureMap] = ()
     transformationSchemes: Sequence[JsonTransformationScheme] = ()
     userDefinedOperatorSchemes: Sequence[JsonUserDefinedOperatorScheme] = ()
+    valueLists: Sequence[JsonValuelist] = ()
+    vtlMappingSchemes: Sequence[JsonVtlMappingScheme] = ()
 
     def to_model(self) -> Sequence[MaintainableArtefact]:
         """Map to pysdmx artefacts."""
         structures = []  # type: ignore[var-annotated]
         structures.extend(
-            i.to_model(
-                self.conceptSchemes, self.codelists, self.valueLists, ()
-            )
-            for i in self.dataStructures
+            i.to_model(self.dataflows) for i in self.agencySchemes
         )
+        structures.extend(i.to_model() for i in self.categorisations)
         structures.extend(i.to_model() for i in self.categorySchemes)
-        structures.extend(
-            i.to_model(self.codelists) for i in self.conceptSchemes
-        )
         structures.extend(i.to_model() for i in self.codelists)
-        structures.extend(i.to_model() for i in self.valueLists)
-        structures.extend(i.to_model(self.codelists) for i in self.hierarchies)
         structures.extend(
-            i.to_model(self.hierarchies, self.codelists)
-            for i in self.hierarchyAssociations
+            i.to_model(self.codelists) for i in self.conceptSchemes
         )
+        structures.extend(i.to_model() for i in self.customTypeSchemes)
+        structures.extend(i.to_model() for i in self.dataConstraints)
         structures.extend(
-            i.to_model(self.dataflows) for i in self.agencySchemes
+            i.to_model(
+                self.dataStructures,
+                self.conceptSchemes,
+                self.valueLists,
+                self.codelists,
+            )
+            for i in self.dataflows
         )
         structures.extend(
             i.to_model(self.provisionAgreements)
             for i in self.dataProviderSchemes
         )
+        structures.extend(
+            i.to_model(
+                self.conceptSchemes, self.codelists, self.valueLists, ()
+            )
+            for i in self.dataStructures
+        )
+        structures.extend(i.to_model(self.codelists) for i in self.hierarchies)
+        structures.extend(
+            i.to_model(self.hierarchies, self.codelists)
+            for i in self.hierarchyAssociations
+        )
+
+        structures.extend(i.to_model() for i in self.metadataflows)
         structures.extend(
             i.to_model(self.metadataProvisionAgreements)
             for i in self.metadataProviderSchemes
         )
+        structures.extend(
+            i.to_model() for i in self.metadataProvisionAgreements
+        )
         structures.extend(
             i.to_model(self.conceptSchemes, self.codelists, self.valueLists)
             for i in self.metadataStructures
         )
         structures.extend(
-            i.to_model(
-                self.dataStructures,
-                self.conceptSchemes,
-                self.valueLists,
-                self.codelists,
-            )
-            for i in self.dataflows
+            i.to_model() for i in self.namePersonalisationSchemes
         )
         structures.extend(i.to_model() for i in self.provisionAgreements)
-        structures.extend(i.to_model() for i in self.metadataflows)
         structures.extend(
-            i.to_model() for i in self.metadataProvisionAgreements
+            i.to_model(bool(len(i.source) > 1 or len(i.target) > 1))
+            for i in self.representationMaps
         )
+        structures.extend(i.to_model() for i in self.rulesetSchemes)
         structures.extend(
             i.to_model(self.representationMaps) for i in self.structureMaps
         )
-        structures.extend(i.to_model() for i in self.categorisations)
-        structures.extend(i.to_model() for i in self.customTypeSchemes)
-        structures.extend(i.to_model() for i in self.vtlMappingSchemes)
-        structures.extend(
-            i.to_model() for i in self.namePersonalisationSchemes
-        )
-        structures.extend(i.to_model() for i in self.rulesetSchemes)
         structures.extend(
             i.to_model(
                 self.customTypeSchemes,
@@ -146,9 +152,8 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
         structures.extend(
             i.to_model() for i in self.userDefinedOperatorSchemes
         )
-        for rm in self.representationMaps:
-            multi = bool(len(rm.source) > 1 or len(rm.target) > 1)
-            structures.append(rm.to_model(multi))
+        structures.extend(i.to_model() for i in self.valueLists)
+        structures.extend(i.to_model() for i in self.vtlMappingSchemes)
         return structures
 
     @classmethod
@@ -264,6 +269,33 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
         hierarchies = tuple(
             [JsonHierarchy.from_model(h) for h in msg.get_hierarchies()]
         )
+        constraints = tuple(
+            [
+                JsonDataConstraint.from_model(c)
+                for c in msg.get_data_constraints()
+            ]
+        )
+        mpas = tuple(
+            [
+                JsonMetadataProvisionAgreement.from_model(c)
+                for c in msg.get_metadata_provision_agreements()
+            ]
+        )
+        mprvs = tuple(
+            [
+                JsonMetadataProviderScheme.from_model(c)
+                for c in msg.get_metadata_provider_schemes()
+            ]
+        )
+        mdfs = tuple(
+            [JsonMetadataflow.from_model(c) for c in msg.get_metadataflows()]
+        )
+        msds = tuple(
+            [
+                JsonMetadataStructure.from_model(c)
+                for c in msg.get_metadata_structures()
+            ]
+        )
         return JsonStructures(
             agencySchemes=agencies,
             categorisations=categorisations,
@@ -271,11 +303,16 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
             codelists=codelists,
             conceptSchemes=concept_schemes,
             customTypeSchemes=custom_types,
+            dataConstraints=constraints,
             dataflows=dataflows,
             dataProviderSchemes=data_providers,
             dataStructures=data_structures,
             hierarchies=hierarchies,
             hierarchyAssociations=hier_associations,
+            metadataflows=mdfs,
+            metadataProviderSchemes=mprvs,
+            metadataProvisionAgreements=mpas,
+            metadataStructures=msds,
             namePersonalisationSchemes=name_personalisations,
             provisionAgreements=agreements,
             representationMaps=representations_maps,

pysdmx/io/json/sdmxjson2/messages/vtl.py

@@ -1,6 +1,6 @@
 """Collection of SDMX-JSON schemas for VTL artefacts."""
 
-from typing import Literal, Optional, Sequence
+from typing import Dict, Literal, Optional, Sequence
 
 from msgspec import Struct
 
@@ -506,33 +506,37 @@ class JsonRulesetScheme(ItemSchemeType, frozen=True, omit_defaults=True):
 class JsonToVtlMapping(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for To VTL mappings."""
 
-    toVtlSubSpace: Sequence[str]
-    type: Optional[str] = None
+    toVtlSubSpace: Dict[str, Sequence[str]]
+    method: Optional[str] = None
 
     def to_model(self) -> ToVtlMapping:
         """Converts deserialized class to pysdmx model class."""
-        return ToVtlMapping(self.toVtlSubSpace, self.type)
+        return ToVtlMapping(self.toVtlSubSpace["keys"], self.method)
 
     @classmethod
     def from_model(cls, mapping: ToVtlMapping) -> "JsonToVtlMapping":
         """Converts a pysdmx "to VTL" mapping to an SDMX-JSON one."""
-        return JsonToVtlMapping(mapping.to_vtl_sub_space, mapping.method)
+        return JsonToVtlMapping(
+            {"keys": mapping.to_vtl_sub_space}, mapping.method
+        )
 
 
 class JsonFromVtlMapping(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for from VTL mappings."""
 
-    fromVtlSuperSpace: Sequence[str]
-    type: Optional[str] = None
+    fromVtlSuperSpace: Dict[str, Sequence[str]]
+    method: Optional[str] = None
 
     def to_model(self) -> FromVtlMapping:
         """Converts deserialized class to pysdmx model class."""
-        return FromVtlMapping(self.fromVtlSuperSpace, self.type)
+        return FromVtlMapping(self.fromVtlSuperSpace["keys"], self.method)
 
     @classmethod
     def from_model(cls, mapping: FromVtlMapping) -> "JsonFromVtlMapping":
         """Converts a pysdmx "from VTL" mapping to an SDMX-JSON one."""
-        return JsonFromVtlMapping(mapping.from_vtl_sub_space, mapping.method)
+        return JsonFromVtlMapping(
+            {"keys": mapping.from_vtl_sub_space}, mapping.method
+        )
 
 
 class JsonVtlMapping(NameableType, frozen=True, omit_defaults=True):

pysdmx/io/json/sdmxjson2/reader/doc_validation.py

@@ -88,6 +88,10 @@ def validate_sdmx_json(input_str: str) -> None:
                     lambda m: f"does not match required"
                     f" pattern {m.group(1)!r}",
                 ),
+                (
+                    r"\[\]\s+is\s+too\s+short",
+                    lambda _m: "[] should be non-empty",
+                ),
             ]
 
             msg: Optional[str] = next(

pysdmx/io/xml/header.py

@@ -31,6 +31,7 @@ from pysdmx.io.xml.__tokens import (
     URN,
     VERSION,
 )
+from pysdmx.io.xml.utils import add_list
 from pysdmx.model import Organisation, Reference
 from pysdmx.model.dataset import ActionType
 from pysdmx.model.message import Header
@@ -87,43 +88,49 @@ def __parse_sender_receiver(
 def __parse_structure(
     structure: Union[Dict[str, Any], None],
 ) -> Union[Dict[str, str], None]:
-    """Parses the structure of the SDMX header."""
+    """Parses the structure/s of the SDMX header."""
     if structure is None:
         return None
 
-    dim_at_obs = structure.get(DIM_OBS, "AllDimensions")
-
-    if STRUCTURE in structure:
-        structure_info = structure[STRUCTURE]
-        sdmx_type = DSD
-    elif STR_USAGE in structure:
-        structure_info = structure[STR_USAGE]
-        sdmx_type = DFW
-    elif PROV_AGREEMENT in structure:
-        structure_info = structure[PROV_AGREEMENT]
-        sdmx_type = PROV_AGREEMENT
-    else:
-        # Provision Agrement is a typo in the SDMX 2.1 schema,
-        # and it is later solved in SDMX 3.0
-        structure_info = structure[PROV_AGREMENT]
-        sdmx_type = PROV_AGREMENT
-
-    if REF in structure_info:
-        reference = structure_info[REF]
-        agency_id = reference[AGENCY_ID]
-        structure_id = reference[ID]
-        version = reference[VERSION]
-        ref_obj = Reference(
-            sdmx_type=sdmx_type,
-            agency=agency_id,
-            id=structure_id,
-            version=version,
-        )
-    elif URN in structure_info:
-        ref_obj = parse_maintainable_urn(structure_info[URN])
-    else:
-        ref_obj = parse_maintainable_urn(structure_info)
-    return {str(ref_obj): dim_at_obs}
+    result = {}
+
+    for struct in add_list(structure):
+        dim_at_obs = struct.get(DIM_OBS, "AllDimensions")
+
+        if STRUCTURE in struct:
+            structure_info = struct[STRUCTURE]
+            sdmx_type = DSD
+        elif STR_USAGE in struct:
+            structure_info = struct[STR_USAGE]
+            sdmx_type = DFW
+        elif PROV_AGREEMENT in struct:
+            structure_info = struct[PROV_AGREEMENT]
+            sdmx_type = PROV_AGREEMENT
+        else:
+            # Provision Agrement is a typo in the SDMX 2.1 schema,
+            # and it is later solved in SDMX 3.0
+            structure_info = struct[PROV_AGREMENT]
+            sdmx_type = PROV_AGREMENT
+
+        if REF in structure_info:
+            reference = structure_info[REF]
+            agency_id = reference[AGENCY_ID]
+            structure_id = reference[ID]
+            version = reference[VERSION]
+            ref_obj = Reference(
+                sdmx_type=sdmx_type,
+                agency=agency_id,
+                id=structure_id,
+                version=version,
+            )
+        elif URN in structure_info:
+            ref_obj = parse_maintainable_urn(structure_info[URN])
+        else:
+            ref_obj = parse_maintainable_urn(structure_info)
+
+        result[str(ref_obj)] = dim_at_obs
+
+    return result
 
 
 def __parse_source(source: Optional[Dict[str, Any]]) -> Optional[str]:

pysdmx/model/__base.py

@@ -1,5 +1,6 @@
+import re
 from datetime import datetime
-from typing import Any, Optional, Sequence, Union
+from typing import Any, Literal, Optional, Sequence, Union
 
 from msgspec import Struct
 
@@ -327,6 +328,50 @@ class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True):
     items: Sequence[Item] = ()
     is_partial: bool = False
 
+    def search(
+        self,
+        query: str,
+        use_regex: bool = False,
+        fields: Literal["name", "description", "all"] = "all",
+    ) -> Sequence[Item]:
+        """Search for items matching the query.
+
+        Args:
+            query: The substring or regex pattern to search for.
+            use_regex: Whether to treat the query as a regex (default: False).
+            fields: The fields to search in (default: all textual fields).
+
+        Returns:
+           Items that match the query.
+        """
+        if not query:
+            raise Invalid(
+                "Invalid search", "The query string cannot be empty."
+            )
+
+        # Determine which fields to search in
+        search_fields = (
+            ["name", "description"] if fields == "all" else [fields]
+        )
+
+        # Transform plain text queries into a regex
+        if not use_regex:
+            query = re.escape(query)
+
+        pattern = re.compile(query, re.IGNORECASE if not use_regex else 0)
+
+        all_items = getattr(self, "all_items", "")
+        items = all_items if all_items else self.items
+
+        return [
+            item  # type: ignore[misc]
+            for item in items
+            if any(
+                pattern.search(str(getattr(item, field, "")))
+                for field in search_fields
+            )
+        ]
+
 
 class DataflowRef(
     Struct, frozen=True, omit_defaults=True, repr_omit_defaults=True, tag=True

pysdmx/model/__init__.py

@@ -29,6 +29,16 @@ from pysdmx.model.code import (
     HierarchyAssociation,
 )
 from pysdmx.model.concept import Concept, ConceptScheme, DataType, Facets
+from pysdmx.model.constraint import (
+    ConstraintAttachment,
+    CubeKeyValue,
+    CubeRegion,
+    CubeValue,
+    DataConstraint,
+    DataKey,
+    DataKeyValue,
+    KeySet,
+)
 from pysdmx.model.dataflow import (
     ArrayBoundaries,
     Component,
@@ -161,9 +171,16 @@ __all__ = [
     "ComponentMap",
     "Concept",
     "ConceptScheme",
+    "ConstraintAttachment",
     "Contact",
+    "CubeKeyValue",
+    "CubeRegion",
+    "CubeValue",
     "DataConsumer",
     "DataConsumerScheme",
+    "DataConstraint",
+    "DataKey",
+    "DataKeyValue",
     "Dataflow",
     "DataflowInfo",
     "DataflowRef",
@@ -180,6 +197,7 @@ __all__ = [
     "HierarchyAssociation",
     "ImplicitComponentMap",
     "ItemReference",
+    "KeySet",
     "MetadataAttribute",
     "MetadataComponent",
     "Metadataflow",

pysdmx/model/category.py

@@ -91,6 +91,15 @@ class CategoryScheme(ItemScheme, frozen=True, omit_defaults=True):
             flows.update(self.__extract_flows(cat))
         return list(flows)
 
+    @property
+    def all_items(self) -> Sequence[Category]:
+        """Get all the categories in the category scheme as a flat list.
+
+        Returns:
+            A flat list of all the categories present in the category scheme.
+        """
+        return self.__get_categories(self.categories)
+
     def __iter__(self) -> Iterator[Category]:
         """Return an iterator over the list of categories."""
         yield from self.categories
@@ -160,6 +169,14 @@ class CategoryScheme(ItemScheme, frozen=True, omit_defaults=True):
             processed_output.append(f"{attr}: {value}")
         return f"{', '.join(processed_output)}"
 
+    def __get_categories(self, cats: Sequence[Category]) -> Sequence[Category]:
+        out = []
+        for cat in cats:
+            out.append(cat)
+            if cat.categories:
+                out.extend(self.__get_categories(cat.categories))
+        return out
+
 
 class Categorisation(
     MaintainableArtefact, frozen=True, omit_defaults=True, kw_only=True

pysdmx/model/constraint.py

@@ -0,0 +1,69 @@
+"""Model for SDMX Data Constraints."""
+
+from datetime import datetime
+from typing import Optional, Sequence
+
+from msgspec import Struct
+
+from pysdmx.model.__base import MaintainableArtefact
+
+
+class CubeValue(Struct, frozen=True, omit_defaults=True):
+    """A value of the cube, with optional business validity."""
+
+    value: str
+    valid_from: Optional[datetime] = None
+    valid_to: Optional[datetime] = None
+
+
+class CubeKeyValue(Struct, frozen=True, omit_defaults=True):
+    """The list of values for a cube's component."""
+
+    id: str
+    values: Sequence[CubeValue]
+
+
+class CubeRegion(Struct, frozen=True, omit_defaults=True):
+    """A cube region, with its associated values (by default, included)."""
+
+    key_values: Sequence[CubeKeyValue]
+    is_included: bool = True
+
+
+class ConstraintAttachment(Struct, frozen=True, omit_defaults=True):
+    """The artefacts to which the data constraint is attached."""
+
+    data_provider: Optional[str]
+    data_structures: Optional[Sequence[str]] = None
+    dataflows: Optional[Sequence[str]] = None
+    provision_agreements: Optional[Sequence[str]] = None
+
+
+class DataKeyValue(Struct, frozen=True, omit_defaults=True):
+    """A key value, i.e. a component of the key (e.g. FREQ=M)."""
+
+    id: str
+    value: str
+
+
+class DataKey(Struct, frozen=True, omit_defaults=True):
+    """A data key, i.e. one value per dimension in the data key."""
+
+    keys_values: Sequence[DataKeyValue]
+    valid_from: Optional[datetime] = None
+    valid_to: Optional[datetime] = None
+
+
+class KeySet(Struct, frozen=True, omit_defaults=True):
+    """A set of keys, inluded by default."""
+
+    keys: Sequence[DataKey]
+    is_included: bool
+
+
+class DataConstraint(MaintainableArtefact, frozen=True, omit_defaults=True):
+    """A data constraint, defining the allowed or available values."""
+
+    constraint_attachment: Optional[ConstraintAttachment] = None
+    cube_regions: Sequence[CubeRegion] = ()
+    key_sets: Sequence[KeySet] = ()