pysdmx 1.10.0rc1__py3-none-any.whl → 1.10.1__py3-none-any.whl

pysdmx/io/json/sdmxjson2/messages/msd.py

@@ -15,7 +15,6 @@ from pysdmx.io.json.sdmxjson2.messages.core import (
 from pysdmx.io.json.sdmxjson2.messages.dsd import (
     _find_concept,
     _get_concept_reference,
-    _get_json_representation,
     _get_representation,
 )
 from pysdmx.model import (
@@ -28,6 +27,13 @@ from pysdmx.model import (
 from pysdmx.util import parse_item_urn
 
 
+def _get_attr_repr(comp: MetadataComponent) -> Optional[JsonRepresentation]:
+    enum = comp.local_enum_ref if comp.local_enum_ref else None
+    return JsonRepresentation.from_model(
+        comp.local_dtype, enum, comp.local_facets, None
+    )
+
+
 class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for an attribute."""
 
@@ -80,11 +86,13 @@ class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
     def from_model(self, cmp: MetadataComponent) -> "JsonMetadataAttribute":
         """Converts a pysdmx metadata attribute to an SDMX-JSON one."""
         concept = _get_concept_reference(cmp)
-        repr = _get_json_representation(cmp)
+        repr = _get_attr_repr(cmp)
 
         min_occurs = cmp.array_def.min_size if cmp.array_def else 0
-        if cmp.array_def is None or cmp.array_def.max_size is None:
-            max_occurs: Union[int, Literal["unbounded"]] = "unbounded"
+        if cmp.array_def is None:
+            max_occurs: Union[int, Literal["unbounded"]] = 1
+        elif cmp.array_def.max_size is None:
+            max_occurs = "unbounded"
         else:
             max_occurs = cmp.array_def.max_size
 
@@ -95,9 +103,9 @@ class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
             minOccurs=min_occurs,
             maxOccurs=max_occurs,
             isPresentational=cmp.is_presentational,
-            metadataAttributes=[
-                JsonMetadataAttribute.from_model(c) for c in cmp.components
-            ],
+            metadataAttributes=tuple(
+                [JsonMetadataAttribute.from_model(c) for c in cmp.components]
+            ),
         )
 
 
@@ -119,9 +127,9 @@ class JsonMetadataAttributes(Struct, frozen=True, omit_defaults=True):
     ) -> "JsonMetadataAttributes":
         """Converts a pysdmx list of metadata attributes to SDMX-JSON."""
         return JsonMetadataAttributes(
-            metadataAttributes=[
-                JsonMetadataAttribute.from_model(a) for a in attributes
-            ]
+            metadataAttributes=tuple(
+                [JsonMetadataAttribute.from_model(a) for a in attributes]
+            )
         )
 
 

pysdmx/io/json/sdmxjson2/messages/schema.py

@@ -21,7 +21,7 @@ class JsonSchemas(msgspec.Struct, frozen=True, omit_defaults=True):
     dataStructures: Sequence[JsonDataStructure]
     valuelists: Sequence[JsonValuelist] = ()
     codelists: Sequence[JsonCodelist] = ()
-    contentConstraints: Sequence[JsonDataConstraint] = ()
+    dataConstraints: Sequence[JsonDataConstraint] = ()
 
     def to_model(
         self,
@@ -32,7 +32,7 @@ class JsonSchemas(msgspec.Struct, frozen=True, omit_defaults=True):
             self.conceptSchemes,
             self.codelists,
             self.valuelists,
-            self.contentConstraints,
+            self.dataConstraints,
         )
         return comps, grps  # type: ignore[return-value]
 

pysdmx/io/json/sdmxjson2/messages/structure.py

@@ -17,6 +17,7 @@ from pysdmx.io.json.sdmxjson2.messages.code import (
     JsonValuelist,
 )
 from pysdmx.io.json.sdmxjson2.messages.concept import JsonConceptScheme
+from pysdmx.io.json.sdmxjson2.messages.constraint import JsonDataConstraint
 from pysdmx.io.json.sdmxjson2.messages.core import JsonHeader
 from pysdmx.io.json.sdmxjson2.messages.dataflow import JsonDataflow
 from pysdmx.io.json.sdmxjson2.messages.dsd import JsonDataStructure
@@ -49,90 +50,95 @@ from pysdmx.model.message import StructureMessage
 class JsonStructures(Struct, frozen=True, omit_defaults=True):
     """The allowed strutures."""
 
-    dataStructures: Sequence[JsonDataStructure] = ()
+    agencySchemes: Sequence[JsonAgencyScheme] = ()
+    categorisations: Sequence[JsonCategorisation] = ()
     categorySchemes: Sequence[JsonCategoryScheme] = ()
-    conceptSchemes: Sequence[JsonConceptScheme] = ()
     codelists: Sequence[JsonCodelist] = ()
-    valueLists: Sequence[JsonValuelist] = ()
+    conceptSchemes: Sequence[JsonConceptScheme] = ()
+    customTypeSchemes: Sequence[JsonCustomTypeScheme] = ()
+    dataConstraints: Sequence[JsonDataConstraint] = ()
+    dataflows: Sequence[JsonDataflow] = ()
+    dataProviderSchemes: Sequence[JsonDataProviderScheme] = ()
+    dataStructures: Sequence[JsonDataStructure] = ()
     hierarchies: Sequence[JsonHierarchy] = ()
     hierarchyAssociations: Sequence[JsonHierarchyAssociation] = ()
-    agencySchemes: Sequence[JsonAgencyScheme] = ()
-    dataProviderSchemes: Sequence[JsonDataProviderScheme] = ()
-    metadataProviderSchemes: Sequence[JsonMetadataProviderScheme] = ()
-    dataflows: Sequence[JsonDataflow] = ()
-    provisionAgreements: Sequence[JsonProvisionAgreement] = ()
     metadataflows: Sequence[JsonMetadataflow] = ()
+    metadataProviderSchemes: Sequence[JsonMetadataProviderScheme] = ()
     metadataProvisionAgreements: Sequence[JsonMetadataProvisionAgreement] = ()
     metadataStructures: Sequence[JsonMetadataStructure] = ()
-    structureMaps: Sequence[JsonStructureMap] = ()
-    representationMaps: Sequence[JsonRepresentationMap] = ()
-    categorisations: Sequence[JsonCategorisation] = ()
-    customTypeSchemes: Sequence[JsonCustomTypeScheme] = ()
-    vtlMappingSchemes: Sequence[JsonVtlMappingScheme] = ()
     namePersonalisationSchemes: Sequence[JsonNamePersonalisationScheme] = ()
+    provisionAgreements: Sequence[JsonProvisionAgreement] = ()
+    representationMaps: Sequence[JsonRepresentationMap] = ()
     rulesetSchemes: Sequence[JsonRulesetScheme] = ()
+    structureMaps: Sequence[JsonStructureMap] = ()
     transformationSchemes: Sequence[JsonTransformationScheme] = ()
     userDefinedOperatorSchemes: Sequence[JsonUserDefinedOperatorScheme] = ()
+    valueLists: Sequence[JsonValuelist] = ()
+    vtlMappingSchemes: Sequence[JsonVtlMappingScheme] = ()
 
     def to_model(self) -> Sequence[MaintainableArtefact]:
         """Map to pysdmx artefacts."""
         structures = []  # type: ignore[var-annotated]
         structures.extend(
-            i.to_model(
-                self.conceptSchemes, self.codelists, self.valueLists, ()
-            )
-            for i in self.dataStructures
+            i.to_model(self.dataflows) for i in self.agencySchemes
         )
+        structures.extend(i.to_model() for i in self.categorisations)
         structures.extend(i.to_model() for i in self.categorySchemes)
-        structures.extend(
-            i.to_model(self.codelists) for i in self.conceptSchemes
-        )
         structures.extend(i.to_model() for i in self.codelists)
-        structures.extend(i.to_model() for i in self.valueLists)
-        structures.extend(i.to_model(self.codelists) for i in self.hierarchies)
         structures.extend(
-            i.to_model(self.hierarchies, self.codelists)
-            for i in self.hierarchyAssociations
+            i.to_model(self.codelists) for i in self.conceptSchemes
         )
+        structures.extend(i.to_model() for i in self.customTypeSchemes)
+        structures.extend(i.to_model() for i in self.dataConstraints)
         structures.extend(
-            i.to_model(self.dataflows) for i in self.agencySchemes
+            i.to_model(
+                self.dataStructures,
+                self.conceptSchemes,
+                self.valueLists,
+                self.codelists,
+            )
+            for i in self.dataflows
         )
         structures.extend(
             i.to_model(self.provisionAgreements)
             for i in self.dataProviderSchemes
         )
+        structures.extend(
+            i.to_model(
+                self.conceptSchemes, self.codelists, self.valueLists, ()
+            )
+            for i in self.dataStructures
+        )
+        structures.extend(i.to_model(self.codelists) for i in self.hierarchies)
+        structures.extend(
+            i.to_model(self.hierarchies, self.codelists)
+            for i in self.hierarchyAssociations
+        )
+
+        structures.extend(i.to_model() for i in self.metadataflows)
         structures.extend(
             i.to_model(self.metadataProvisionAgreements)
             for i in self.metadataProviderSchemes
         )
+        structures.extend(
+            i.to_model() for i in self.metadataProvisionAgreements
+        )
         structures.extend(
             i.to_model(self.conceptSchemes, self.codelists, self.valueLists)
             for i in self.metadataStructures
         )
         structures.extend(
-            i.to_model(
-                self.dataStructures,
-                self.conceptSchemes,
-                self.valueLists,
-                self.codelists,
-            )
-            for i in self.dataflows
+            i.to_model() for i in self.namePersonalisationSchemes
         )
         structures.extend(i.to_model() for i in self.provisionAgreements)
-        structures.extend(i.to_model() for i in self.metadataflows)
         structures.extend(
-            i.to_model() for i in self.metadataProvisionAgreements
+            i.to_model(bool(len(i.source) > 1 or len(i.target) > 1))
+            for i in self.representationMaps
         )
+        structures.extend(i.to_model() for i in self.rulesetSchemes)
         structures.extend(
             i.to_model(self.representationMaps) for i in self.structureMaps
         )
-        structures.extend(i.to_model() for i in self.categorisations)
-        structures.extend(i.to_model() for i in self.customTypeSchemes)
-        structures.extend(i.to_model() for i in self.vtlMappingSchemes)
-        structures.extend(
-            i.to_model() for i in self.namePersonalisationSchemes
-        )
-        structures.extend(i.to_model() for i in self.rulesetSchemes)
         structures.extend(
             i.to_model(
                 self.customTypeSchemes,
@@ -146,9 +152,8 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
         structures.extend(
             i.to_model() for i in self.userDefinedOperatorSchemes
         )
-        for rm in self.representationMaps:
-            multi = bool(len(rm.source) > 1 or len(rm.target) > 1)
-            structures.append(rm.to_model(multi))
+        structures.extend(i.to_model() for i in self.valueLists)
+        structures.extend(i.to_model() for i in self.vtlMappingSchemes)
         return structures
 
     @classmethod
@@ -264,6 +269,33 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
         hierarchies = tuple(
             [JsonHierarchy.from_model(h) for h in msg.get_hierarchies()]
         )
+        constraints = tuple(
+            [
+                JsonDataConstraint.from_model(c)
+                for c in msg.get_data_constraints()
+            ]
+        )
+        mpas = tuple(
+            [
+                JsonMetadataProvisionAgreement.from_model(c)
+                for c in msg.get_metadata_provision_agreements()
+            ]
+        )
+        mprvs = tuple(
+            [
+                JsonMetadataProviderScheme.from_model(c)
+                for c in msg.get_metadata_provider_schemes()
+            ]
+        )
+        mdfs = tuple(
+            [JsonMetadataflow.from_model(c) for c in msg.get_metadataflows()]
+        )
+        msds = tuple(
+            [
+                JsonMetadataStructure.from_model(c)
+                for c in msg.get_metadata_structures()
+            ]
+        )
         return JsonStructures(
             agencySchemes=agencies,
             categorisations=categorisations,
@@ -271,11 +303,16 @@ class JsonStructures(Struct, frozen=True, omit_defaults=True):
             codelists=codelists,
             conceptSchemes=concept_schemes,
             customTypeSchemes=custom_types,
+            dataConstraints=constraints,
             dataflows=dataflows,
             dataProviderSchemes=data_providers,
             dataStructures=data_structures,
             hierarchies=hierarchies,
             hierarchyAssociations=hier_associations,
+            metadataflows=mdfs,
+            metadataProviderSchemes=mprvs,
+            metadataProvisionAgreements=mpas,
+            metadataStructures=msds,
             namePersonalisationSchemes=name_personalisations,
             provisionAgreements=agreements,
             representationMaps=representations_maps,

pysdmx/io/json/sdmxjson2/messages/vtl.py

@@ -1,6 +1,6 @@
 """Collection of SDMX-JSON schemas for VTL artefacts."""
 
-from typing import Literal, Optional, Sequence
+from typing import Dict, Literal, Optional, Sequence
 
 from msgspec import Struct
 
@@ -506,33 +506,37 @@ class JsonRulesetScheme(ItemSchemeType, frozen=True, omit_defaults=True):
 class JsonToVtlMapping(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for To VTL mappings."""
 
-    toVtlSubSpace: Sequence[str]
-    type: Optional[str] = None
+    toVtlSubSpace: Dict[str, Sequence[str]]
+    method: Optional[str] = None
 
     def to_model(self) -> ToVtlMapping:
         """Converts deserialized class to pysdmx model class."""
-        return ToVtlMapping(self.toVtlSubSpace, self.type)
+        return ToVtlMapping(self.toVtlSubSpace["keys"], self.method)
 
     @classmethod
     def from_model(cls, mapping: ToVtlMapping) -> "JsonToVtlMapping":
         """Converts a pysdmx "to VTL" mapping to an SDMX-JSON one."""
-        return JsonToVtlMapping(mapping.to_vtl_sub_space, mapping.method)
+        return JsonToVtlMapping(
+            {"keys": mapping.to_vtl_sub_space}, mapping.method
+        )
 
 
 class JsonFromVtlMapping(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for from VTL mappings."""
 
-    fromVtlSuperSpace: Sequence[str]
-    type: Optional[str] = None
+    fromVtlSuperSpace: Dict[str, Sequence[str]]
+    method: Optional[str] = None
 
     def to_model(self) -> FromVtlMapping:
         """Converts deserialized class to pysdmx model class."""
-        return FromVtlMapping(self.fromVtlSuperSpace, self.type)
+        return FromVtlMapping(self.fromVtlSuperSpace["keys"], self.method)
 
     @classmethod
     def from_model(cls, mapping: FromVtlMapping) -> "JsonFromVtlMapping":
         """Converts a pysdmx "from VTL" mapping to an SDMX-JSON one."""
-        return JsonFromVtlMapping(mapping.from_vtl_sub_space, mapping.method)
+        return JsonFromVtlMapping(
+            {"keys": mapping.from_vtl_sub_space}, mapping.method
+        )
 
 
 class JsonVtlMapping(NameableType, frozen=True, omit_defaults=True):

pysdmx/io/json/sdmxjson2/reader/doc_validation.py

@@ -88,6 +88,10 @@ def validate_sdmx_json(input_str: str) -> None:
                     lambda m: f"does not match required"
                     f" pattern {m.group(1)!r}",
                 ),
+                (
+                    r"\[\]\s+is\s+too\s+short",
+                    lambda _m: "[] should be non-empty",
+                ),
             ]
 
             msg: Optional[str] = next(

pysdmx/io/xml/__write_data_aux.py

@@ -26,6 +26,8 @@ def check_dimension_at_observation(
     for key, value in dimension_at_observation.items():
         if key not in datasets:
             raise Invalid(f"Dataset {key} not found in Message content.")
+        if value == ALL_DIM:
+            continue
         writing_validation(datasets[key])
         dataset = datasets[key]
         components = dataset.structure.components  # type: ignore[union-attr]

pysdmx/model/__base.py

@@ -1,5 +1,6 @@
+import re
 from datetime import datetime
-from typing import Any, Optional, Sequence, Union
+from typing import Any, Literal, Optional, Sequence, Union
 
 from msgspec import Struct
 
@@ -327,6 +328,50 @@ class ItemScheme(MaintainableArtefact, frozen=True, omit_defaults=True):
     items: Sequence[Item] = ()
     is_partial: bool = False
 
+    def search(
+        self,
+        query: str,
+        use_regex: bool = False,
+        fields: Literal["name", "description", "all"] = "all",
+    ) -> Sequence[Item]:
+        """Search for items matching the query.
+
+        Args:
+            query: The substring or regex pattern to search for.
+            use_regex: Whether to treat the query as a regex (default: False).
+            fields: The fields to search in (default: all textual fields).
+
+        Returns:
+           Items that match the query.
+        """
+        if not query:
+            raise Invalid(
+                "Invalid search", "The query string cannot be empty."
+            )
+
+        # Determine which fields to search in
+        search_fields = (
+            ["name", "description"] if fields == "all" else [fields]
+        )
+
+        # Transform plain text queries into a regex
+        if not use_regex:
+            query = re.escape(query)
+
+        pattern = re.compile(query, re.IGNORECASE if not use_regex else 0)
+
+        all_items = getattr(self, "all_items", "")
+        items = all_items if all_items else self.items
+
+        return [
+            item  # type: ignore[misc]
+            for item in items
+            if any(
+                pattern.search(str(getattr(item, field, "")))
+                for field in search_fields
+            )
+        ]
+
 
 class DataflowRef(
     Struct, frozen=True, omit_defaults=True, repr_omit_defaults=True, tag=True

pysdmx/model/__init__.py

@@ -29,6 +29,16 @@ from pysdmx.model.code import (
     HierarchyAssociation,
 )
 from pysdmx.model.concept import Concept, ConceptScheme, DataType, Facets
+from pysdmx.model.constraint import (
+    ConstraintAttachment,
+    CubeKeyValue,
+    CubeRegion,
+    CubeValue,
+    DataConstraint,
+    DataKey,
+    DataKeyValue,
+    KeySet,
+)
 from pysdmx.model.dataflow import (
     ArrayBoundaries,
     Component,
@@ -161,9 +171,16 @@ __all__ = [
     "ComponentMap",
     "Concept",
     "ConceptScheme",
+    "ConstraintAttachment",
     "Contact",
+    "CubeKeyValue",
+    "CubeRegion",
+    "CubeValue",
     "DataConsumer",
     "DataConsumerScheme",
+    "DataConstraint",
+    "DataKey",
+    "DataKeyValue",
     "Dataflow",
     "DataflowInfo",
     "DataflowRef",
@@ -180,6 +197,7 @@ __all__ = [
     "HierarchyAssociation",
     "ImplicitComponentMap",
     "ItemReference",
+    "KeySet",
     "MetadataAttribute",
     "MetadataComponent",
     "Metadataflow",

pysdmx/model/category.py

@@ -91,6 +91,15 @@ class CategoryScheme(ItemScheme, frozen=True, omit_defaults=True):
             flows.update(self.__extract_flows(cat))
         return list(flows)
 
+    @property
+    def all_items(self) -> Sequence[Category]:
+        """Get all the categories in the category scheme as a flat list.
+
+        Returns:
+            A flat list of all the categories present in the category scheme.
+        """
+        return self.__get_categories(self.categories)
+
     def __iter__(self) -> Iterator[Category]:
         """Return an iterator over the list of categories."""
         yield from self.categories
@@ -160,6 +169,14 @@ class CategoryScheme(ItemScheme, frozen=True, omit_defaults=True):
             processed_output.append(f"{attr}: {value}")
         return f"{', '.join(processed_output)}"
 
+    def __get_categories(self, cats: Sequence[Category]) -> Sequence[Category]:
+        out = []
+        for cat in cats:
+            out.append(cat)
+            if cat.categories:
+                out.extend(self.__get_categories(cat.categories))
+        return out
+
 
 class Categorisation(
     MaintainableArtefact, frozen=True, omit_defaults=True, kw_only=True

pysdmx/model/constraint.py

@@ -0,0 +1,69 @@
+"""Model for SDMX Data Constraints."""
+
+from datetime import datetime
+from typing import Optional, Sequence
+
+from msgspec import Struct
+
+from pysdmx.model.__base import MaintainableArtefact
+
+
+class CubeValue(Struct, frozen=True, omit_defaults=True):
+    """A value of the cube, with optional business validity."""
+
+    value: str
+    valid_from: Optional[datetime] = None
+    valid_to: Optional[datetime] = None
+
+
+class CubeKeyValue(Struct, frozen=True, omit_defaults=True):
+    """The list of values for a cube's component."""
+
+    id: str
+    values: Sequence[CubeValue]
+
+
+class CubeRegion(Struct, frozen=True, omit_defaults=True):
+    """A cube region, with its associated values (by default, included)."""
+
+    key_values: Sequence[CubeKeyValue]
+    is_included: bool = True
+
+
+class ConstraintAttachment(Struct, frozen=True, omit_defaults=True):
+    """The artefacts to which the data constraint is attached."""
+
+    data_provider: Optional[str]
+    data_structures: Optional[Sequence[str]] = None
+    dataflows: Optional[Sequence[str]] = None
+    provision_agreements: Optional[Sequence[str]] = None
+
+
+class DataKeyValue(Struct, frozen=True, omit_defaults=True):
+    """A key value, i.e. a component of the key (e.g. FREQ=M)."""
+
+    id: str
+    value: str
+
+
+class DataKey(Struct, frozen=True, omit_defaults=True):
+    """A data key, i.e. one value per dimension in the data key."""
+
+    keys_values: Sequence[DataKeyValue]
+    valid_from: Optional[datetime] = None
+    valid_to: Optional[datetime] = None
+
+
+class KeySet(Struct, frozen=True, omit_defaults=True):
+    """A set of keys, inluded by default."""
+
+    keys: Sequence[DataKey]
+    is_included: bool
+
+
+class DataConstraint(MaintainableArtefact, frozen=True, omit_defaults=True):
+    """A data constraint, defining the allowed or available values."""
+
+    constraint_attachment: Optional[ConstraintAttachment] = None
+    cube_regions: Sequence[CubeRegion] = ()
+    key_sets: Sequence[KeySet] = ()

pysdmx/model/dataflow.py

@@ -101,10 +101,11 @@ class Component(
     one of: *D* (for Dataset), *O* (for Observation), any string identifying a
     component ID (FREQ) or comma-separated list of component IDs
     (FREQ,REF_AREA). The latter can be used to identify the dimension, group
-    or series to which the attribute is attached. The attachment level of a
-    component may vary with the statistical domain, i.e. a component attached
-    to a series in a particular domain may be attached to, say, the dataset in
-    another domain.
+    or series to which the attribute is attached. It can also be used to
+    identify the measure(s) to which the attribute relates, in case multiple
+    measures are defined. The attachment level of a component may vary with the
+    statistical domain, i.e. a component attached to a series in a particular
+    domain may be attached to, say, the dataset in another domain.
 
     The *codes* field indicates the expected (i.e. allowed) set of values a
     component can take within a particular domain. In addition to
@@ -128,7 +129,9 @@ class Component(
             Attributes can be attached at different levels such as
             D (for dataset-level attributes), O (for observation-level
             attributes) or a combination of dimension IDs, separated by
-            commas, for series- and group-level attributes).
+            commas, for series- and group-level attributes, as well as for
+            attributes attached to one or more measures, when multiple
+            measures are defined).
             A post_init check makes this attribute mandatory for attributes.
         array_def: Any additional constraints for array types.
         urn: The URN of the component.