pysdmx 1.8.0__py3-none-any.whl → 1.9.0__py3-none-any.whl

pysdmx/__extras_check.py

@@ -55,3 +55,17 @@ def __check_vtl_extra() -> None:
                 " and prettify",
             )
         ) from None
+
+
+def __check_json_extra() -> None:
+    try:
+        import jsonschema  # noqa: F401
+        import sdmxschemas  # noqa: F401
+    except ImportError:
+        raise ImportError(
+            ERROR_MESSAGE.format(
+                extra_name="json",
+                extra_desc="the validation of SDMX-JSON Structure Messages "
+                "(hint, use validate=False if you don't need validation)",
+            )
+        ) from None

pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.8.0"
+__version__ = "1.9.0"

pysdmx/io/json/fusion/messages/category.py

@@ -1,7 +1,7 @@
 """Collection of Fusion-JSON schemas for categories and category schemes."""
 
 from collections import defaultdict
-from typing import Dict, Optional, Sequence
+from typing import Dict, Optional, Sequence, Tuple, Union
 
 from msgspec import Struct
 
@@ -11,6 +11,8 @@ from pysdmx.model import (
     Agency,
     Category,
     DataflowRef,
+    ItemReference,
+    Reference,
 )
 from pysdmx.model import (
     Categorisation as CT,
@@ -21,7 +23,7 @@ from pysdmx.model import (
 from pysdmx.model import (
     Dataflow as DF,
 )
-from pysdmx.util import find_by_urn
+from pysdmx.util import find_by_urn, parse_urn
 
 
 class FusionCategorisation(Struct, frozen=True, rename={"agency": "agencyId"}):
@@ -57,14 +59,46 @@ class FusionCategory(Struct, frozen=True):
     descriptions: Optional[Sequence[FusionString]] = None
     items: Sequence["FusionCategory"] = ()
 
-    def to_model(self) -> Category:
+    def __add_flows(
+        self, cni: str, cf: Dict[str, list[DF]]
+    ) -> Sequence[DataflowRef]:
+        if cni in cf:
+            return [
+                DataflowRef(
+                    (
+                        df.agency.id
+                        if isinstance(df.agency, Agency)
+                        else df.agency
+                    ),
+                    df.id,
+                    df.version,
+                    df.name,
+                )
+                for df in cf[cni]
+            ]
+        else:
+            return ()
+
+    def to_model(
+        self,
+        cat_flows: dict[str, list[DF]],
+        cat_other: dict[str, list[Union[ItemReference, Reference]]],
+        parent_id: Optional[str] = None,
+    ) -> Category:
         """Converts a FusionCode to a standard code."""
         description = self.descriptions[0].value if self.descriptions else None
+        cni = f"{parent_id}.{self.id}" if parent_id else self.id
+        dataflows = self.__add_flows(cni, cat_flows)
+        others = cat_other.get(cni, ())
         return Category(
             id=self.id,
             name=self.names[0].value,
             description=description,
-            categories=[c.to_model() for c in self.items],
+            categories=[
+                c.to_model(cat_flows, cat_other, cni) for c in self.items
+            ],
+            dataflows=dataflows,
+            other_references=others,
         )
 
 
@@ -78,16 +112,42 @@ class FusionCategoryScheme(Struct, frozen=True, rename={"agency": "agencyId"}):
     version: str = "1.0"
     items: Sequence[FusionCategory] = ()
 
-    def to_model(self) -> CS:
+    def __group_refs(
+        self,
+        categorisations: Sequence[FusionCategorisation],
+        dataflows: Sequence[FusionDataflow],
+    ) -> Tuple[
+        dict[str, list[DF]], dict[str, list[Union[ItemReference, Reference]]]
+    ]:
+        flows: defaultdict[str, list[DF]] = defaultdict(list)
+        other: defaultdict[str, list[Union[ItemReference, Reference]]] = (
+            defaultdict(list)
+        )
+        for c in categorisations:
+            ref = parse_urn(c.structureReference)
+            src = c.categoryReference[c.categoryReference.find(")") + 2 :]
+            if ref.sdmx_type == "Dataflow":
+                d = find_by_urn(dataflows, c.structureReference)
+                flows[src].append(d.to_model())
+            else:
+                other[src].append(ref)
+        return (flows, other)
+
+    def to_model(
+        self,
+        categorisations: Sequence[FusionCategorisation] = (),
+        dataflows: Sequence[FusionDataflow] = (),
+    ) -> CS:
         """Converts a JsonCodelist to a standard codelist."""
         description = self.descriptions[0].value if self.descriptions else None
+        cat_flows, cat_others = self.__group_refs(categorisations, dataflows)
         return CS(
             id=self.id,
             name=self.names[0].value,
             agency=self.agency,
             description=description,
             version=self.version,
-            items=[c.to_model() for c in self.items],
+            items=[c.to_model(cat_flows, cat_others) for c in self.items],
         )
 
 
@@ -98,43 +158,11 @@ class FusionCategorySchemeMessage(Struct, frozen=True):
     Categorisation: Sequence[FusionCategorisation] = ()
     Dataflow: Sequence[FusionDataflow] = ()
 
-    def __group_flows(self) -> defaultdict[str, list[DF]]:
-        out: defaultdict[str, list[DF]] = defaultdict(list)
-        for c in self.Categorisation:
-            d = find_by_urn(self.Dataflow, c.structureReference)
-            src = c.categoryReference[c.categoryReference.find(")") + 2 :]
-            out[src].append(d.to_model())
-        return out
-
-    def __add_flows(
-        self, cat: Category, cni: str, cf: Dict[str, list[DF]]
-    ) -> None:
-        if cat.categories:
-            for c in cat.categories:
-                self.__add_flows(c, f"{cni}.{c.id}", cf)
-        if cni in cf:
-            dfrefs = [
-                DataflowRef(
-                    (
-                        df.agency.id
-                        if isinstance(df.agency, Agency)
-                        else df.agency
-                    ),
-                    df.id,
-                    df.version,
-                    df.name,
-                )
-                for df in cf[cni]
-            ]
-            cat.dataflows = dfrefs
-
     def to_model(self) -> CS:
         """Returns the requested category scheme."""
-        cf = self.__group_flows()
-        cs = self.CategoryScheme[0].to_model()
-        for c in cs:
-            self.__add_flows(c, c.id, cf)
-        return cs
+        return self.CategoryScheme[0].to_model(
+            self.Categorisation, self.Dataflow
+        )
 
 
 class FusionCategorisationMessage(Struct, frozen=True):

pysdmx/io/json/fusion/messages/msd.py

@@ -29,7 +29,7 @@ class FusionMetadataAttribute(Struct, frozen=True):
     id: str
     concept: str
     minOccurs: int
-    maxOccurs: Union[int, Literal["unbounded"]]
+    maxOccurs: Union[int, Literal["unbounded"]] = "unbounded"
     presentational: Optional[bool] = False
     representation: Optional[FusionRepresentation] = None
     metadataAttributes: Sequence["FusionMetadataAttribute"] = ()

pysdmx/io/json/sdmxjson2/messages/category.py

@@ -1,7 +1,7 @@
 """Collection of SDMX-JSON schemas for categories and category schemes."""
 
 from collections import defaultdict
-from typing import Dict, Sequence
+from typing import Dict, Optional, Sequence, Tuple, Union
 
 from msgspec import Struct
 
@@ -20,8 +20,10 @@ from pysdmx.model import (
     CategoryScheme,
     Dataflow,
     DataflowRef,
+    ItemReference,
+    Reference,
 )
-from pysdmx.util import find_by_urn
+from pysdmx.util import find_by_urn, parse_urn
 
 
 class JsonCategorisation(
@@ -84,14 +86,46 @@ class JsonCategory(NameableType, frozen=True, omit_defaults=True):
 
     categories: Sequence["JsonCategory"] = ()
 
-    def to_model(self) -> Category:
+    def __add_flows(
+        self, cni: str, cf: Dict[str, list[Dataflow]]
+    ) -> Sequence[DataflowRef]:
+        if cni in cf:
+            return [
+                DataflowRef(
+                    (
+                        df.agency.id
+                        if isinstance(df.agency, Agency)
+                        else df.agency
+                    ),
+                    df.id,
+                    df.version,
+                    df.name,
+                )
+                for df in cf[cni]
+            ]
+        else:
+            return ()
+
+    def to_model(
+        self,
+        cat_flows: dict[str, list[Dataflow]],
+        cat_other: dict[str, list[Union[ItemReference, Reference]]],
+        parent_id: Optional[str] = None,
+    ) -> Category:
         """Converts a FusionCode to a standard code."""
+        cni = f"{parent_id}.{self.id}" if parent_id else self.id
+        dataflows = self.__add_flows(cni, cat_flows)
+        others = cat_other.get(cni, ())
         return Category(
             id=self.id,
             name=self.name,
             description=self.description,
-            categories=[c.to_model() for c in self.categories],
+            categories=[
+                c.to_model(cat_flows, cat_other, cni) for c in self.categories
+            ],
             annotations=[a.to_model() for a in self.annotations],
+            dataflows=dataflows,
+            other_references=others,
         )
 
     @classmethod
@@ -126,15 +160,42 @@ class JsonCategoryScheme(
 
     categories: Sequence[JsonCategory] = ()
 
-    def to_model(self) -> CategoryScheme:
-        """Converts a JsonCodelist to a standard codelist."""
+    def __group_refs(
+        self,
+        categorisations: Sequence[JsonCategorisation] = (),
+        dataflows: Sequence[JsonDataflow] = (),
+    ) -> Tuple[
+        dict[str, list[Dataflow]],
+        dict[str, list[Union[ItemReference, Reference]]],
+    ]:
+        flows: defaultdict[str, list[Dataflow]] = defaultdict(list)
+        other: defaultdict[str, list[Union[ItemReference, Reference]]] = (
+            defaultdict(list)
+        )
+        for c in categorisations:
+            ref = parse_urn(c.source)
+            src = c.target[c.target.find(")") + 2 :]
+            if ref.sdmx_type == "Dataflow":
+                d = find_by_urn(dataflows, c.source)
+                flows[src].append(d.to_model())
+            else:
+                other[src].append(ref)
+        return (flows, other)
+
+    def to_model(
+        self,
+        categorisations: Sequence[JsonCategorisation] = (),
+        dataflows: Sequence[JsonDataflow] = (),
+    ) -> CategoryScheme:
+        """Converts a JsonCategoryScheme to a standard one."""
+        cat_flows, cat_other = self.__group_refs(categorisations, dataflows)
         return CategoryScheme(
             id=self.id,
             name=self.name,
             agency=self.agency,
             description=self.description,
             version=self.version,
-            items=[c.to_model() for c in self.categories],
+            items=[c.to_model(cat_flows, cat_other) for c in self.categories],
             is_external_reference=self.isExternalReference,
             is_partial=self.isPartial,
             valid_from=self.validFrom,
@@ -179,49 +240,21 @@ class JsonCategorySchemes(Struct, frozen=True, omit_defaults=True):
     categorisations: Sequence[JsonCategorisation] = ()
     dataflows: Sequence[JsonDataflow] = ()
 
+    def to_model(self) -> CategoryScheme:
+        """Returns the requested codelist."""
+        return self.categorySchemes[0].to_model(
+            self.categorisations, self.dataflows
+        )
+
 
 class JsonCategorySchemeMessage(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for /categoryscheme queries."""
 
     data: JsonCategorySchemes
 
-    def __group_flows(self) -> defaultdict[str, list[Dataflow]]:
-        out: defaultdict[str, list[Dataflow]] = defaultdict(list)
-        for c in self.data.categorisations:
-            d = find_by_urn(self.data.dataflows, c.source)
-            src = c.target[c.target.find(")") + 2 :]
-            out[src].append(d.to_model())
-        return out
-
-    def __add_flows(
-        self, cat: Category, cni: str, cf: Dict[str, list[Dataflow]]
-    ) -> None:
-        if cat.categories:
-            for c in cat.categories:
-                self.__add_flows(c, f"{cni}.{c.id}", cf)
-        if cni in cf:
-            dfrefs = [
-                DataflowRef(
-                    (
-                        df.agency.id
-                        if isinstance(df.agency, Agency)
-                        else df.agency
-                    ),
-                    df.id,
-                    df.version,
-                    df.name,
-                )
-                for df in cf[cni]
-            ]
-            cat.dataflows = dfrefs
-
     def to_model(self) -> CategoryScheme:
-        """Returns the requested codelist."""
-        cf = self.__group_flows()
-        cs = self.data.categorySchemes[0].to_model()
-        for c in cs:
-            self.__add_flows(c, c.id, cf)
-        return cs
+        """Returns the requested category scheme."""
+        return self.data.to_model()
 
 
 class JsonCategorisations(Struct, frozen=True, omit_defaults=True):

pysdmx/io/json/sdmxjson2/messages/core.py

@@ -299,6 +299,7 @@ class JsonHeader(msgspec.Struct, frozen=True, omit_defaults=True):
             test=self.test,
             prepared=self.prepared,
             sender=self.sender,
+            receiver=self.receivers if self.receivers else (),
         )
 
     @classmethod
@@ -313,7 +314,7 @@ class JsonHeader(msgspec.Struct, frozen=True, omit_defaults=True):
             header.prepared,
             header.sender,
             header.test,
-            receivers=(header.receiver,) if header.receiver else None,
+            receivers=header.receiver if header.receiver else None,
             schema=(
                 "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/"
                 "develop/structure-message/tools/schemas/2.0.0/"

pysdmx/io/json/sdmxjson2/messages/msd.py

@@ -37,6 +37,7 @@ class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
     maxOccurs: Union[int, Literal["unbounded"]]
     isPresentational: bool
     localRepresentation: Optional[JsonRepresentation] = None
+    metadataAttributes: Sequence["JsonMetadataAttribute"] = ()
 
     def to_model(
         self, cs: Sequence[JsonConceptScheme], cls: Sequence[Codelist]
@@ -72,7 +73,7 @@ class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
             local_codes=codes,
             array_def=ab,
             local_enum_ref=local_enum_ref,
-            components=(),
+            components=[a.to_model(cs, cls) for a in self.metadataAttributes],
         )
 
     @classmethod
@@ -94,6 +95,9 @@ class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
             minOccurs=min_occurs,
             maxOccurs=max_occurs,
             isPresentational=cmp.is_presentational,
+            metadataAttributes=[
+                JsonMetadataAttribute.from_model(c) for c in cmp.components
+            ],
         )
 
 

pysdmx/io/json/sdmxjson2/reader/doc_validation.py

@@ -0,0 +1,108 @@
+"""SDMX-JSON document validation against JSON schemas."""
+
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Mapping, Match, Optional
+
+from jsonschema import Draft202012Validator
+from jsonschema.exceptions import ValidationError
+from sdmxschemas import SDMX_JSON_20_DATA_PATH as SCHEMA_PATH_JSON20_DATA
+from sdmxschemas import (
+    SDMX_JSON_20_METADATA_PATH as SCHEMA_PATH_JSON20_METADATA,
+)
+from sdmxschemas import (
+    SDMX_JSON_20_STRUCTURE_PATH as SCHEMA_PATH_JSON20_STRUCTURE,
+)
+
+from pysdmx import errors
+
+_SCHEMA_FILES: Mapping[str, Path] = {
+    "structure": SCHEMA_PATH_JSON20_STRUCTURE,
+    "metadata": SCHEMA_PATH_JSON20_METADATA,
+    "data": SCHEMA_PATH_JSON20_DATA,
+}
+
+
+def _schema_for(instance: Mapping[str, Any]) -> dict[str, Any]:
+    schema_url = instance.get("meta", {}).get("schema")
+    p = next(p for p in _SCHEMA_FILES.values() if p.name in schema_url)
+    with p.open("r", encoding="utf-8") as f:
+        schema = json.load(f)
+    return schema
+
+
+def validate_sdmx_json(input_str: str) -> None:
+    """Validates an SDMX-JSON message against the appropriate JSON schema.
+
+    Args: input_str: The SDMX-JSON message to validate.
+    Raises:
+        invalid: If the SDMX-JSON message does not validate against the schema.
+
+    """
+    instance = json.loads(input_str)
+    schema = _schema_for(instance)
+    validator = Draft202012Validator(schema)
+
+    failures = sorted(
+        validator.iter_errors(instance),
+        key=lambda e: (list(e.path), e.message),
+    )
+    if failures:
+
+        def compact(e: ValidationError) -> str:
+            path = "$" if not e.path else "$." + ".".join(map(str, e.path))
+            sub = " | ".join(
+                getattr(e, "context", [])
+                and [c.message for c in e.context]
+                or []
+            )
+            raw = f"{e.message} | {sub}"
+
+            patterns: list[tuple[str, Callable[[Match[str]], str]]] = [
+                (
+                    r"Additional properties are not allowed.*'([^']+)'",
+                    lambda m: f"unexpected property '{m.group(1)}'",
+                ),
+                (
+                    r"is not of type '([^']+)'",
+                    lambda m: f"invalid type (expected {m.group(1)})",
+                ),
+                (
+                    r"""['"]?([^'"\n]+)['"]?\s+is not one of\s+\[([^\]]+)\]""",
+                    lambda m: "invalid value {!r}"
+                    " (expected one of: {})".format(
+                        m.group(1),
+                        ", ".join(
+                            s.strip().strip("'\"")
+                            for s in m.group(2).split(",")
+                        ),
+                    ),
+                ),
+                (
+                    r"'([^']+)' is a required property",
+                    lambda m: f"missing property '{m.group(1)}'",
+                ),
+                (
+                    r"""does not match ['"]([^'"]+)['"]""",
+                    lambda m: f"does not match required"
+                    f" pattern {m.group(1)!r}",
+                ),
+            ]
+
+            msg: Optional[str] = next(
+                (
+                    fmt(re.search(rx, raw))  # type: ignore[arg-type]
+                    for rx, fmt in patterns
+                    if re.search(rx, raw)
+                ),
+                None,
+            )
+            msg = msg or e.message
+            return f"{path}: {msg}"
+
+        summary = "; ".join(compact(e) for e in failures[:3])
+        more = (
+            f" (+{len(failures) - 3} more errors)" if len(failures) > 3 else ""
+        )
+        raise errors.Invalid("Validation Error", f"{summary}{more}")

pysdmx/io/json/sdmxjson2/reader/metadata.py

@@ -3,20 +3,27 @@
 import msgspec
 
 from pysdmx import errors
+from pysdmx.__extras_check import __check_json_extra
 from pysdmx.io.json.sdmxjson2.messages import JsonMetadataMessage
+from pysdmx.io.json.sdmxjson2.reader.doc_validation import validate_sdmx_json
 from pysdmx.model import decoders
 from pysdmx.model.message import MetadataMessage
 
 
-def read(input_str: str) -> MetadataMessage:
+def read(input_str: str, validate: bool = True) -> MetadataMessage:
     """Read an SDMX-JSON 2.0.0 Metadata Message.
 
     Args:
         input_str: SDMX-JSON reference metadata message to read.
+        validate: If True, the JSON data will be validated against the schemas.
 
     Returns:
         A pysdmx MetadataMessage
     """
+    if validate:
+        __check_json_extra()
+        validate_sdmx_json(input_str)
+
     try:
         msg = msgspec.json.Decoder(
             JsonMetadataMessage, dec_hook=decoders

pysdmx/io/json/sdmxjson2/reader/structure.py

@@ -3,20 +3,27 @@
 import msgspec
 
 from pysdmx import errors
+from pysdmx.__extras_check import __check_json_extra
 from pysdmx.io.json.sdmxjson2.messages import JsonStructureMessage
+from pysdmx.io.json.sdmxjson2.reader.doc_validation import validate_sdmx_json
 from pysdmx.model import decoders
 from pysdmx.model.message import StructureMessage
 
 
-def read(input_str: str) -> StructureMessage:
-    """Read an SDMX-JSON 2.0.0 Stucture Message.
+def read(input_str: str, validate: bool = True) -> StructureMessage:
+    """Read an SDMX-JSON 2.0.0 Structure Message.
 
     Args:
         input_str: SDMX-JSON structure message to read.
+        validate: If True, the JSON data will be validated against the schemas.
 
     Returns:
         A pysdmx StructureMessage
     """
+    if validate:
+        __check_json_extra()
+        validate_sdmx_json(input_str)
+
     try:
         msg = msgspec.json.Decoder(
             JsonStructureMessage, dec_hook=decoders

pysdmx/io/reader.py

@@ -26,11 +26,18 @@ def read_sdmx(  # noqa: C901
 
     Check the :ref:`formats supported <io-reader-formats-supported>`
 
+    .. important::
+        When reading a SDMX-JSON structure message, you can read it
+        without installing the pysdmx[json] extra
+        by passing the parameter ``validate=False`` to this function.
+        Otherwise, if not installed, an error will be raised
+        if using ``validate=True``, as it is the default value.
+
     Args:
         sdmx_document: Path to file
           (`pathlib.Path <https://docs.python.org/3/library/pathlib.html>`_),
           URL, or string.
-        validate: Validate the input file (only for SDMX-ML).
+        validate: Validate the input file (only for SDMX-ML and SDMX-JSON).
         pem: When using a URL, in case the service exposed
           a certificate created by an unknown certificate
           authority, you can pass a PEM file for this
@@ -78,7 +85,7 @@ def read_sdmx(  # noqa: C901
             read as read_struct,
         )
 
-        struct_msg = read_struct(input_str)
+        struct_msg = read_struct(input_str, validate=validate)
         header = struct_msg.header
         result_structures = (
             struct_msg.structures if struct_msg.structures else []
@@ -88,7 +95,7 @@ def read_sdmx(  # noqa: C901
             read as read_refmeta,
         )
 
-        ref_msg = read_refmeta(input_str)
+        ref_msg = read_refmeta(input_str, validate=validate)
         header = ref_msg.header
         reports = ref_msg.get_reports()
     elif read_format == Format.DATA_SDMX_ML_2_1_GEN:
@@ -230,6 +237,13 @@ def get_datasets(
       not equal to AllDimensions or Generic formats
     - Execution of VTL scripts over PandasDataset
 
+    .. important::
+        When reading a SDMX-JSON structure message (as the structure argument),
+        you can read it without installing the pysdmx[json] extra
+        by passing the parameter ``validate=False`` to this function.
+        Otherwise, if not installed, an error will be raised
+        if using ``validate=True``, as it is the default value.
+
     Args:
         data: Path to file
           (`pathlib.Path <https://docs.python.org/3/library/pathlib.html>`_),
@@ -238,7 +252,7 @@ def get_datasets(
           Path to file
           (`pathlib.Path <https://docs.python.org/3/library/pathlib.html>`_),
           URL, or string for the structure message, if needed.
-        validate: Validate the input file (only for SDMX-ML).
+        validate: Validate the input file (only for SDMX-ML and SDMX-JSON).
         pem: When using a URL, in case the service exposed
             a certificate created by an unknown certificate
             authority, you can pass a PEM file for this

pysdmx/io/xml/__data_aux.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 import pandas as pd
 
-from pysdmx.errors import Invalid, NotImplemented
+from pysdmx.errors import Invalid
 from pysdmx.io.xml.__tokens import (
     AGENCY_ID,
     DATASET,
@@ -10,6 +10,7 @@ from pysdmx.io.xml.__tokens import (
     HEADER,
     ID,
     PROV_AGREEMENT,
+    PROV_AGREMENT,
     REF,
     STR_ID,
     STR_REF,
@@ -92,9 +93,13 @@ def __get_elements_from_structure(structure: Dict[str, Any]) -> Any:
         structure_type = "ProvisionAgreement"
         tuple_ids = __get_ids_from_structure(structure[PROV_AGREEMENT])
     else:
-        raise NotImplemented(
-            "Unsupported", "ProvisionAgrement not implemented"
-        )
+        # This section handles ProvisionAgrement.
+        # IMPORTANT: This is a typo in the SDMX standard XML schema 2.1
+        # fixed in 3.0 version.
+        # We intentionally read that exact key to be compatible
+        # with such files.
+        structure_type = "ProvisionAgreement"
+        tuple_ids = __get_ids_from_structure(structure[PROV_AGREMENT])
     return tuple_ids + (structure_type,)