pysdmx 1.5.2__py3-none-any.whl → 1.7.0__py3-none-any.whl

pysdmx/io/json/sdmxjson2/reader/metadata.py

@@ -0,0 +1,32 @@
+"""Writer interface for SDMX-JSON 2.0.0 Reference Metadata messages."""
+
+import msgspec
+
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages import JsonMetadataMessage
+from pysdmx.model import decoders
+from pysdmx.model.message import MetadataMessage
+
+
+def read(input_str: str) -> MetadataMessage:
+    """Read an SDMX-JSON 2.0.0 Metadata Message.
+
+    Args:
+        input_str: SDMX-JSON reference metadata message to read.
+
+    Returns:
+        A pysdmx MetadataMessage
+    """
+    try:
+        msg = msgspec.json.Decoder(
+            JsonMetadataMessage, dec_hook=decoders
+        ).decode(input_str)
+        return msg.to_model()
+    except msgspec.DecodeError as de:
+        raise errors.Invalid(
+            "Invalid message",
+            (
+                "The supplied file could not be read as SDMX-JSON 2.0.0 "
+                "reference metadata message."
+            ),
+        ) from de

pysdmx/io/json/sdmxjson2/reader/structure.py

@@ -0,0 +1,32 @@
+"""Reader interface for SDMX-JSON 2.0.0 Structure messages."""
+
+import msgspec
+
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages import JsonStructureMessage
+from pysdmx.model import decoders
+from pysdmx.model.message import StructureMessage
+
+
+def read(input_str: str) -> StructureMessage:
+    """Read an SDMX-JSON 2.0.0 Stucture Message.
+
+    Args:
+        input_str: SDMX-JSON structure message to read.
+
+    Returns:
+        A pysdmx StructureMessage
+    """
+    try:
+        msg = msgspec.json.Decoder(
+            JsonStructureMessage, dec_hook=decoders
+        ).decode(input_str)
+        return msg.to_model()
+    except msgspec.DecodeError as de:
+        raise errors.Invalid(
+            "Invalid message",
+            (
+                "The supplied file could not be read as SDMX-JSON 2.0.0 "
+                "structure message."
+            ),
+        ) from de

pysdmx/io/json/sdmxjson2/writer/__init__.py

@@ -0,0 +1,9 @@
+"""Collection of writers for SDMX-JSON messages."""
+
+from pysdmx.io.json.sdmxjson2 import messages as msg
+from pysdmx.io.serde import Serializers
+
+serializers = Serializers(
+    metadata_message=msg.JsonMetadataMessage,
+    structure_message=msg.JsonStructureMessage,
+)

pysdmx/io/json/sdmxjson2/writer/metadata.py

@@ -0,0 +1,60 @@
+"""Writer interface for SDMX-JSON 2.0.0 Reference Metadata messages."""
+
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+import msgspec
+
+from pysdmx.io.json.sdmxjson2.messages import JsonMetadataMessage
+from pysdmx.model import MetadataReport, encoders
+from pysdmx.model.message import Header, MetadataMessage
+
+
+def write(
+    reports: Sequence[MetadataReport],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+) -> Optional[str]:
+    """Write metadata reports in SDMX-JSON 2.0.0.
+
+    Args:
+        reports: The reference metadata reports to be serialized.
+        output_path: The path to save the JSON file. If None or empty, the
+            serialized content is returned as a string instead.
+        prettyprint: Whether to format the JSON output with indentation (True)
+            or output compact JSON without extra whitespace (False).
+        header: The header to be used in the SDMX-JSON message
+            (will be generated if no header is supplied).
+
+    Returns:
+        The JSON string if output_path is None or empty, None otherwise.
+    """
+    if not header:
+        header = Header()
+    sm = MetadataMessage(header, reports)
+    jsm = JsonMetadataMessage.from_model(sm)
+
+    encoder = msgspec.json.Encoder(enc_hook=encoders)
+    serialized_data = encoder.encode(jsm)
+
+    # Apply pretty-printing if requested
+    if prettyprint:
+        serialized_data = msgspec.json.format(serialized_data, indent=4)
+
+    # If output_path is provided, write to file
+    if output_path:
+        # Convert to Path object if string
+        if isinstance(output_path, str):
+            output_path = Path(output_path)
+
+        # Create parent directories if they don't exist
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to file
+        with open(output_path, "wb") as f:
+            f.write(serialized_data)
+        return None
+    else:
+        # Return as string
+        return serialized_data.decode("utf-8")

pysdmx/io/json/sdmxjson2/writer/structure.py

@@ -0,0 +1,61 @@
+"""Writer interface for SDMX-JSON 2.0.0 Structure messages."""
+
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+import msgspec
+
+from pysdmx.io.json.sdmxjson2.messages import JsonStructureMessage
+from pysdmx.model import encoders
+from pysdmx.model.__base import MaintainableArtefact
+from pysdmx.model.message import Header, StructureMessage
+
+
+def write(
+    structures: Sequence[MaintainableArtefact],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+) -> Optional[str]:
+    """Write maintainable SDMX artefacts in SDMX-JSON 2.0.0.
+
+    Args:
+        structures: The maintainable SDMX artefacts to be serialized.
+        output_path: The path to save the JSON file. If None or empty, the
+            serialized content is returned as a string instead.
+        prettyprint: Whether to format the JSON output with indentation (True)
+            or output compact JSON without extra whitespace (False).
+        header: The header to be used in the SDMX-JSON message
+            (will be generated if no header is supplied).
+
+    Returns:
+        The JSON string if output_path is None or empty, None otherwise.
+    """
+    if not header:
+        header = Header()
+    sm = StructureMessage(header, structures)
+    jsm = JsonStructureMessage.from_model(sm)
+
+    encoder = msgspec.json.Encoder(enc_hook=encoders)
+    serialized_data = encoder.encode(jsm)
+
+    # Apply pretty-printing if requested
+    if prettyprint:
+        serialized_data = msgspec.json.format(serialized_data, indent=4)
+
+    # If output_path is provided, write to file
+    if output_path:
+        # Convert to Path object if string
+        if isinstance(output_path, str):
+            output_path = Path(output_path)
+
+        # Create parent directories if they don't exist
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to file
+        with open(output_path, "wb") as f:
+            f.write(serialized_data)
+        return None
+    else:
+        # Return as string
+        return serialized_data.decode("utf-8")

pysdmx/io/reader.py

@@ -8,10 +8,10 @@ from pysdmx.errors import Invalid
 from pysdmx.io.format import Format
 from pysdmx.io.input_processor import process_string_to_read
 from pysdmx.model import Schema
-from pysdmx.model.__base import ItemScheme
-from pysdmx.model.dataflow import Dataflow, DataStructureDefinition
+from pysdmx.model.__base import MaintainableArtefact
 from pysdmx.model.dataset import Dataset
 from pysdmx.model.message import Message
+from pysdmx.model.metadata import MetadataReport
 from pysdmx.model.submission import SubmissionResult
 from pysdmx.util import parse_short_urn
 from pysdmx.util._model_utils import schema_generator
@@ -43,10 +43,9 @@ def read_sdmx(  # noqa: C901
 
     header = None
     result_data: Sequence[Dataset] = []
-    result_structures: Sequence[
-        Union[ItemScheme, Dataflow, DataStructureDefinition]
-    ] = []
+    result_structures: Sequence[MaintainableArtefact] = []
     result_submission: Sequence[SubmissionResult] = []
+    reports: Sequence[MetadataReport] = []
     if read_format == Format.STRUCTURE_SDMX_ML_2_1:
         from pysdmx.io.xml.header import read as read_header
         from pysdmx.io.xml.sdmx21.reader.structure import (
@@ -74,6 +73,24 @@ def read_sdmx(  # noqa: C901
         header = read_header(input_str, validate=validate)
         # SDMX-ML 3.1 Structure
         result_structures = read_structure(input_str, validate=validate)
+    elif read_format == Format.STRUCTURE_SDMX_JSON_2_0_0:
+        from pysdmx.io.json.sdmxjson2.reader.structure import (
+            read as read_struct,
+        )
+
+        struct_msg = read_struct(input_str)
+        header = struct_msg.header
+        result_structures = (
+            struct_msg.structures if struct_msg.structures else []
+        )
+    elif read_format == Format.REFMETA_SDMX_JSON_2_0_0:
+        from pysdmx.io.json.sdmxjson2.reader.metadata import (
+            read as read_refmeta,
+        )
+
+        ref_msg = read_refmeta(input_str)
+        header = ref_msg.header
+        reports = ref_msg.get_reports()
     elif read_format == Format.DATA_SDMX_ML_2_1_GEN:
         from pysdmx.io.xml.header import read as read_header
         from pysdmx.io.xml.sdmx21.reader.generic import read as read_generic
@@ -127,18 +144,19 @@ def read_sdmx(  # noqa: C901
         # SDMX-CSV 1.0
         result_data = read_csv_v1(input_str)
     else:
-        # SDMX-CSV 2.0
-        from pysdmx.io.csv.sdmx20.reader import read as read_csv_v2
+        # SDMX-CSV 2.1
+        from pysdmx.io.csv.sdmx21.reader import read as read_csv_v2
 
         result_data = read_csv_v2(input_str)
 
-    if not (result_data or result_structures or result_submission):
+    if not (result_data or result_structures or result_submission or reports):
         raise Invalid("Empty SDMX Message")
 
     # Returning a Message class
     if read_format in (
         Format.DATA_SDMX_CSV_1_0_0,
         Format.DATA_SDMX_CSV_2_0_0,
+        Format.DATA_SDMX_CSV_2_1_0,
         Format.DATA_SDMX_ML_2_1_GEN,
         Format.DATA_SDMX_ML_2_1_STR,
         Format.DATA_SDMX_ML_3_0,
@@ -149,7 +167,8 @@ def read_sdmx(  # noqa: C901
         return Message(header=header, data=result_data)
     elif read_format == Format.REGISTRY_SDMX_ML_2_1:
         return Message(header=header, submission=result_submission)
-
+    elif read_format == Format.REFMETA_SDMX_JSON_2_0_0:
+        return Message(header=header, reports=reports)
     # TODO: Ensure we have changed the signature of the structure readers
     return Message(header=header, structures=result_structures)
 

pysdmx/io/serde.py

@@ -12,6 +12,15 @@ class Deserializer(Protocol):
         """Returns the domain objects."""
 
 
+@runtime_checkable
+class Serializer(Protocol):
+    """Creates an SDMX message from domain objects."""
+
+    @classmethod
+    def from_model(self, message: Any) -> Any:
+        """Returns the SDMX message."""
+
+
 @dataclass
 class Deserializers:
     """Collection of deserializers for a format."""
@@ -34,6 +43,14 @@ class Deserializers:
     transformation_scheme: Deserializer
 
 
+@dataclass
+class Serializers:
+    """Collection of serializers for a format."""
+
+    structure_message: Serializer
+    metadata_message: Serializer
+
+
 @dataclass
 class GdsDeserializers:
     """Collection of GDS deserializers for a format."""

pysdmx/io/writer.py

@@ -7,11 +7,14 @@ from typing import Any, Optional, Sequence
 
 from pysdmx.errors import Invalid
 from pysdmx.io.format import Format
+from pysdmx.model import MetadataReport
+from pysdmx.model.__base import MaintainableArtefact
 from pysdmx.model.dataset import Dataset
 
 WRITERS = {
     Format.DATA_SDMX_CSV_1_0_0: "pysdmx.io.csv.sdmx10.writer",
     Format.DATA_SDMX_CSV_2_0_0: "pysdmx.io.csv.sdmx20.writer",
+    Format.DATA_SDMX_CSV_2_1_0: "pysdmx.io.csv.sdmx21.writer",
     Format.DATA_SDMX_ML_2_1_GEN: "pysdmx.io.xml.sdmx21.writer.generic",
     Format.DATA_SDMX_ML_2_1_STR: "pysdmx.io.xml.sdmx21.writer."
     "structure_specific",
@@ -22,14 +25,23 @@ WRITERS = {
     Format.DATA_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer."
     "structure_specific",
     Format.STRUCTURE_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer.structure",
+    Format.STRUCTURE_SDMX_JSON_2_0_0: (
+        "pysdmx.io.json.sdmxjson2.writer.structure"
+    ),
+    Format.REFMETA_SDMX_JSON_2_0_0: (
+        "pysdmx.io.json.sdmxjson2.writer.metadata"
+    ),
 }
 
 STRUCTURE_WRITERS = (
     Format.STRUCTURE_SDMX_ML_2_1,
     Format.STRUCTURE_SDMX_ML_3_0,
     Format.STRUCTURE_SDMX_ML_3_1,
+    Format.STRUCTURE_SDMX_JSON_2_0_0,
 )
 
+REFMETA_WRITERS = (Format.REFMETA_SDMX_JSON_2_0_0,)
+
 
 def write_sdmx(
     sdmx_objects: Any,
@@ -97,20 +109,36 @@ def write_sdmx(
     writer = module.write
 
     is_structure = sdmx_format in STRUCTURE_WRITERS
+    is_ref_meta = sdmx_format in REFMETA_WRITERS
     is_xml = "xml" in WRITERS[sdmx_format]
-    key = "structures" if is_structure else "datasets"
+    is_json = "json" in WRITERS[sdmx_format]
+    if is_structure:
+        key = "structures"
+    elif is_ref_meta:
+        key = "reports"
+    else:
+        key = "datasets"
     value = sdmx_objects if isinstance(sdmx_objects, list) else [sdmx_objects]
 
-    if is_structure and any(isinstance(x, Dataset) for x in value):
+    if is_structure and not all(
+        isinstance(x, MaintainableArtefact) for x in value
+    ):
         raise Invalid(
-            "Datasets cannot be written to structure formats. "
-            "Use data formats instead."
+            "Only maintainable artefacts can be written to structure formats."
         )
-    elif not is_structure and not all(isinstance(x, Dataset) for x in value):
+    elif is_ref_meta and not all(isinstance(x, MetadataReport) for x in value):
         raise Invalid(
-            "Only Datasets can be written to data formats. "
-            "Use structure formats for other SDMX objects."
+            (
+                "Only metadata reports can be written to reference "
+                "metadata formats."
+            )
         )
+    elif (
+        not is_structure
+        and not is_ref_meta
+        and not all(isinstance(x, Dataset) for x in value)
+    ):
+        raise Invalid("Only Datasets can be written to data formats.")
 
     args = {
         key: value,
@@ -120,7 +148,7 @@ def write_sdmx(
                 "prettyprint": kwargs.get("prettyprint"),
                 "header": kwargs.get("header"),
             }
-            if is_xml
+            if is_xml or is_json
             else {}
         ),
         **(
@@ -132,7 +160,15 @@ def write_sdmx(
             if is_xml and not is_structure
             else {}
         ),
+        **(
+            {
+                "labels": kwargs.get("labels"),
+                "keys": kwargs.get("keys"),
+                "time_format": kwargs.get("time_format"),
+            }
+            if not is_xml
+            else {}
+        ),
     }
     args = {k: v for k, v in args.items() if v is not None}
-
     return writer(**args)

pysdmx/io/xml/__ss_aux_reader.py

@@ -30,8 +30,7 @@ def _reading_str_series(dataset: Dict[str, Any]) -> pd.DataFrame:
         if OBS in data:
             del keys[OBS]
             data[OBS] = add_list(data[OBS])
-            for j in data[OBS]:
-                test_list.append({**keys, **j})
+            test_list.extend([{**keys, **j} for j in data[OBS]])
         else:
             test_list.append(keys)
         test_list, df = __process_df(test_list, df)

pysdmx/io/xml/__structure_aux_reader.py

@@ -397,7 +397,7 @@ class StructureParser(Struct):
         """
         if json_fac is None:
             return
-        for key, _value in json_fac.items():
+        for key in json_fac:
             if key == TEXT_TYPE and json_fac[TEXT_TYPE] in list(DataType):
                 json_obj["dtype"] = DataType(json_fac[TEXT_TYPE])
 
@@ -907,9 +907,9 @@ class StructureParser(Struct):
         item_json_info = self.__format_name_description(item_json_info)
         if CONTACT in item_json_info and item_name_class == AGENCY:
             item_json_info[CONTACT] = add_list(item_json_info[CONTACT])
-            contacts = []
-            for e in item_json_info[CONTACT]:
-                contacts.append(self.__format_contact(e))
+            contacts = [
+                self.__format_contact(e) for e in item_json_info[CONTACT]
+            ]
             item_json_info[CONTACT.lower() + "s"] = contacts
             del item_json_info[CONTACT]
 
@@ -949,9 +949,11 @@ class StructureParser(Struct):
                         group_dimensions = [group_dimensions]
 
                     group["dimensions"] = [
-                        d[DIM_REF]
-                        if isinstance(d[DIM_REF], str)
-                        else d[DIM_REF][REF][ID]
+                        (
+                            d[DIM_REF]
+                            if isinstance(d[DIM_REF], str)
+                            else d[DIM_REF][REF][ID]
+                        )
                         for d in group_dimensions
                     ]
 
@@ -994,9 +996,12 @@ class StructureParser(Struct):
             items = []
             if item in element:
                 element[item] = add_list(element[item])
-                for item_elem in element[item]:
-                    # Dynamic
-                    items.append(self.__format_item(item_elem, item))
+                items.extend(
+                    [
+                        self.__format_item(item_elem, item)
+                        for item_elem in element[item]
+                    ]
+                )
                 del element[item]
             element["items"] = items
             element = self.__format_agency(element)

pysdmx/io/xml/__structure_aux_writer.py

@@ -310,9 +310,9 @@ def __write_maintainable(
         f"{str(maintainable.is_external_reference).lower()!r}"
     )
     if not references_30 and not (isinstance(maintainable, AgencyScheme)):
-        outfile["Attributes"] += (
-            f" isFinal={str(maintainable.is_final).lower()!r}"
-        )
+        outfile[
+            "Attributes"
+        ] += f" isFinal={str(maintainable.is_final).lower()!r}"
 
     if isinstance(maintainable.agency, str):
         outfile["Attributes"] += f" agencyID={maintainable.agency!r}"
@@ -462,7 +462,7 @@ def __write_components(  # noqa: C901
         )
 
     position = 1
-    for _, comps in components.items():
+    for comps in components.values():
         if comps:
             role_name = ROLE_MAPPING[comps[0].role]
             if role_name == MEASURE:
@@ -809,9 +809,9 @@ def __write_scheme(  # noqa: C901
         DSD,
         DFW,
     ]:
-        data["Attributes"] += (
-            f" isPartial={str(item_scheme.is_partial).lower()!r}"
-        )
+        data[
+            "Attributes"
+        ] += f" isPartial={str(item_scheme.is_partial).lower()!r}"
     if scheme in [
         RULE_SCHEME,
         UDO_SCHEME,
@@ -820,9 +820,9 @@ def __write_scheme(  # noqa: C901
         CUSTOM_TYPE_SCHEME,
         NAME_PER_SCHEME,
     ]:
-        data["Attributes"] += (
-            f" {_write_vtl(item_scheme, indent, references_30)}"
-        )
+        data[
+            "Attributes"
+        ] += f" {_write_vtl(item_scheme, indent, references_30)}"
 
     outfile = ""
 
@@ -1129,9 +1129,11 @@ def _write_vtl(  # noqa: C901
             ref_codelist = (
                 item_or_scheme.codelist
                 if isinstance(item_or_scheme.codelist, Reference)
-                else parse_urn(item_or_scheme.codelist)
-                if isinstance(item_or_scheme.codelist, str)
-                else parse_short_urn(item_or_scheme.codelist.short_urn)
+                else (
+                    parse_urn(item_or_scheme.codelist)
+                    if isinstance(item_or_scheme.codelist, str)
+                    else parse_short_urn(item_or_scheme.codelist.short_urn)
+                )
             )
             if references_30:
                 data += (

pysdmx/io/xml/__write_data_aux.py

@@ -1,6 +1,4 @@
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-import pandas as pd
+from typing import Dict, Optional, Sequence
 
 from pysdmx.errors import Invalid
 from pysdmx.io.pd import PandasDataset
@@ -56,13 +54,15 @@ def writing_validation(dataset: PandasDataset) -> None:
         for comp in dataset.structure.components
         if comp.role in (Role.DIMENSION, Role.MEASURE)
     ]
-    for att in dataset.structure.components.attributes:
+    required_components.extend(
+        att.id
+        for att in dataset.structure.components.attributes
         if (
             att.required
             and att.attachment_level is not None
             and att.attachment_level != "D"
-        ):
-            required_components.append(att.id)
+        )
+    )
     non_required = [
         comp.id
         for comp in dataset.structure.components
@@ -87,54 +87,3 @@ def writing_validation(dataset: PandasDataset) -> None:
         )
     if not dataset.structure.components.measures:
         raise Invalid("The dataset structure must have at least one measure.")
-
-
-def get_codes(
-    dimension_code: str, structure: Schema, data: pd.DataFrame
-) -> Tuple[List[str], List[str], List[Dict[str, Any]]]:
-    """This function divides the components in Series and Obs."""
-    series_codes = []
-    groups = structure.groups
-    group_codes = []
-    obs_codes = [dimension_code, structure.components.measures[0].id]
-
-    # Getting the series and obs codes
-    for dim in structure.components.dimensions:
-        if dim.id != dimension_code:
-            series_codes.append(dim.id)
-
-    # Adding the attributes based on the attachment level
-    for att in structure.components.attributes:
-        matching_group = next(
-            (
-                group
-                for group in groups or []
-                if set(group.dimensions)
-                == set(att.attachment_level.split(","))  # type: ignore[union-attr]
-            ),
-            None,
-        )
-
-        if (
-            att.attachment_level != "D"
-            and att.id in data.columns
-            and groups is not None
-            and matching_group
-        ):
-            group_codes.append(
-                {
-                    "group_id": matching_group.id,
-                    "attribute": att.id,
-                    "dimensions": matching_group.dimensions,
-                }
-            )
-        elif att.attachment_level == "O" and att.id in data.columns:
-            obs_codes.append(att.id)
-        elif (
-            att.attachment_level is not None
-            and att.attachment_level != "D"
-            and att.id in data.columns
-        ):
-            series_codes.append(att.id)
-
-    return series_codes, obs_codes, group_codes

pysdmx/io/xml/__write_structure_specific_aux.py

@@ -13,10 +13,10 @@ from pysdmx.io.xml.__write_aux import (
     get_structure,
 )
 from pysdmx.io.xml.__write_data_aux import (
-    get_codes,
     writing_validation,
 )
 from pysdmx.io.xml.config import CHUNKSIZE
+from pysdmx.toolkit.pd._data_utils import get_codes
 from pysdmx.util import parse_short_urn
 
 
@@ -198,8 +198,12 @@ def __group_processing(
             .to_dict(orient="records")
         )
 
-        for record in grouped_data:
-            out_list.append(__format_group_str(record, group["group_id"]))
+        out_list.extend(
+            [
+                __format_group_str(record, group["group_id"])
+                for record in grouped_data
+            ]
+        )
 
     return "".join(out_list)
 

pysdmx/io/xml/doc_validation.py

@@ -39,9 +39,7 @@ def validate_doc(input_str: str) -> None:
     doc = etree.parse(bytes_infile, parser=parser)
     if not xmlschema.validate(doc):
         log_errors = list(xmlschema.error_log)  # type: ignore[call-overload]
-        unhandled_errors = []
-        for e in log_errors:
-            unhandled_errors.append(e.message)
+        unhandled_errors = [e.message for e in log_errors]
         severe_errors = unhandled_errors.copy()
         for e in unhandled_errors:
             for allowed_error in ALLOWED_ERRORS_CONTENT: