pysdmx 1.5.2__py3-none-any.whl → 1.6.0__py3-none-any.whl

pysdmx/io/json/sdmxjson2/reader/metadata.py

@@ -0,0 +1,32 @@
+"""Writer interface for SDMX-JSON 2.0.0 Reference Metadata messages."""
+
+import msgspec
+
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages import JsonMetadataMessage
+from pysdmx.model import decoders
+from pysdmx.model.message import MetadataMessage
+
+
+def read(input_str: str) -> MetadataMessage:
+    """Read an SDMX-JSON 2.0.0 Metadata Message.
+
+    Args:
+        input_str: SDMX-JSON reference metadata message to read.
+
+    Returns:
+        A pysdmx MetadataMessage
+    """
+    try:
+        msg = msgspec.json.Decoder(
+            JsonMetadataMessage, dec_hook=decoders
+        ).decode(input_str)
+        return msg.to_model()
+    except msgspec.DecodeError as de:
+        raise errors.Invalid(
+            "Invalid message",
+            (
+                "The supplied file could not be read as SDMX-JSON 2.0.0 "
+                "reference metadata message."
+            ),
+        ) from de

pysdmx/io/json/sdmxjson2/reader/structure.py

@@ -0,0 +1,32 @@
+"""Reader interface for SDMX-JSON 2.0.0 Structure messages."""
+
+import msgspec
+
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages import JsonStructureMessage
+from pysdmx.model import decoders
+from pysdmx.model.message import StructureMessage
+
+
+def read(input_str: str) -> StructureMessage:
+    """Read an SDMX-JSON 2.0.0 Stucture Message.
+
+    Args:
+        input_str: SDMX-JSON structure message to read.
+
+    Returns:
+        A pysdmx StructureMessage
+    """
+    try:
+        msg = msgspec.json.Decoder(
+            JsonStructureMessage, dec_hook=decoders
+        ).decode(input_str)
+        return msg.to_model()
+    except msgspec.DecodeError as de:
+        raise errors.Invalid(
+            "Invalid message",
+            (
+                "The supplied file could not be read as SDMX-JSON 2.0.0 "
+                "structure message."
+            ),
+        ) from de

pysdmx/io/json/sdmxjson2/writer/__init__.py

@@ -0,0 +1,9 @@
+"""Collection of writers for SDMX-JSON messages."""
+
+from pysdmx.io.json.sdmxjson2 import messages as msg
+from pysdmx.io.serde import Serializers
+
+serializers = Serializers(
+    metadata_message=msg.JsonMetadataMessage,
+    structure_message=msg.JsonStructureMessage,
+)

pysdmx/io/json/sdmxjson2/writer/metadata.py

@@ -0,0 +1,60 @@
+"""Writer interface for SDMX-JSON 2.0.0 Reference Metadata messages."""
+
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+import msgspec
+
+from pysdmx.io.json.sdmxjson2.messages import JsonMetadataMessage
+from pysdmx.model import MetadataReport, encoders
+from pysdmx.model.message import Header, MetadataMessage
+
+
+def write(
+    reports: Sequence[MetadataReport],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+) -> Optional[str]:
+    """Write metadata reports in SDMX-JSON 2.0.0.
+
+    Args:
+        reports: The reference metadata reports to be serialized.
+        output_path: The path to save the JSON file. If None or empty, the
+            serialized content is returned as a string instead.
+        prettyprint: Whether to format the JSON output with indentation (True)
+            or output compact JSON without extra whitespace (False).
+        header: The header to be used in the SDMX-JSON message
+            (will be generated if no header is supplied).
+
+    Returns:
+        The JSON string if output_path is None or empty, None otherwise.
+    """
+    if not header:
+        header = Header()
+    sm = MetadataMessage(header, reports)
+    jsm = JsonMetadataMessage.from_model(sm)
+
+    encoder = msgspec.json.Encoder(enc_hook=encoders)
+    serialized_data = encoder.encode(jsm)
+
+    # Apply pretty-printing if requested
+    if prettyprint:
+        serialized_data = msgspec.json.format(serialized_data, indent=4)
+
+    # If output_path is provided, write to file
+    if output_path:
+        # Convert to Path object if string
+        if isinstance(output_path, str):
+            output_path = Path(output_path)
+
+        # Create parent directories if they don't exist
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to file
+        with open(output_path, "wb") as f:
+            f.write(serialized_data)
+        return None
+    else:
+        # Return as string
+        return serialized_data.decode("utf-8")

pysdmx/io/json/sdmxjson2/writer/structure.py

@@ -0,0 +1,61 @@
+"""Writer interface for SDMX-JSON 2.0.0 Structure messages."""
+
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+import msgspec
+
+from pysdmx.io.json.sdmxjson2.messages import JsonStructureMessage
+from pysdmx.model import encoders
+from pysdmx.model.__base import MaintainableArtefact
+from pysdmx.model.message import Header, StructureMessage
+
+
+def write(
+    structures: Sequence[MaintainableArtefact],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+) -> Optional[str]:
+    """Write maintainable SDMX artefacts in SDMX-JSON 2.0.0.
+
+    Args:
+        structures: The maintainable SDMX artefacts to be serialized.
+        output_path: The path to save the JSON file. If None or empty, the
+            serialized content is returned as a string instead.
+        prettyprint: Whether to format the JSON output with indentation (True)
+            or output compact JSON without extra whitespace (False).
+        header: The header to be used in the SDMX-JSON message
+            (will be generated if no header is supplied).
+
+    Returns:
+        The JSON string if output_path is None or empty, None otherwise.
+    """
+    if not header:
+        header = Header()
+    sm = StructureMessage(header, structures)
+    jsm = JsonStructureMessage.from_model(sm)
+
+    encoder = msgspec.json.Encoder(enc_hook=encoders)
+    serialized_data = encoder.encode(jsm)
+
+    # Apply pretty-printing if requested
+    if prettyprint:
+        serialized_data = msgspec.json.format(serialized_data, indent=4)
+
+    # If output_path is provided, write to file
+    if output_path:
+        # Convert to Path object if string
+        if isinstance(output_path, str):
+            output_path = Path(output_path)
+
+        # Create parent directories if they don't exist
+        output_path.parent.mkdir(parents=True, exist_ok=True)
+
+        # Write to file
+        with open(output_path, "wb") as f:
+            f.write(serialized_data)
+        return None
+    else:
+        # Return as string
+        return serialized_data.decode("utf-8")

pysdmx/io/reader.py

@@ -8,10 +8,10 @@ from pysdmx.errors import Invalid
 from pysdmx.io.format import Format
 from pysdmx.io.input_processor import process_string_to_read
 from pysdmx.model import Schema
-from pysdmx.model.__base import ItemScheme
-from pysdmx.model.dataflow import Dataflow, DataStructureDefinition
+from pysdmx.model.__base import MaintainableArtefact
 from pysdmx.model.dataset import Dataset
 from pysdmx.model.message import Message
+from pysdmx.model.metadata import MetadataReport
 from pysdmx.model.submission import SubmissionResult
 from pysdmx.util import parse_short_urn
 from pysdmx.util._model_utils import schema_generator
@@ -43,10 +43,9 @@ def read_sdmx(  # noqa: C901
 
     header = None
     result_data: Sequence[Dataset] = []
-    result_structures: Sequence[
-        Union[ItemScheme, Dataflow, DataStructureDefinition]
-    ] = []
+    result_structures: Sequence[MaintainableArtefact] = []
     result_submission: Sequence[SubmissionResult] = []
+    reports: Sequence[MetadataReport] = []
     if read_format == Format.STRUCTURE_SDMX_ML_2_1:
         from pysdmx.io.xml.header import read as read_header
         from pysdmx.io.xml.sdmx21.reader.structure import (
@@ -74,6 +73,24 @@ def read_sdmx(  # noqa: C901
         header = read_header(input_str, validate=validate)
         # SDMX-ML 3.1 Structure
         result_structures = read_structure(input_str, validate=validate)
+    elif read_format == Format.STRUCTURE_SDMX_JSON_2_0_0:
+        from pysdmx.io.json.sdmxjson2.reader.structure import (
+            read as read_struct,
+        )
+
+        struct_msg = read_struct(input_str)
+        header = struct_msg.header
+        result_structures = (
+            struct_msg.structures if struct_msg.structures else []
+        )
+    elif read_format == Format.REFMETA_SDMX_JSON_2_0_0:
+        from pysdmx.io.json.sdmxjson2.reader.metadata import (
+            read as read_refmeta,
+        )
+
+        ref_msg = read_refmeta(input_str)
+        header = ref_msg.header
+        reports = ref_msg.get_reports()
     elif read_format == Format.DATA_SDMX_ML_2_1_GEN:
         from pysdmx.io.xml.header import read as read_header
         from pysdmx.io.xml.sdmx21.reader.generic import read as read_generic
@@ -127,18 +144,19 @@ def read_sdmx(  # noqa: C901
         # SDMX-CSV 1.0
         result_data = read_csv_v1(input_str)
     else:
-        # SDMX-CSV 2.0
-        from pysdmx.io.csv.sdmx20.reader import read as read_csv_v2
+        # SDMX-CSV 2.1
+        from pysdmx.io.csv.sdmx21.reader import read as read_csv_v2
 
         result_data = read_csv_v2(input_str)
 
-    if not (result_data or result_structures or result_submission):
+    if not (result_data or result_structures or result_submission or reports):
         raise Invalid("Empty SDMX Message")
 
     # Returning a Message class
     if read_format in (
         Format.DATA_SDMX_CSV_1_0_0,
         Format.DATA_SDMX_CSV_2_0_0,
+        Format.DATA_SDMX_CSV_2_1_0,
         Format.DATA_SDMX_ML_2_1_GEN,
         Format.DATA_SDMX_ML_2_1_STR,
         Format.DATA_SDMX_ML_3_0,
@@ -149,7 +167,8 @@ def read_sdmx(  # noqa: C901
         return Message(header=header, data=result_data)
     elif read_format == Format.REGISTRY_SDMX_ML_2_1:
         return Message(header=header, submission=result_submission)
-
+    elif read_format == Format.REFMETA_SDMX_JSON_2_0_0:
+        return Message(header=header, reports=reports)
     # TODO: Ensure we have changed the signature of the structure readers
     return Message(header=header, structures=result_structures)
 

pysdmx/io/serde.py

@@ -12,6 +12,15 @@ class Deserializer(Protocol):
         """Returns the domain objects."""
 
 
+@runtime_checkable
+class Serializer(Protocol):
+    """Creates an SDMX message from domain objects."""
+
+    @classmethod
+    def from_model(self, message: Any) -> Any:
+        """Returns the SDMX message."""
+
+
 @dataclass
 class Deserializers:
     """Collection of deserializers for a format."""
@@ -34,6 +43,14 @@ class Deserializers:
     transformation_scheme: Deserializer
 
 
+@dataclass
+class Serializers:
+    """Collection of serializers for a format."""
+
+    structure_message: Serializer
+    metadata_message: Serializer
+
+
 @dataclass
 class GdsDeserializers:
     """Collection of GDS deserializers for a format."""

pysdmx/io/writer.py

@@ -7,11 +7,14 @@ from typing import Any, Optional, Sequence
 
 from pysdmx.errors import Invalid
 from pysdmx.io.format import Format
+from pysdmx.model import MetadataReport
+from pysdmx.model.__base import MaintainableArtefact
 from pysdmx.model.dataset import Dataset
 
 WRITERS = {
     Format.DATA_SDMX_CSV_1_0_0: "pysdmx.io.csv.sdmx10.writer",
     Format.DATA_SDMX_CSV_2_0_0: "pysdmx.io.csv.sdmx20.writer",
+    Format.DATA_SDMX_CSV_2_1_0: "pysdmx.io.csv.sdmx21.writer",
     Format.DATA_SDMX_ML_2_1_GEN: "pysdmx.io.xml.sdmx21.writer.generic",
     Format.DATA_SDMX_ML_2_1_STR: "pysdmx.io.xml.sdmx21.writer."
     "structure_specific",
@@ -22,14 +25,23 @@ WRITERS = {
     Format.DATA_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer."
     "structure_specific",
     Format.STRUCTURE_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer.structure",
+    Format.STRUCTURE_SDMX_JSON_2_0_0: (
+        "pysdmx.io.json.sdmxjson2.writer.structure"
+    ),
+    Format.REFMETA_SDMX_JSON_2_0_0: (
+        "pysdmx.io.json.sdmxjson2.writer.metadata"
+    ),
 }
 
 STRUCTURE_WRITERS = (
     Format.STRUCTURE_SDMX_ML_2_1,
     Format.STRUCTURE_SDMX_ML_3_0,
     Format.STRUCTURE_SDMX_ML_3_1,
+    Format.STRUCTURE_SDMX_JSON_2_0_0,
 )
 
+REFMETA_WRITERS = (Format.REFMETA_SDMX_JSON_2_0_0,)
+
 
 def write_sdmx(
     sdmx_objects: Any,
@@ -97,20 +109,36 @@ def write_sdmx(
     writer = module.write
 
     is_structure = sdmx_format in STRUCTURE_WRITERS
+    is_ref_meta = sdmx_format in REFMETA_WRITERS
     is_xml = "xml" in WRITERS[sdmx_format]
-    key = "structures" if is_structure else "datasets"
+    is_json = "json" in WRITERS[sdmx_format]
+    if is_structure:
+        key = "structures"
+    elif is_ref_meta:
+        key = "reports"
+    else:
+        key = "datasets"
     value = sdmx_objects if isinstance(sdmx_objects, list) else [sdmx_objects]
 
-    if is_structure and any(isinstance(x, Dataset) for x in value):
+    if is_structure and not all(
+        isinstance(x, MaintainableArtefact) for x in value
+    ):
         raise Invalid(
-            "Datasets cannot be written to structure formats. "
-            "Use data formats instead."
+            "Only maintainable artefacts can be written to structure formats."
         )
-    elif not is_structure and not all(isinstance(x, Dataset) for x in value):
+    elif is_ref_meta and not all(isinstance(x, MetadataReport) for x in value):
         raise Invalid(
-            "Only Datasets can be written to data formats. "
-            "Use structure formats for other SDMX objects."
+            (
+                "Only metadata reports can be written to reference "
+                "metadata formats."
+            )
         )
+    elif (
+        not is_structure
+        and not is_ref_meta
+        and not all(isinstance(x, Dataset) for x in value)
+    ):
+        raise Invalid("Only Datasets can be written to data formats.")
 
     args = {
         key: value,
@@ -120,7 +148,7 @@ def write_sdmx(
                 "prettyprint": kwargs.get("prettyprint"),
                 "header": kwargs.get("header"),
             }
-            if is_xml
+            if is_xml or is_json
             else {}
         ),
         **(
@@ -132,7 +160,15 @@ def write_sdmx(
             if is_xml and not is_structure
             else {}
         ),
+        **(
+            {
+                "labels": kwargs.get("labels"),
+                "keys": kwargs.get("keys"),
+                "time_format": kwargs.get("time_format"),
+            }
+            if not is_xml
+            else {}
+        ),
     }
     args = {k: v for k, v in args.items() if v is not None}
-
     return writer(**args)

pysdmx/io/xml/__write_data_aux.py

@@ -1,6 +1,4 @@
-from typing import Any, Dict, List, Optional, Sequence, Tuple
-
-import pandas as pd
+from typing import Dict, Optional, Sequence
 
 from pysdmx.errors import Invalid
 from pysdmx.io.pd import PandasDataset
@@ -87,54 +85,3 @@ def writing_validation(dataset: PandasDataset) -> None:
         )
     if not dataset.structure.components.measures:
         raise Invalid("The dataset structure must have at least one measure.")
-
-
-def get_codes(
-    dimension_code: str, structure: Schema, data: pd.DataFrame
-) -> Tuple[List[str], List[str], List[Dict[str, Any]]]:
-    """This function divides the components in Series and Obs."""
-    series_codes = []
-    groups = structure.groups
-    group_codes = []
-    obs_codes = [dimension_code, structure.components.measures[0].id]
-
-    # Getting the series and obs codes
-    for dim in structure.components.dimensions:
-        if dim.id != dimension_code:
-            series_codes.append(dim.id)
-
-    # Adding the attributes based on the attachment level
-    for att in structure.components.attributes:
-        matching_group = next(
-            (
-                group
-                for group in groups or []
-                if set(group.dimensions)
-                == set(att.attachment_level.split(","))  # type: ignore[union-attr]
-            ),
-            None,
-        )
-
-        if (
-            att.attachment_level != "D"
-            and att.id in data.columns
-            and groups is not None
-            and matching_group
-        ):
-            group_codes.append(
-                {
-                    "group_id": matching_group.id,
-                    "attribute": att.id,
-                    "dimensions": matching_group.dimensions,
-                }
-            )
-        elif att.attachment_level == "O" and att.id in data.columns:
-            obs_codes.append(att.id)
-        elif (
-            att.attachment_level is not None
-            and att.attachment_level != "D"
-            and att.id in data.columns
-        ):
-            series_codes.append(att.id)
-
-    return series_codes, obs_codes, group_codes

pysdmx/io/xml/__write_structure_specific_aux.py

@@ -13,10 +13,10 @@ from pysdmx.io.xml.__write_aux import (
     get_structure,
 )
 from pysdmx.io.xml.__write_data_aux import (
-    get_codes,
     writing_validation,
 )
 from pysdmx.io.xml.config import CHUNKSIZE
+from pysdmx.toolkit.pd._data_utils import get_codes
 from pysdmx.util import parse_short_urn
 
 

pysdmx/io/xml/sdmx21/writer/generic.py

@@ -21,11 +21,11 @@ from pysdmx.io.xml.__write_aux import (
 from pysdmx.io.xml.__write_data_aux import (
     check_content_dataset,
     check_dimension_at_observation,
-    get_codes,
     writing_validation,
 )
 from pysdmx.io.xml.config import CHUNKSIZE
 from pysdmx.model.message import Header
+from pysdmx.toolkit.pd._data_utils import get_codes
 from pysdmx.util import parse_short_urn
 
 

pysdmx/model/code.py

@@ -21,7 +21,13 @@ from typing import Iterator, Literal, Optional, Sequence, Union
 
 from msgspec import Struct
 
-from pysdmx.model.__base import Agency, Item, ItemScheme, MaintainableArtefact
+from pysdmx.model.__base import (
+    Agency,
+    Annotation,
+    Item,
+    ItemScheme,
+    MaintainableArtefact,
+)
 
 
 class Code(Item, frozen=True, omit_defaults=True):
@@ -139,6 +145,8 @@ class HierarchicalCode(
         rel_valid_from: Start of the hierarchical relationship validity.
         rel_valid_to: End of the hierarchical relationship validity.
         codes: The child codes.
+        annotations: Annotations attached to the code.
+        urn: The URN of the code.
     """
 
     id: str
@@ -149,6 +157,8 @@ class HierarchicalCode(
     rel_valid_from: Optional[datetime] = None
     rel_valid_to: Optional[datetime] = None
     codes: Sequence["HierarchicalCode"] = ()
+    annotations: Sequence[Annotation] = ()
+    urn: Optional[str] = None
 
     def __iter__(self) -> Iterator["HierarchicalCode"]:
         """Return an iterator over the list of codes."""

pysdmx/model/dataflow.py

@@ -130,6 +130,9 @@ class Component(
             commas, for series- and group-level attributes).
             A post_init check makes this attribute mandatory for attributes.
         array_def: Any additional constraints for array types.
+        urn: The URN of the component.
+        local_enum_ref: The URN of the enumeration (codelist or valuelist) from
+            which the local codes are taken.
     """
 
     id: str
@@ -144,6 +147,7 @@ class Component(
     attachment_level: Optional[str] = None
     array_def: Optional[ArrayBoundaries] = None
     urn: Optional[str] = None
+    local_enum_ref: Optional[str] = None
 
     def __post_init__(self) -> None:
         """Validate the component."""
@@ -217,6 +221,20 @@ class Component(
         else:
             return None
 
+    @property
+    def enum_ref(self) -> Optional[str]:
+        """Returns the URN of the enumeration from which the codes are taken.
+
+        Returns:
+            The URN of the enumeration from which the codes are taken.
+        """
+        if self.local_enum_ref:
+            return self.local_enum_ref
+        elif isinstance(self.concept, Concept) and self.concept.enum_ref:
+            return self.concept.enum_ref
+        else:
+            return None
+
     def __str__(self) -> str:
         """Custom string representation without the class name."""
         processed_output = []
@@ -393,6 +411,7 @@ class DataflowInfo(
         end_period: The oldest period for which data are available.
         last_updated: When the dataflow was last updated.
         dsd_ref: The URN of the data structure used by the dataflow.
+        groups: The sequence of groups defined in the data structure.
     """
 
     id: str
@@ -408,6 +427,7 @@ class DataflowInfo(
     end_period: Optional[str] = None
     last_updated: Optional[datetime] = None
     dsd_ref: Optional[str] = None
+    groups: Optional[Sequence[Group]] = None
 
     def __str__(self) -> str:
         """Custom string representation without the class name."""
@@ -472,6 +492,7 @@ class Schema(Struct, frozen=True, omit_defaults=True, repr_omit_defaults=True):
     version: str = "1.0"
     artefacts: Sequence[str] = ()
     generated: datetime = datetime.now(timezone.utc)
+    name: Optional[str] = None
     groups: Optional[Sequence[Group]] = None
 
     def __str__(self) -> str:
@@ -533,6 +554,7 @@ class DataStructureDefinition(MaintainableArtefact, frozen=True, kw_only=True):
         valid_from: The date from which the data structure is valid.
         valid_to: The date until which the data structure is valid.
         version: The version of the data structure.
+        components: The list of relevant components for this data structure.
         evolving_structure: Whether new dimensions may be added under a
             minor version update.
     """
@@ -572,6 +594,7 @@ class DataStructureDefinition(MaintainableArtefact, frozen=True, kw_only=True):
             components=self.components,
             version=self.version,
             artefacts=self.__extract_artefacts(),
+            name=self.name,
         )
 
     @property

pysdmx/model/map.py

@@ -6,7 +6,7 @@ from typing import Any, Iterator, Literal, Optional, Sequence, Tuple, Union
 
 from msgspec import Struct
 
-from pysdmx.model.__base import MaintainableArtefact
+from pysdmx.model.__base import Agency, MaintainableArtefact
 from pysdmx.util._date_pattern_map import convert_dpm
 
 
@@ -77,7 +77,7 @@ class DatePatternMap(_BaseMap, frozen=True, omit_defaults=True, tag=True):
             the target structure (e.g. `FREQ`). In this case, the input date
             can be converted to a different format, depending on the
             frequency of the converted data.
-        resolvePeriod: The point in time to resolve to when mapping from low
+        resolve_period: The point in time to resolve to when mapping from low
             frequency to higher frequency periods.
     """
 
@@ -88,7 +88,7 @@ class DatePatternMap(_BaseMap, frozen=True, omit_defaults=True, tag=True):
     id: Optional[str] = None
     locale: str = "en"
     pattern_type: Literal["fixed", "variable"] = "fixed"
-    resolvePeriod: Optional[
+    resolve_period: Optional[
         Literal["startOfPeriod", "endOfPeriod", "midPeriod"]
     ] = None
 
@@ -264,6 +264,14 @@ class MultiRepresentationMap(
     target: Sequence[str] = []
     maps: Sequence[MultiValueMap] = []
 
+    @property
+    def short_urn(self) -> str:
+        """Returns the short URN for the MultiRepresentationMap."""
+        agency = (
+            self.agency.id if isinstance(self.agency, Agency) else self.agency
+        )
+        return f"RepresentationMap={agency}:{self.id}({self.version})"
+
     def __iter__(
         self,
     ) -> Iterator[MultiValueMap]:
@@ -300,7 +308,7 @@ class MultiComponentMap(_BaseMap, frozen=True, omit_defaults=True, tag=True):
 
     source: Sequence[str]
     target: Sequence[str]
-    values: MultiRepresentationMap
+    values: Union[MultiRepresentationMap, str]
 
 
 class RepresentationMap(MaintainableArtefact, frozen=True, omit_defaults=True):