pysdmx 1.3.0__py3-none-any.whl → 1.4.0rc1__py3-none-any.whl

pysdmx/io/xml/sdmx21/writer/structure_specific.py

@@ -1,258 +1,29 @@
 # mypy: disable-error-code="union-attr"
 """Module for writing SDMX-ML 2.1 Structure Specific data messages."""
 
-from typing import Any, Dict, List, Optional, Sequence
-
-import pandas as pd
+from pathlib import Path
+from typing import Dict, Optional, Sequence, Union
 
 from pysdmx.io.format import Format
 from pysdmx.io.pd import PandasDataset
-from pysdmx.io.xml.sdmx21.writer.__write_aux import (
-    ABBR_MSG,
-    ALL_DIM,
-    __escape_xml,
+from pysdmx.io.xml.__write_aux import (
     __write_header,
     create_namespaces,
     get_end_message,
-    get_structure,
 )
-from pysdmx.io.xml.sdmx21.writer.__write_data_aux import (
+from pysdmx.io.xml.__write_data_aux import (
     check_content_dataset,
     check_dimension_at_observation,
-    get_codes,
-    writing_validation,
 )
-from pysdmx.io.xml.sdmx21.writer.config import CHUNKSIZE
+from pysdmx.io.xml.__write_structure_specific_aux import (
+    __write_data_structure_specific,
+)
 from pysdmx.model.message import Header
-from pysdmx.util import parse_short_urn
-
-
-def __memory_optimization_writing(
-    dataset: PandasDataset, prettyprint: bool
-) -> str:
-    """Memory optimization for writing data."""
-    outfile = ""
-    length_ = len(dataset.data)
-    if len(dataset.data) > CHUNKSIZE:
-        previous = 0
-        next_ = CHUNKSIZE
-        while previous <= length_:
-            # Sliding a window for efficient access to the data
-            # and avoid memory issues
-            outfile += __obs_processing(
-                dataset.data.iloc[previous:next_], prettyprint
-            )
-            previous = next_
-            next_ += CHUNKSIZE
-
-            if next_ >= length_:
-                outfile += __obs_processing(
-                    dataset.data.iloc[previous:], prettyprint
-                )
-                previous = next_
-    else:
-        outfile += __obs_processing(dataset.data, prettyprint)
-
-    return outfile
-
-
-def __write_data_structure_specific(
-    datasets: Dict[str, PandasDataset],
-    dim_mapping: Dict[str, str],
-    prettyprint: bool = True,
-) -> str:
-    """Write data to SDMX-ML 2.1 Structure-Specific format.
-
-    Args:
-        datasets: dict. Datasets to be written.
-        dim_mapping: dict. URN-DimensionAtObservation mapping.
-        prettyprint: bool. Prettyprint or not.
-
-    Returns:
-        The data in SDMX-ML 2.1 Structure-Specific format, as string.
-    """
-    outfile = ""
-
-    for i, (short_urn, dataset) in enumerate(datasets.items()):
-        dataset.data = dataset.data.fillna("").astype(str)
-        outfile += __write_data_single_dataset(
-            dataset=dataset,
-            prettyprint=prettyprint,
-            count=i + 1,
-            dim=dim_mapping[short_urn],
-        )
-
-    return outfile
-
-
-def __write_data_single_dataset(
-    dataset: PandasDataset,
-    prettyprint: bool = True,
-    count: int = 1,
-    dim: str = ALL_DIM,
-) -> str:
-    """Write data to SDMX-ML 2.1 Structure-Specific format.
-
-    Args:
-        dataset: PandasDataset. Dataset to be written.
-        prettyprint: bool. Prettyprint or not.
-        count: int. Count for namespace.
-        dim: str. Dimension to be written.
-
-    Returns:
-        The data in SDMX-ML 2.1 Structure-Specific format, as string.
-    """
-
-    def __remove_optional_attributes_empty_data(str_to_check: str) -> str:
-        """This function removes data when optional attributes are found."""
-        for att in dataset.structure.components.attributes:
-            if not att.required:
-                str_to_check = str_to_check.replace(f"{att.id}='' ", "")
-                str_to_check = str_to_check.replace(f'{att.id}="" ', "")
-        return str_to_check
-
-    outfile = ""
-    structure_urn = get_structure(dataset)
-    id_structure = parse_short_urn(structure_urn).id
-    sdmx_type = parse_short_urn(structure_urn).id
-
-    nl = "\n" if prettyprint else ""
-    child1 = "\t" if prettyprint else ""
-
-    attached_attributes_str = ""
-    for k, v in dataset.attributes.items():
-        attached_attributes_str += f"{k}={str(v)!r} "
-
-    # Datasets
-    outfile += (
-        f"{nl}{child1}<{ABBR_MSG}:DataSet {attached_attributes_str}"
-        f"ss:structureRef={id_structure!r} "
-        f'xsi:type="ns{count}:DataSetType" '
-        f'ss:dataScope="{sdmx_type}" '
-        f'action="{dataset.action.value}">{nl}'
-    )
-    data = ""
-    if dim == ALL_DIM:
-        data += __memory_optimization_writing(dataset, prettyprint)
-    else:
-        writing_validation(dataset)
-        series_codes, obs_codes = get_codes(
-            dimension_code=dim,
-            structure=dataset.structure,  # type: ignore[arg-type]
-            data=dataset.data,
-        )
-
-        data += __series_processing(
-            data=dataset.data,
-            series_codes=series_codes,
-            obs_codes=obs_codes,
-            prettyprint=prettyprint,
-        )
-
-        # Remove optional attributes empty data
-        data = __remove_optional_attributes_empty_data(data)
-
-    # Adding to outfile
-    outfile += data
-
-    outfile += f"{child1}</{ABBR_MSG}:DataSet>"
-
-    return outfile.replace("'", '"')
-
-
-def __obs_processing(data: pd.DataFrame, prettyprint: bool = True) -> str:
-    def __format_obs_str(element: Dict[str, Any]) -> str:
-        """Formats the observation as key=value pairs."""
-        nl = "\n" if prettyprint else ""
-        child2 = "\t\t" if prettyprint else ""
-
-        out = f"{child2}<Obs "
-
-        for k, v in element.items():
-            out += f"{k}={__escape_xml(str(v))!r} "
-
-        out += f"/>{nl}"
-
-        return out
-
-    parser = lambda x: __format_obs_str(x)  # noqa: E731
-
-    iterator = map(parser, data.to_dict(orient="records"))
-
-    return "".join(iterator)
-
-
-def __series_processing(
-    data: pd.DataFrame,
-    series_codes: List[str],
-    obs_codes: List[str],
-    prettyprint: bool = True,
-) -> str:
-    def __generate_series_str() -> str:
-        """Generates the series item with its observations."""
-        out_list: List[str] = []
-        data.groupby(by=series_codes)[obs_codes].apply(
-            lambda x: __format_dict_ser(out_list, x)
-        )
-
-        return "".join(out_list)
-
-    def __format_dict_ser(
-        output_list: List[str],
-        obs: Any,
-    ) -> Any:
-        """Formats the series as key=value pairs."""
-        # Creating the observation dict,
-        # we always get the first element on Series
-        # as we are grouping by it
-        data_dict["Series"][0]["Obs"] = obs.to_dict(orient="records")
-        output_list.append(__format_ser_str(data_dict["Series"][0]))
-        # We remove the data for series as it is no longer necessary
-        del data_dict["Series"][0]
-
-    def __format_ser_str(data_info: Dict[Any, Any]) -> str:
-        """Formats the series as key=value pairs."""
-        child2 = "\t\t" if prettyprint else ""
-        child3 = "\t\t\t" if prettyprint else ""
-        nl = "\n" if prettyprint else ""
-
-        out_element = f"{child2}<Series "
-
-        for k, v in data_info.items():
-            if k != "Obs":
-                out_element += f"{k}={__escape_xml(str(v))!r} "
-
-        out_element += f">{nl}"
-
-        for obs in data_info["Obs"]:
-            out_element += f"{child3}<Obs "
-
-            for k, v in obs.items():
-                out_element += f"{k}={__escape_xml(str(v))!r} "
-
-            out_element += f"/>{nl}"
-
-        out_element += f"{child2}</Series>{nl}"
-
-        return out_element
-
-    # Getting each datapoint from data and creating dict
-    data = data.sort_values(series_codes, axis=0)
-    data_dict = {
-        "Series": data[series_codes]
-        .drop_duplicates()
-        .reset_index(drop=True)
-        .to_dict(orient="records")
-    }
-
-    out = __generate_series_str()
-
-    return out
 
 
 def write(
     datasets: Sequence[PandasDataset],
-    output_path: str = "",
+    output_path: Optional[Union[str, Path]] = None,
     prettyprint: bool = True,
     header: Optional[Header] = None,
     dimension_at_observation: Optional[Dict[str, str]] = None,
@@ -309,7 +80,11 @@ def write(
 
     outfile += get_end_message(type_, prettyprint)
 
-    if output_path == "":
+    output_path = (
+        str(output_path) if isinstance(output_path, Path) else output_path
+    )
+
+    if output_path is None or output_path == "":
         return outfile
 
     with open(output_path, "w", encoding="UTF-8", errors="replace") as f:

pysdmx/io/xml/sdmx30/__init__.py

@@ -0,0 +1 @@
+"""SDMX 3.0 XML reader and writer."""

pysdmx/io/xml/sdmx30/reader/__init__.py

@@ -0,0 +1 @@
+"""SDMX 3.0 XML reader module."""

pysdmx/io/xml/sdmx30/reader/structure.py

@@ -0,0 +1,39 @@
+"""Parsers for reading metadata."""
+
+from typing import Sequence, Union
+
+from pysdmx.errors import Invalid
+from pysdmx.io.xml.__parse_xml import parse_xml
+from pysdmx.io.xml.__structure_aux_reader import StructureParser
+from pysdmx.io.xml.__tokens import (
+    STRUCTURE,
+    STRUCTURES,
+)
+from pysdmx.model.__base import (
+    ItemScheme,
+)
+from pysdmx.model.dataflow import (
+    Dataflow,
+    DataStructureDefinition,
+)
+
+
+def read(
+    input_str: str,
+    validate: bool = True,
+) -> Sequence[Union[ItemScheme, DataStructureDefinition, Dataflow]]:
+    """Reads an SDMX-ML 3.0 Structure data and returns the structures.
+
+    Args:
+        input_str: SDMX-ML structure message to read.
+        validate: If True, the XML data will be validated against the XSD.
+
+    Returns:
+        dict: Dictionary with the parsed structures.
+    """
+    dict_info = parse_xml(input_str, validate)
+    if STRUCTURE not in dict_info:
+        raise Invalid("This SDMX document is not SDMX-ML 3.0 Structure.")
+    return StructureParser(is_sdmx_30=True).format_structures(
+        dict_info[STRUCTURE][STRUCTURES]
+    )

pysdmx/io/xml/sdmx30/reader/structure_specific.py

@@ -0,0 +1,39 @@
+"""SDMX XML 3.0 StructureSpecificData reader module."""
+
+from typing import Sequence
+
+from pysdmx.errors import Invalid
+from pysdmx.io.pd import PandasDataset
+from pysdmx.io.xml.__data_aux import (
+    get_data_objects,
+)
+from pysdmx.io.xml.__parse_xml import parse_xml
+from pysdmx.io.xml.__ss_aux_reader import (
+    _parse_structure_specific_data,
+)
+from pysdmx.io.xml.__tokens import (
+    STR_REF,
+    STR_SPE,
+)
+
+
+def read(input_str: str, validate: bool = True) -> Sequence[PandasDataset]:
+    """Reads an SDMX-ML 3.0 and returns a Sequence of Datasets.
+
+    Args:
+        input_str: SDMX-ML data to read.
+        validate: If True, the XML data will be validated against the XSD.
+    """
+    dict_info = parse_xml(input_str, validate=validate)
+    if STR_SPE not in dict_info:
+        raise Invalid(
+            "This SDMX document is not an SDMX-ML StructureSpecificData."
+        )
+    dataset_info, str_info = get_data_objects(dict_info[STR_SPE])
+    datasets = []
+    for dataset in dataset_info:
+        ds = _parse_structure_specific_data(
+            dataset, str_info[dataset[STR_REF]]
+        )
+        datasets.append(ds)
+    return datasets

pysdmx/io/xml/sdmx30/writer/__init__.py

@@ -0,0 +1 @@
+"""SDMX 3.0 writer package."""

pysdmx/io/xml/sdmx30/writer/structure.py

@@ -0,0 +1,67 @@
+"""Module for writing metadata to XML files."""
+
+from pathlib import Path
+from typing import Dict, Optional, Sequence, Union
+
+from pysdmx.io.format import Format
+from pysdmx.io.xml.__structure_aux_writer import (
+    STR_DICT_TYPE_LIST_30,
+    __write_structures,
+)
+from pysdmx.io.xml.__write_aux import (
+    __write_header,
+    create_namespaces,
+    get_end_message,
+)
+from pysdmx.model.__base import MaintainableArtefact
+from pysdmx.model.message import Header
+
+
+def write(
+    structures: Sequence[MaintainableArtefact],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+) -> Optional[str]:
+    """This function writes a SDMX-ML file from the Message Content.
+
+    Args:
+        structures: The content to be written
+        output_path: The path to save the file
+        prettyprint: Prettyprint or not
+        header: The header to be used (generated if None)
+
+    Returns:
+        The XML string if output_path is empty, None otherwise
+    """
+    type_ = Format.STRUCTURE_SDMX_ML_3_0
+    elements = {structure.short_urn: structure for structure in structures}
+    if header is None:
+        header = Header()
+
+    content: Dict[str, Dict[str, MaintainableArtefact]] = {}
+    for urn, element in elements.items():
+        list_ = STR_DICT_TYPE_LIST_30[type(element)]
+        if list_ not in content:
+            content[list_] = {}
+        content[list_][urn] = element
+
+    # Generating the initial tag with namespaces
+    outfile = create_namespaces(type_, prettyprint=prettyprint)
+    # Generating the header
+    outfile += __write_header(header, prettyprint, data_message=False)
+    # Writing the content
+    outfile += __write_structures(content, prettyprint, references_30=True)
+
+    outfile += get_end_message(type_, prettyprint)
+
+    output_path = (
+        str(output_path) if isinstance(output_path, Path) else output_path
+    )
+
+    if output_path is None or output_path == "":
+        return outfile
+
+    with open(output_path, "w", encoding="UTF-8", errors="replace") as f:
+        f.write(outfile)
+    return None

pysdmx/io/xml/sdmx30/writer/structure_specific.py

@@ -0,0 +1,108 @@
+# mypy: disable-error-code="union-attr"
+"""Module for writing SDMX-ML 3.0 Structure Specific data messages."""
+
+from pathlib import Path
+from typing import Dict, Optional, Sequence, Union
+
+from pysdmx.io.format import Format
+from pysdmx.io.pd import PandasDataset
+from pysdmx.io.xml.__tokens import (
+    DSD_LOW,
+    PROV_AGREEMENT,
+    REGISTRY_LOW,
+)
+from pysdmx.io.xml.__write_aux import (
+    __write_header,
+    create_namespaces,
+    get_end_message,
+)
+from pysdmx.io.xml.__write_data_aux import (
+    check_content_dataset,
+    check_dimension_at_observation,
+)
+from pysdmx.io.xml.__write_structure_specific_aux import (
+    __write_data_structure_specific,
+)
+from pysdmx.model.message import Header
+from pysdmx.util import parse_short_urn
+
+
+def write(
+    datasets: Sequence[PandasDataset],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+    dimension_at_observation: Optional[Dict[str, str]] = None,
+) -> Optional[str]:
+    """Write data to SDMX-ML 3.0 Structure Specific format.
+
+    Args:
+        datasets: The datasets to be written.
+        output_path: The path to save the file.
+        prettyprint: Prettyprint or not.
+        header: The header to be used (generated if None).
+        dimension_at_observation:
+          The mapping between the dataset and the dimension at observation.
+
+    Returns:
+        The XML string if path is empty, None otherwise.
+    """
+    ss_namespaces = ""
+    type_ = Format.DATA_SDMX_ML_3_0
+
+    # Checking if we have datasets,
+    # we need to ensure we can write them correctly
+    check_content_dataset(datasets)
+    content = {dataset.short_urn: dataset for dataset in datasets}
+
+    if header is None:
+        header = Header()
+
+    # Checking the dimension at observation mapping
+    dim_mapping = check_dimension_at_observation(
+        content, dimension_at_observation
+    )
+    header.structure = dim_mapping
+    add_namespace_structure = True
+    for i, (short_urn, dimension) in enumerate(header.structure.items()):
+        reference = parse_short_urn(short_urn)
+        pre_urn = (
+            REGISTRY_LOW if reference.sdmx_type == PROV_AGREEMENT else DSD_LOW
+        )
+        ss_namespaces += (
+            f'xmlns:ns{i + 1}="urn:sdmx:org.sdmx'
+            f".infomodel.{pre_urn}.{short_urn}"
+            f':ObsLevelDim:{dimension}" '
+        )
+
+    # Generating the initial tag with namespaces
+    outfile = create_namespaces(type_, ss_namespaces, prettyprint)
+    # Generating the header
+    outfile += __write_header(
+        header,
+        prettyprint,
+        add_namespace_structure,
+        data_message=True,
+        references_30=True,
+    )
+    # Writing the content
+    outfile += __write_data_structure_specific(
+        datasets=content,
+        dim_mapping=dim_mapping,
+        prettyprint=prettyprint,
+        references_30=True,
+    )
+
+    outfile += get_end_message(type_, prettyprint)
+
+    output_path = (
+        str(output_path) if isinstance(output_path, Path) else output_path
+    )
+
+    if output_path is None or output_path == "":
+        return outfile
+
+    with open(output_path, "w", encoding="UTF-8", errors="replace") as f:
+        f.write(outfile)
+
+    return None