pysdmx 1.3.0__py3-none-any.whl → 1.4.0rc1__py3-none-any.whl

pysdmx/io/xml/sdmx21/reader/structure_specific.py

@@ -1,108 +1,24 @@
-"""SDMX 2.1 XML StructureSpecificData reader module."""
+"""SDMX XML 2.1 StructureSpecificData reader module."""
 
-import itertools
-from typing import Any, Dict, Sequence
-
-import numpy as np
-import pandas as pd
+from typing import Sequence
 
 from pysdmx.errors import Invalid
 from pysdmx.io.pd import PandasDataset
-from pysdmx.io.xml.sdmx21.__tokens import (
-    EXCLUDED_ATTRIBUTES,
-    GROUP,
-    OBS,
-    SERIES,
+from pysdmx.io.xml.__data_aux import (
+    get_data_objects,
+)
+from pysdmx.io.xml.__parse_xml import parse_xml
+from pysdmx.io.xml.__ss_aux_reader import (
+    _parse_structure_specific_data,
+)
+from pysdmx.io.xml.__tokens import (
     STR_REF,
     STR_SPE,
 )
-from pysdmx.io.xml.sdmx21.reader.__data_aux import (
-    __process_df,
-    get_data_objects,
-)
-from pysdmx.io.xml.sdmx21.reader.__parse_xml import parse_xml
-from pysdmx.io.xml.utils import add_list
-from pysdmx.model.dataset import ActionType
-
-
-def __reading_str_series(dataset: Dict[str, Any]) -> pd.DataFrame:
-    # Structure Specific Series
-    test_list = []
-    df = None
-    dataset[SERIES] = add_list(dataset[SERIES])
-    for data in dataset[SERIES]:
-        keys = dict(itertools.islice(data.items(), len(data)))
-        if OBS in data:
-            del keys[OBS]
-            data[OBS] = add_list(data[OBS])
-            for j in data[OBS]:
-                test_list.append({**keys, **j})
-        else:
-            test_list.append(keys)
-        test_list, df = __process_df(test_list, df)
-
-    test_list, df = __process_df(test_list, df, is_end=True)
-
-    return df
-
-
-def __reading_group_data(dataset: Dict[str, Any]) -> pd.DataFrame:
-    # Structure Specific Group Data
-    test_list = []
-    df = None
-    dataset[GROUP] = add_list(dataset[GROUP])
-    for data in dataset[GROUP]:
-        test_list.append(dict(data.items()))
-        test_list, df = __process_df(test_list, df)
-    test_list, df = __process_df(test_list, df, is_end=True)
-
-    cols_to_delete = [x for x in df.columns if ":type" in x]
-    for x in cols_to_delete:
-        del df[x]
-
-    df = df.drop_duplicates(keep="first").reset_index(drop=True)
-
-    return df
-
-
-def __get_at_att_str(dataset: Dict[str, Any]) -> Dict[str, Any]:
-    """Gets the elements of the dataset if it is Structure Specific Data."""
-    return {k: dataset[k] for k in dataset if k not in EXCLUDED_ATTRIBUTES}
-
-
-def __parse_structure_specific_data(
-    dataset: Dict[str, Any], structure_info: Dict[str, Any]
-) -> PandasDataset:
-    attached_attributes = __get_at_att_str(dataset)
-
-    df = pd.DataFrame()
-
-    # Parsing data
-    if SERIES in dataset:
-        # Structure Specific Series
-        df = __reading_str_series(dataset)
-        if GROUP in dataset:
-            df_group = __reading_group_data(dataset)
-            common_columns = list(
-                set(df.columns).intersection(set(df_group.columns))
-            )
-            df = pd.merge(df, df_group, on=common_columns, how="left")
-    elif OBS in dataset:
-        dataset[OBS] = add_list(dataset[OBS])
-        # Structure Specific All dimensions
-        df = pd.DataFrame(dataset[OBS]).replace(np.nan, "")
-
-    urn = f"{structure_info['structure_type']}={structure_info['unique_id']}"
-    action = dataset.get("action", "Information")
-    action = ActionType(action)
-
-    return PandasDataset(
-        structure=urn, attributes=attached_attributes, data=df, action=action
-    )
 
 
 def read(input_str: str, validate: bool = True) -> Sequence[PandasDataset]:
-    """Reads an SDMX-ML 2.1 Generic file and returns a Sequence of Datasets.
+    """Reads an SDMX-ML 2.1 and returns a Sequence of Datasets.
 
     Args:
         input_str: SDMX-ML data to read.
@@ -111,12 +27,12 @@ def read(input_str: str, validate: bool = True) -> Sequence[PandasDataset]:
     dict_info = parse_xml(input_str, validate=validate)
     if STR_SPE not in dict_info:
         raise Invalid(
-            "This SDMX document is not an SDMX-ML 2.1 StructureSpecificData."
+            "This SDMX document is not an SDMX-ML StructureSpecificData."
         )
     dataset_info, str_info = get_data_objects(dict_info[STR_SPE])
     datasets = []
     for dataset in dataset_info:
-        ds = __parse_structure_specific_data(
+        ds = _parse_structure_specific_data(
             dataset, str_info[dataset[STR_REF]]
         )
         datasets.append(ds)

pysdmx/io/xml/sdmx21/reader/submission.py

@@ -3,7 +3,8 @@
 from typing import Any, Dict, Sequence
 
 from pysdmx.errors import Invalid
-from pysdmx.io.xml.sdmx21.__tokens import (
+from pysdmx.io.xml.__parse_xml import parse_xml
+from pysdmx.io.xml.__tokens import (
     ACTION,
     MAINTAINABLE_OBJECT,
     REG_INTERFACE,
@@ -14,7 +15,6 @@ from pysdmx.io.xml.sdmx21.__tokens import (
     SUBMITTED_STRUCTURE,
     URN,
 )
-from pysdmx.io.xml.sdmx21.reader.__parse_xml import parse_xml
 from pysdmx.model.submission import SubmissionResult
 from pysdmx.util import parse_urn
 

pysdmx/io/xml/sdmx21/writer/error.py

@@ -1,7 +1,7 @@
 """Module for writing SDMX-ML 2.1 Error messages."""
 
 from pysdmx.io.format import Format
-from pysdmx.io.xml.sdmx21.writer.__write_aux import __namespaces_from_type
+from pysdmx.io.xml.__write_aux import __namespaces_from_type
 
 
 def write() -> None:

pysdmx/io/xml/sdmx21/writer/generic.py

@@ -1,13 +1,14 @@
 # mypy: disable-error-code="union-attr"
 """Module for writing SDMX-ML 2.1 Generic data messages."""
 
-from typing import Any, Dict, List, Optional, Sequence, Tuple
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Sequence, Tuple, Union
 
 import pandas as pd
 
 from pysdmx.io.format import Format
 from pysdmx.io.pd import PandasDataset
-from pysdmx.io.xml.sdmx21.writer.__write_aux import (
+from pysdmx.io.xml.__write_aux import (
     ABBR_GEN,
     ABBR_MSG,
     ALL_DIM,
@@ -17,13 +18,13 @@ from pysdmx.io.xml.sdmx21.writer.__write_aux import (
     get_end_message,
     get_structure,
 )
-from pysdmx.io.xml.sdmx21.writer.__write_data_aux import (
+from pysdmx.io.xml.__write_data_aux import (
     check_content_dataset,
     check_dimension_at_observation,
     get_codes,
     writing_validation,
 )
-from pysdmx.io.xml.sdmx21.writer.config import CHUNKSIZE
+from pysdmx.io.xml.config import CHUNKSIZE
 from pysdmx.model.message import Header
 from pysdmx.util import parse_short_urn
 
@@ -159,6 +160,7 @@ def __write_data_single_dataset(
     outfile = ""
     structure_urn = get_structure(dataset)
     id_structure = parse_short_urn(structure_urn).id
+    dataset.data = dataset.data.fillna("").astype(str).replace("nan", "")
 
     nl = "\n" if prettyprint else ""
     child1 = "\t" if prettyprint else ""
@@ -277,7 +279,7 @@ def __series_processing(
 ) -> str:
     def __generate_series_str() -> str:
         out_list: List[str] = []
-        data.groupby(by=series_codes + series_att_codes).apply(
+        data.groupby(by=series_codes + series_att_codes)[data.columns].apply(
             lambda x: __format_dict_ser(out_list, x)
         )
 
@@ -383,7 +385,7 @@ def __format_ser_str(
 
 def write(
     datasets: Sequence[PandasDataset],
-    output_path: str = "",
+    output_path: Optional[Union[str, Path]] = None,
     prettyprint: bool = True,
     header: Optional[Header] = None,
     dimension_at_observation: Optional[Dict[str, str]] = None,
@@ -425,7 +427,11 @@ def write(
 
     outfile += get_end_message(type_, prettyprint)
 
-    if output_path == "":
+    output_path = (
+        str(output_path) if isinstance(output_path, Path) else output_path
+    )
+
+    if output_path is None or output_path == "":
         return outfile
 
     with open(output_path, "w", encoding="UTF-8", errors="replace") as f: