pysdmx 1.10.1__py3-none-any.whl → 1.11.0__py3-none-any.whl

pysdmx/io/json/sdmxjson2/writer/v2_1/structure.py

@@ -0,0 +1,33 @@
+"""Writer interface for SDMX-JSON 2.1.0 Structure messages."""
+
+from pathlib import Path
+from typing import Optional, Sequence, Union
+
+from pysdmx.io.json.sdmxjson2.writer._helper import write_structure_msg
+from pysdmx.model.__base import MaintainableArtefact
+from pysdmx.model.message import Header
+
+
+def write(
+    structures: Sequence[MaintainableArtefact],
+    output_path: Optional[Union[str, Path]] = None,
+    prettyprint: bool = True,
+    header: Optional[Header] = None,
+) -> Optional[str]:
+    """Write maintainable SDMX artefacts in SDMX-JSON 2.1.0.
+
+    Args:
+        structures: The maintainable SDMX artefacts to be serialized.
+        output_path: The path to save the JSON file. If None or empty, the
+            serialized content is returned as a string instead.
+        prettyprint: Whether to format the JSON output with indentation (True)
+            or output compact JSON without extra whitespace (False).
+        header: The header to be used in the SDMX-JSON message
+            (will be generated if no header is supplied).
+
+    Returns:
+        The JSON string if output_path is None or empty, None otherwise.
+    """
+    return write_structure_msg(
+        structures, output_path, prettyprint, header, "2.1"
+    )

pysdmx/io/reader.py

@@ -80,7 +80,10 @@ def read_sdmx(  # noqa: C901
         header = read_header(input_str, validate=validate)
         # SDMX-ML 3.1 Structure
         result_structures = read_structure(input_str, validate=validate)
-    elif read_format == Format.STRUCTURE_SDMX_JSON_2_0_0:
+    elif read_format in (
+        Format.STRUCTURE_SDMX_JSON_2_0_0,
+        Format.STRUCTURE_SDMX_JSON_2_1_0,
+    ):
         from pysdmx.io.json.sdmxjson2.reader.structure import (
             read as read_struct,
         )
@@ -90,7 +93,10 @@ def read_sdmx(  # noqa: C901
         result_structures = (
             struct_msg.structures if struct_msg.structures else []
         )
-    elif read_format == Format.REFMETA_SDMX_JSON_2_0_0:
+    elif read_format in (
+        Format.REFMETA_SDMX_JSON_2_0_0,
+        Format.REFMETA_SDMX_JSON_2_1_0,
+    ):
         from pysdmx.io.json.sdmxjson2.reader.metadata import (
             read as read_refmeta,
         )
@@ -174,7 +180,10 @@ def read_sdmx(  # noqa: C901
         return Message(header=header, data=result_data)
     elif read_format == Format.REGISTRY_SDMX_ML_2_1:
         return Message(header=header, submission=result_submission)
-    elif read_format == Format.REFMETA_SDMX_JSON_2_0_0:
+    elif read_format in (
+        Format.REFMETA_SDMX_JSON_2_0_0,
+        Format.REFMETA_SDMX_JSON_2_1_0,
+    ):
         return Message(header=header, reports=reports)
     # TODO: Ensure we have changed the signature of the structure readers
     return Message(header=header, structures=result_structures)

pysdmx/io/writer.py

@@ -24,10 +24,16 @@ WRITERS = {
     Format.DATA_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer.structure_specific",
     Format.STRUCTURE_SDMX_ML_3_1: "pysdmx.io.xml.sdmx31.writer.structure",
     Format.STRUCTURE_SDMX_JSON_2_0_0: (
-        "pysdmx.io.json.sdmxjson2.writer.structure"
+        "pysdmx.io.json.sdmxjson2.writer.v2_0.structure"
+    ),
+    Format.STRUCTURE_SDMX_JSON_2_1_0: (
+        "pysdmx.io.json.sdmxjson2.writer.v2_1.structure"
     ),
     Format.REFMETA_SDMX_JSON_2_0_0: (
-        "pysdmx.io.json.sdmxjson2.writer.metadata"
+        "pysdmx.io.json.sdmxjson2.writer.v2_0.metadata"
+    ),
+    Format.REFMETA_SDMX_JSON_2_1_0: (
+        "pysdmx.io.json.sdmxjson2.writer.v2_1.metadata"
     ),
 }
 
@@ -36,9 +42,13 @@ STRUCTURE_WRITERS = (
     Format.STRUCTURE_SDMX_ML_3_0,
     Format.STRUCTURE_SDMX_ML_3_1,
     Format.STRUCTURE_SDMX_JSON_2_0_0,
+    Format.STRUCTURE_SDMX_JSON_2_1_0,
 )
 
-REFMETA_WRITERS = (Format.REFMETA_SDMX_JSON_2_0_0,)
+REFMETA_WRITERS = (
+    Format.REFMETA_SDMX_JSON_2_0_0,
+    Format.REFMETA_SDMX_JSON_2_1_0,
+)
 
 
 def write_sdmx(

pysdmx/io/xml/__ss_aux_reader.py

@@ -1,7 +1,7 @@
 """SDMX XML StructureSpecificData reader aux module."""
 
 import itertools
-from typing import Any, Dict
+from typing import Any, Dict, List
 
 import numpy as np
 import pandas as pd
@@ -40,23 +40,22 @@ def _reading_str_series(dataset: Dict[str, Any]) -> pd.DataFrame:
     return df
 
 
-def _reading_group_data(dataset: Dict[str, Any]) -> pd.DataFrame:
+def _reading_group_data(dataset: Dict[str, Any]) -> List[pd.DataFrame]:
     # Structure Specific Group Data
-    test_list = []
-    df = None
+    group_dfs = []
     dataset[GROUP] = add_list(dataset[GROUP])
     for data in dataset[GROUP]:
-        test_list.append(dict(data.items()))
-        test_list, df = __process_df(test_list, df)
-    test_list, df = __process_df(test_list, df, is_end=True)
+        group_dict = dict(data.items())
+        group_df = pd.DataFrame([group_dict])
 
-    cols_to_delete = [x for x in df.columns if ":type" in x]
-    for x in cols_to_delete:
-        del df[x]
+        # Remove :type columns
+        cols_to_delete = [x for x in group_df.columns if ":type" in x]
+        for x in cols_to_delete:
+            del group_df[x]
 
-    df = df.drop_duplicates(keep="first").reset_index(drop=True)
+        group_dfs.append(group_df)
 
-    return df
+    return group_dfs
 
 
 def _get_at_att_str(dataset: Dict[str, Any]) -> Dict[str, Any]:
@@ -76,11 +75,34 @@ def _parse_structure_specific_data(
         # Structure Specific Series
         df = _reading_str_series(dataset)
         if GROUP in dataset:
-            df_group = _reading_group_data(dataset)
-            common_columns = list(
-                set(df.columns).intersection(set(df_group.columns))
-            )
-            df = pd.merge(df, df_group, on=common_columns, how="left")
+            group_dfs = _reading_group_data(dataset)
+            original_columns = df.columns.tolist()
+            for group_df in group_dfs:
+                # Find non-NaN columns in this group
+                non_nan_cols = [
+                    col
+                    for col in group_df.columns
+                    if not group_df[col].isna().all()
+                ]
+
+                # Merge keys are intersection of original and non-NaN cols
+                merge_cols = list(
+                    set(original_columns).intersection(set(non_nan_cols))
+                )
+
+                group_df = group_df.drop_duplicates(merge_cols, keep="first")
+                df = pd.merge(
+                    df,
+                    group_df,
+                    on=merge_cols,
+                    how="left",
+                    suffixes=("", "_drop"),
+                )
+                for col in list(df.columns):
+                    if col.endswith("_drop"):
+                        original = col[:-5]
+                        df[original] = df[original].fillna(df[col])
+                        df.drop(col, axis=1, inplace=True)
     elif OBS in dataset:
         dataset[OBS] = add_list(dataset[OBS])
         # Structure Specific All dimensions

pysdmx/io/xml/__structure_aux_reader.py

@@ -1,9 +1,10 @@
 """Parsers for reading metadata."""
 
 from datetime import datetime
-from typing import Any, Callable, Dict, List, Optional, Sequence, Union
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union
 
 from msgspec import Struct
+from msgspec.structs import asdict
 
 from pysdmx.io.xml.__tokens import (
     AGENCIES,
@@ -29,6 +30,8 @@ from pysdmx.io.xml.__tokens import (
     CLS,
     CODE,
     CODES_LOW,
+    COMPONENT_MAP,
+    COMPONENT_MAPS,
     COMPS,
     CON,
     CON_ID,
@@ -44,6 +47,7 @@ from pysdmx.io.xml.__tokens import (
     CUSTOM_TYPE_SCHEMES,
     CUSTOM_TYPES,
     DATA_PROV,
+    DATE_PATTERN_MAP,
     DEPARTMENT,
     DESC,
     DFW,
@@ -64,6 +68,8 @@ from pysdmx.io.xml.__tokens import (
     FACETS,
     FAX,
     FAXES,
+    FIXED_VALUE_MAP,
+    FIXED_VALUE_MAPS,
     GROUP,
     GROUP_DIM,
     GROUPS_LOW,
@@ -98,6 +104,8 @@ from pysdmx.io.xml.__tokens import (
     PROV_AGREEMENT,
     PROV_AGREEMENTS,
     REF,
+    REPRESENTATION_MAP,
+    REPRESENTATION_MAPS,
     REQUIRED,
     ROLE,
     RULE,
@@ -110,6 +118,8 @@ from pysdmx.io.xml.__tokens import (
     STR_URL_LOW,
     STR_USAGE,
     STRUCTURE,
+    STRUCTURE_MAP,
+    STRUCTURE_MAPS,
     TELEPHONE,
     TELEPHONES,
     TEXT,
@@ -152,10 +162,19 @@ from pysdmx.model import (
     AgencyScheme,
     Code,
     Codelist,
+    ComponentMap,
     Concept,
     ConceptScheme,
     DataType,
+    DatePatternMap,
     Facets,
+    FixedValueMap,
+    ImplicitComponentMap,
+    MultiComponentMap,
+    MultiValueMap,
+    RepresentationMap,
+    StructureMap,
+    ValueMap,
     VtlCodelistMapping,
     VtlConceptMapping,
 )
@@ -207,6 +226,11 @@ STRUCTURES_MAPPING = {
     UDO_SCHEME: UserDefinedOperatorScheme,
     TRANS_SCHEME: TransformationScheme,
     VTL_MAPPING_SCHEME: VtlMappingScheme,
+    STRUCTURE_MAP: StructureMap,
+    COMPONENT_MAP: ComponentMap,
+    FIXED_VALUE_MAP: FixedValueMap,
+    DATE_PATTERN_MAP: DatePatternMap,
+    REPRESENTATION_MAP: RepresentationMap,
     NAME_PER_SCHEME: NamePersonalisationScheme,
     CUSTOM_TYPE_SCHEME: CustomTypeScheme,
     PROV_AGREEMENTS: ProvisionAgreement,
@@ -308,6 +332,10 @@ class StructureParser(Struct):
     rulesets: Dict[str, RulesetScheme] = {}
     udos: Dict[str, UserDefinedOperatorScheme] = {}
     vtl_mappings: Dict[str, VtlMappingScheme] = {}
+    structure_maps: Dict[str, StructureMap] = {}
+    component_maps: Dict[str, ComponentMap] = {}
+    fixed_value_maps: Dict[str, FixedValueMap] = {}
+    representation_maps: Dict[str, RepresentationMap] = {}
     name_personalisations: Dict[str, NamePersonalisationScheme] = {}
     custom_types: Dict[str, CustomTypeScheme] = {}
     transformations: Dict[str, TransformationScheme] = {}
@@ -529,16 +557,11 @@ class StructureParser(Struct):
         if FACETS.lower() in rep:
             representation_info[LOCAL_FACETS_LOW] = rep.pop(FACETS.lower())
 
-    def __format_con_id(self, concept_ref: Dict[str, Any]) -> Dict[str, Any]:
-        rep = {}
+    def __format_con_id(
+        self, concept_ref: Union[str, Dict[str, Any]]
+    ) -> Dict[str, Any]:
         if isinstance(concept_ref, str):
             item_reference = parse_urn(concept_ref)
-            scheme_reference = Reference(
-                sdmx_type=CS,
-                agency=item_reference.agency,
-                id=item_reference.id,
-                version=item_reference.version,
-            )
         else:
             item_reference = ItemReference(
                 sdmx_type=concept_ref[CLASS],
@@ -547,27 +570,44 @@ class StructureParser(Struct):
                 version=concept_ref[PAR_VER],
                 item_id=concept_ref[ID],
             )
-            scheme_reference = Reference(
-                sdmx_type=CS,
-                agency=concept_ref[AGENCY_ID],
-                id=concept_ref[PAR_ID],
-                version=concept_ref[PAR_VER],
-            )
+        scheme_reference = Reference(
+            sdmx_type=CS,
+            agency=item_reference.agency,
+            id=item_reference.id,
+            version=item_reference.version,
+        )
 
         concept_scheme = self.concepts.get(str(scheme_reference))
+
         if concept_scheme is None:
             return {CON: item_reference}
+
+        short_urn = str(item_reference)
         for con in concept_scheme.concepts:
-            if isinstance(concept_ref, str):
-                if con.id == item_reference.item_id:
-                    rep[CON] = parse_urn(concept_ref)
-                    break
-            elif con.id == concept_ref[ID]:
-                rep[CON] = con
-                break
-        if CON not in rep:
-            return {CON: item_reference}
-        return rep
+            con_short = str(
+                ItemReference(
+                    sdmx_type=item_reference.sdmx_type,
+                    agency=item_reference.agency,
+                    id=item_reference.id,
+                    version=item_reference.version,
+                    item_id=con.id,
+                )
+            )
+
+            if con_short == short_urn:
+                if con.urn is None:
+                    con = Concept(
+                        **{
+                            **asdict(con),
+                            "urn": (
+                                "urn:sdmx:org.sdmx.infomodel.conceptscheme."
+                                f"{short_urn}"
+                            ),
+                        }
+                    )
+                return {CON: con}
+
+        return {CON: item_reference}
 
     @staticmethod
     def __get_attachment_level(  # noqa: C901
@@ -751,6 +791,11 @@ class StructureParser(Struct):
             del comp[ATT_REL]
 
         if ME_REL in comp:
+            measures = add_list(comp[ME_REL][MSR])
+            if len(measures) == 1 and measures[0] == "OBS_VALUE":
+                comp[ATT_LVL] = "O"
+            else:
+                comp[ATT_LVL] = ",".join(measures)
             del comp[ME_REL]
 
         if AS_STATUS in comp or USAGE in comp:
@@ -762,14 +807,10 @@ class StructureParser(Struct):
                 comp[REQUIRED] = False
             del comp[status_key]
 
-        if "position" in comp:
-            del comp["position"]
-
-        if ANNOTATIONS in comp:
-            del comp[ANNOTATIONS]
-
-        if CON_ROLE in comp:
-            del comp[CON_ROLE]
+        unwanted_keys = ["position", ANNOTATIONS, CON_ROLE]
+        for key in unwanted_keys:
+            if key in comp:
+                del comp[key]
 
         return Component(**comp)
 
@@ -1035,6 +1076,124 @@ class StructureParser(Struct):
             )
         return json_elem
 
+    def __build_component_map(
+        self, child_dict: Dict[str, Any]
+    ) -> Union[ComponentMap, MultiComponentMap, ImplicitComponentMap]:
+        if "values" not in child_dict:
+            return ImplicitComponentMap(
+                source=child_dict["source"],
+                target=child_dict["target"],
+            )
+
+        src_list = add_list(child_dict.get("source"))
+        tgt_list = add_list(child_dict.get("target"))
+
+        if len(src_list) != 1 or len(tgt_list) != 1:
+            return MultiComponentMap(
+                source=src_list,
+                target=tgt_list,
+                values=child_dict["values"],
+            )
+
+        return ComponentMap(
+            source=src_list[0],
+            target=tgt_list[0],
+            values=child_dict["values"],
+        )
+
+    def __build_representation_mapping(
+        self, child_dict: Dict[str, Any]
+    ) -> Union[ValueMap, MultiValueMap]:
+        src = child_dict.get("source")
+        tgt = child_dict.get("target")
+
+        src_list = add_list(src) if src is not None else []
+        tgt_list = add_list(tgt) if tgt is not None else []
+
+        if len(src_list) != 1 or len(tgt_list) != 1:
+            return MultiValueMap(
+                source=src_list,
+                target=tgt_list,
+                valid_from=child_dict.get("valid_from"),
+                valid_to=child_dict.get("valid_to"),
+            )
+
+        return ValueMap(
+            source=src_list[0],
+            target=tgt_list[0],
+            valid_from=child_dict.get("valid_from"),
+            valid_to=child_dict.get("valid_to"),
+        )
+
+    def __format_maps(self, element: Dict[str, Any]) -> Dict[str, Any]:
+        if "sourcePattern" in element:
+            element["pattern_type"] = (
+                # DatePatternMap.pattern_type defaults value is fixed
+                "variable" if "FrequencyDimension" in element else "fixed"
+            )
+
+        renames = {
+            "Source": "source",
+            "Target": "target",
+            "Value": "value",
+            "SourceCodelist": "source",
+            "TargetCodelist": "target",
+            "SourceValue": "source",
+            "TargetValue": "target",
+            "RepresentationMap": "values",
+            "sourcePattern": "pattern",
+            "resolvePeriod": "resolve_period",
+            "TargetFrequencyID": "frequency",
+            "FrequencyDimension": "frequency",
+            "validFrom": "valid_from",
+            "validTo": "valid_to",
+        }
+
+        for xml_key, py_key in renames.items():
+            if xml_key in element:
+                element[py_key] = element.pop(xml_key)
+
+        child_class_mapping: Dict[str, Type[Any]] = {
+            "ComponentMap": ComponentMap,
+            "FixedValueMap": FixedValueMap,
+            "DatePatternMap": DatePatternMap,
+            "RepresentationMapping": ValueMap,
+        }
+
+        MapChild = Union[
+            ComponentMap,
+            MultiComponentMap,
+            ImplicitComponentMap,
+            FixedValueMap,
+            DatePatternMap,
+            ValueMap,
+            MultiValueMap,
+        ]
+        consolidated_children: List[MapChild] = []
+
+        for xml_tag, target_class in child_class_mapping.items():
+            if xml_tag not in element:
+                continue
+
+            for child_dict in add_list(element.pop(xml_tag)):
+                self.__format_maps(child_dict)
+
+                if xml_tag == "ComponentMap":
+                    consolidated_children.append(
+                        self.__build_component_map(child_dict)
+                    )
+                elif xml_tag == "RepresentationMapping":
+                    consolidated_children.append(
+                        self.__build_representation_mapping(child_dict)
+                    )
+                else:
+                    consolidated_children.append(target_class(**child_dict))
+
+        if consolidated_children:
+            element["maps"] = consolidated_children
+
+        return element
+
     def __format_scheme(
         self, json_elem: Dict[str, Any], scheme: str, item: str
     ) -> Dict[str, ItemScheme]:
@@ -1116,6 +1275,7 @@ class StructureParser(Struct):
             element = self.__format_validity(element)
             element = self.__format_groups(element)
             element = self.__format_components(element)
+            element = self.__format_maps(element)
             if item == PROV_AGREEMENT:
                 element = self.__format_prov_agreement(element)
 
@@ -1300,6 +1460,34 @@ class StructureParser(Struct):
                 ),
                 "transformations",
             ),
+            STRUCTURE_MAPS: process_structure(
+                STRUCTURE_MAPS,
+                lambda data: self.__format_schema(
+                    data, STRUCTURE_MAP, STRUCTURE_MAP
+                ),
+                "structure_maps",
+            ),
+            COMPONENT_MAPS: process_structure(
+                COMPONENT_MAPS,
+                lambda data: self.__format_schema(
+                    data, COMPONENT_MAP, COMPONENT_MAP
+                ),
+                "component_maps",
+            ),
+            FIXED_VALUE_MAPS: process_structure(
+                FIXED_VALUE_MAPS,
+                lambda data: self.__format_schema(
+                    data, FIXED_VALUE_MAP, FIXED_VALUE_MAP
+                ),
+                "fixed_value_maps",
+            ),
+            REPRESENTATION_MAPS: process_structure(
+                REPRESENTATION_MAPS,
+                lambda data: self.__format_schema(
+                    data, REPRESENTATION_MAP, REPRESENTATION_MAP
+                ),
+                "representation_maps",
+            ),
             TRANS_SCHEMES: process_structure(
                 TRANS_SCHEMES,
                 lambda data: self.__format_scheme(