pysdmx 1.5.1__py3-none-any.whl → 1.6.0__py3-none-any.whl

pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.5.1"
+__version__ = "1.6.0"

pysdmx/api/fmr/__init__.py

@@ -571,13 +571,16 @@ class RegistryClient(__BaseRegistryClient):
         d = DataflowDetails(detail) if isinstance(detail, str) else detail
         sq, dr = super()._df_details(d)
         if sq:
-            cmps = self.get_schema("dataflow", agency, id, version).components
+            schema = self.get_schema("dataflow", agency, id, version)
+            cmps = schema.components
+            grps = schema.groups
         else:
             cmps = None
+            grps = None
         query = super()._dataflow_details_q(agency, id, version, dr)
         out = self.__fetch(query)
         return super()._out(
-            out, self.deser.dataflow_info, cmps, agency, id, version
+            out, self.deser.dataflow_info, cmps, grps, agency, id, version
         )
 
     def get_dataflows(
@@ -1034,12 +1037,14 @@ class AsyncRegistryClient(__BaseRegistryClient):
                 version,
             )
             cmps = schema.components
+            grps = schema.groups
         else:
             cmps = None
+            grps = None
         query = super()._dataflow_details_q(agency, id, version, dr)
         out = await self.__fetch(query)
         return super()._out(
-            out, self.deser.dataflow_info, cmps, agency, id, version
+            out, self.deser.dataflow_info, cmps, grps, agency, id, version
         )
 
     async def get_dataflows(

pysdmx/api/fmr/maintenance.py

@@ -0,0 +1,158 @@
+"""Upload metadata to an FMR instance."""
+
+from enum import Enum
+from typing import Optional, Sequence, Union
+
+import httpx
+import msgspec
+
+from pysdmx.io.json.sdmxjson2.writer import serializers
+from pysdmx.model import MetadataReport
+from pysdmx.model.__base import MaintainableArtefact
+from pysdmx.model.message import (
+    Header,
+    MetadataMessage,
+    StructureMessage,
+)
+from pysdmx.util._net_utils import map_httpx_errors
+
+
+class StructureAction(Enum):
+    """Enumeration that defines the action when updating metadata in the FMR.
+
+    Arguments:
+        Append: Metadata uploaded with action 'Append' may only add new
+            metadata and may not overwrite any existing metadata, i.e. any
+            attempt to update existing metadata will be rejected.
+        Merge: Metadata uploaded with action 'Merge' may add new metadata and
+            replace existing metadata. However, for Item Schemes (codelists,
+            concept schemes, etc.), the items submitted will be added to the
+            existing scheme. For example, if a codelist exists with codes A, B,
+            and C, and the same codelist is submitted with codes B and X, then
+            the resulting codelist will have codes A, B, C, X, i.e. code B has
+            been replaced while code X has been added.
+        Replace: Metadata uploaded with action 'Replace' may add new metadata,
+            and can also replace existing metadata with new ones. This is the
+            default.
+    """
+
+    Append = "Append"
+    Merge = "Merge"
+    Replace = "Replace"
+
+
+class RegistryMaintenanceClient:
+    """EXPERIMENTAL: A client to update metadata in the FMR."""
+
+    def __init__(
+        self,
+        api_endpoint: str,
+        user: str,
+        password: str,
+        pem: Optional[str] = None,
+        timeout: float = 60.0,
+    ):
+        """Instantiate a new client to update metadata in the target endpoint.
+
+        Args:
+            api_endpoint: The endpoint of the targeted service.
+            user: Username for authentication.
+            password: Password for authentication.
+            pem: In case the service exposed a certificate created by an
+                unknown certificate authority, you can pass a pem file for
+                this authority using this parameter.
+            timeout: The maximum number of seconds to wait before considering
+                that a request timed out. Defaults to 10 seconds.
+        """
+        if api_endpoint.endswith("/"):
+            api_endpoint = api_endpoint[0:-1]
+        self._api_endpoint = f"{api_endpoint}"
+        self._user = user
+        self._password = password
+        self._timeout = timeout
+        self._ssl_context = (
+            httpx.create_ssl_context(
+                verify=pem,
+            )
+            if pem
+            else httpx.create_ssl_context()
+        )
+        self._encoder = msgspec.json.Encoder()
+
+    def __post(
+        self,
+        message: Union[MetadataMessage, StructureMessage],
+        action: StructureAction,
+        endpoint: str,
+    ) -> None:
+        with httpx.Client(verify=self._ssl_context) as client:
+            try:
+                url = f"{endpoint}"
+                auth = httpx.BasicAuth(self._user, self._password)
+                headers = {
+                    "Content-Type": "application/text",
+                    "Action": action.value,
+                }
+                if isinstance(message, MetadataMessage):
+                    serializer = serializers.metadata_message
+                else:
+                    serializer = serializers.structure_message
+                bodyjs = self._encoder.encode(serializer.from_model(message))
+                r = client.post(
+                    url,
+                    headers=headers,
+                    content=bodyjs,
+                    timeout=self._timeout,
+                    auth=auth,
+                )
+                r.raise_for_status()
+            except (httpx.RequestError, httpx.HTTPStatusError) as e:
+                map_httpx_errors(e)
+
+    def put_structures(
+        self,
+        artefacts: Sequence[MaintainableArtefact],
+        header: Optional[Header] = None,
+        action: StructureAction = StructureAction.Replace,
+    ) -> None:
+        """EXPERIMENTAL: Upload SDMX structures to the FMR.
+
+        This method is experimental and its interface or behavior may change
+        without notice.
+
+        Args:
+            artefacts: The sequence of SDMX maintainable artefacts to upload.
+            header: Optional SDMX Header to include in the message. If not
+                supplied, pysdmx will generate one for you.
+            action: How to apply the changes in case of already existing
+                structures.
+        """
+        if not header:
+            header = Header()
+        message = StructureMessage(header=header, structures=artefacts)
+        endpoint = f"{self._api_endpoint}/ws/secure/sdmxapi/rest"
+        return self.__post(message, action, endpoint)
+
+    def put_metadata_reports(
+        self,
+        reports: Sequence[MetadataReport],
+        header: Optional[Header] = None,
+        action: StructureAction = StructureAction.Replace,
+    ) -> None:
+        """EXPERIMENTAL: Upload SDMX metadata reports to the FMR.
+
+        This method is experimental and its interface or behavior may change
+        without notice.
+
+        Args:
+            reports: A sequence of metadata reports to upload.
+            header: Optional SDMX Header to include in the message. If not
+                supplied, pysdmx will generate one for you.
+            action: How to apply the changes in case of already existing
+                structures.
+        """
+        if not header:
+            header = Header()
+        message = MetadataMessage(header=header, reports=reports)
+        endpoint = f"{self._api_endpoint}/ws/secure/sdmx/v2/metadata"
+        return self.__post(message, action, endpoint)

pysdmx/api/qb/structure.py

@@ -273,6 +273,7 @@ _API_RESOURCES = {
     "V2.1.0": _V2_0_RESOURCES,
     "V2.2.0": _V2_0_RESOURCES,
     "V2.2.1": _V2_0_RESOURCES,
+    "V2.2.2": _V2_0_RESOURCES,
     "LATEST": _V2_0_RESOURCES,
 }
 

pysdmx/api/qb/util.py

@@ -25,6 +25,7 @@ class ApiVersion(IntEnum):
     V2_1_0 = 9
     V2_2_0 = 10
     V2_2_1 = 11
+    V2_2_2 = 12
 
 
 MULT_SEP = re.compile(r"\+")

pysdmx/io/csv/__csv_aux_reader.py

@@ -0,0 +1,99 @@
+import pandas as pd
+
+from pysdmx.errors import Invalid
+from pysdmx.io.pd import PandasDataset
+from pysdmx.model.dataset import ActionType
+
+ACTION_SDMX_CSV_MAPPER_READING = {
+    "A": ActionType.Append,
+    "D": ActionType.Delete,
+    "R": ActionType.Replace,
+    "I": ActionType.Information,
+}
+
+
+def __generate_dataset_from_sdmx_csv(  # noqa: C901
+    data: pd.DataFrame,
+    references_21: bool = False,
+) -> PandasDataset:
+    urn = ""
+    df_csv = pd.DataFrame()
+    action = None
+
+    if {"STRUCTURE", "STRUCTURE_ID"}.issubset(data.columns):
+        action = ActionType.Information
+        if "ACTION" in data.columns:
+            unique_values = list(data["ACTION"].unique())
+            if len(unique_values) > 1 and "D" in unique_values:
+                unique_values.remove("D")
+                data = data[data["ACTION"] != "D"]
+            if len(unique_values) == 1:  # If there is only one value, use it
+                action_value = unique_values[0]
+                if action_value == "M":
+                    if not references_21:
+                        raise Invalid(
+                            "Invalid value on ACTION column",
+                            "Value 'M' is only allowed for"
+                            " SDMX-CSV 2.1 files.",
+                        )
+                    action = ActionType.Append
+                elif action_value in ACTION_SDMX_CSV_MAPPER_READING:
+                    action = ACTION_SDMX_CSV_MAPPER_READING[action_value]
+                else:
+                    raise Invalid(
+                        "Invalid value on ACTION column",
+                        "Invalid SDMX-CSV file. "
+                        "Check the docs for the proper values "
+                        "on ACTION column.",
+                    )
+
+                del data["ACTION"]  # Remove ACTION column from DataFrame
+            else:
+                raise Invalid(
+                    "Invalid value on ACTION column",
+                    "Invalid SDMX-CSV file. "
+                    "Cannot have more than one value on ACTION column, "
+                    "or 2 if D is present",
+                )
+        # Remove columns that are not needed
+        if "STRUCTURE_NAME" in data.columns:
+            data = data.drop(columns=["STRUCTURE_NAME"])
+        if "SERIES_KEYS" in data.columns:
+            data = data.drop(columns=["SERIES_KEYS"])
+        if "OBS_KEYS" in data.columns:
+            data = data.drop(columns=["OBS_KEYS"])
+
+        # For SDMX-CSV version 2, use 'STRUCTURE_ID'
+        # column as the structure id and 'STRUCTURE' as the structure type
+        structure_id = data["STRUCTURE_ID"].iloc[0]
+        structure_type = data["STRUCTURE"].iloc[0]
+        # Drop 'STRUCTURE' and 'STRUCTURE_ID' columns from DataFrame
+        df_csv = data.drop(["STRUCTURE", "STRUCTURE_ID"], axis=1)
+        if structure_id.count(":") == 2:
+            structure_id = ":".join(structure_id.split(":")[:2])
+        if structure_type == "DataStructure".lower():
+            urn = f"DataStructure={structure_id}"
+        elif structure_type == "Dataflow".lower():
+            urn = f"Dataflow={structure_id}"
+        elif structure_type == "dataprovision":
+            urn = f"ProvisionAgreement={structure_id}"
+        else:
+            raise Invalid(
+                "Invalid value on STRUCTURE column",
+                "Invalid SDMX-CSV file. "
+                "Check the docs for the proper values on STRUCTURE column.",
+            )
+    else:
+        # For SDMX-CSV version 1, use 'DATAFLOW' column as the structure id
+        structure_id = data["DATAFLOW"].iloc[0]
+        if structure_id.count(":") == 2:
+            structure_id = ":".join(structure_id.split(":")[:2])
+        # Drop 'DATAFLOW' column from DataFrame
+        df_csv = data.drop(["DATAFLOW"], axis=1)
+
+        urn = f"Dataflow={structure_id}"
+    return PandasDataset(
+        structure=urn,
+        data=df_csv,
+        action=action if action is not None else ActionType.Information,
+    )

pysdmx/io/csv/__csv_aux_writer.py

@@ -0,0 +1,118 @@
+from copy import copy
+from typing import List, Literal, Optional, Sequence
+
+import pandas as pd
+
+from pysdmx.io.pd import PandasDataset
+from pysdmx.model import Schema
+from pysdmx.model.dataset import ActionType
+from pysdmx.toolkit.pd._data_utils import format_labels, get_codes
+
+SDMX_CSV_ACTION_MAPPER = {
+    ActionType.Append: "A",
+    ActionType.Replace: "R",
+    ActionType.Information: "I",
+    ActionType.Delete: "D",
+}
+
+
+def __write_time_period(df: pd.DataFrame, time_format: str) -> None:
+    # TODO: Correct handle of normalized time format
+    raise NotImplementedError("Normalized time format is not implemented yet.")
+
+
+def __write_keys(
+    df: pd.DataFrame, keys: Literal["obs", "series", "both"], schema: Schema
+) -> None:
+    """Writes the keys to the DataFrame.
+
+    Args:
+        df: The DataFrame to write the keys to.
+        keys: to write or not the keys columns
+            If None, no keys are written.
+            If "obs", the keys are write as a single
+            column called "OBS_KEY".
+            If "series", the keys are write as a single
+            column called "SERIES_KEY".
+            If "both", the keys are write as two columns:
+            "OBS_KEY" and "SERIES_KEY".
+        schema: The schema to get the keys
+    """
+    series_codes, obs_codes, group_codes = get_codes(
+        dimension_code="", structure=schema, data=df
+    )
+    del obs_codes[0]
+    obs_parts = []
+    series_parts = []
+    for k, v in df.items():
+        value = v.iloc[0]
+        if k in obs_codes:
+            obs_parts.append(str(value))
+        if k in series_codes:
+            series_parts.append(str(value))
+    obs_values = ".".join(obs_parts)
+    series_values = ".".join(series_parts)
+    if keys == "obs":
+        df.insert(0, "OBS_KEYS", obs_values)
+    elif keys == "series":
+        df.insert(0, "SERIES_KEYS", series_values)
+    else:
+        df.insert(0, "OBS_KEYS", obs_values)
+        df.insert(0, "SERIES_KEYS", series_values)
+
+
+def _write_csv_2_aux(
+    datasets: Sequence[PandasDataset],
+    labels: Optional[Literal["name", "id", "both"]] = None,
+    time_format: Optional[Literal["original", "normalized"]] = None,
+    keys: Optional[Literal["obs", "series", "both"]] = None,
+    references_21: bool = False,
+) -> List[pd.DataFrame]:
+    dataframes = []
+    for dataset in datasets:
+        # Create a copy of the dataset
+        df: pd.DataFrame = copy(dataset.data)
+        structure_ref, unique_id = dataset.short_urn.split("=", maxsplit=1)
+
+        # Add additional attributes to the dataset
+        for k, v in dataset.attributes.items():
+            df[k] = v
+
+        if structure_ref in ["DataStructure", "Dataflow"]:
+            structure_ref = structure_ref.lower()
+        else:
+            structure_ref = "dataprovision"
+
+        if references_21 and dataset.action in [
+            ActionType.Information,
+            ActionType.Append,
+        ]:
+            action_value = "M"
+        else:
+            action_value = SDMX_CSV_ACTION_MAPPER[dataset.action]
+
+        if time_format is not None and time_format != "original":
+            __write_time_period(df, time_format)
+        if keys is not None and isinstance(dataset.structure, Schema):
+            __write_keys(df, keys, dataset.structure)
+        if labels is not None and isinstance(dataset.structure, Schema):
+            format_labels(df, labels, dataset.structure.components)
+            df.insert(0, "STRUCTURE", structure_ref)
+            df.insert(
+                1,
+                "STRUCTURE_ID",
+                f"{unique_id}:{dataset.structure.name}"
+                if labels == "both"
+                else unique_id,
+            )
+            action_position = 2
+            if labels == "name":
+                action_position += 1
+                df.insert(2, "STRUCTURE_NAME", dataset.structure.name)
+            df.insert(action_position, "ACTION", action_value)
+        else:
+            df.insert(0, "STRUCTURE", structure_ref)
+            df.insert(1, "STRUCTURE_ID", unique_id)
+            df.insert(2, "ACTION", action_value)
+        dataframes.append(df)
+    return dataframes

pysdmx/io/csv/sdmx10/reader/__init__.py

@@ -6,23 +6,10 @@ from typing import Sequence
 import pandas as pd
 
 from pysdmx.errors import Invalid
+from pysdmx.io.csv.__csv_aux_reader import __generate_dataset_from_sdmx_csv
 from pysdmx.io.pd import PandasDataset
 
 
-def __generate_dataset_from_sdmx_csv(data: pd.DataFrame) -> PandasDataset:
-    # For SDMX-CSV version 1, use 'DATAFLOW' column as the structure id
-    structure_id = data["DATAFLOW"].iloc[0]
-    # Drop 'DATAFLOW' column from DataFrame
-    df_csv = data.drop(["DATAFLOW"], axis=1)
-    urn = f"Dataflow={structure_id}"
-
-    # Return a Dataset object with the extracted information
-    return PandasDataset(
-        structure=urn,
-        data=df_csv,
-    )
-
-
 def read(input_str: str) -> Sequence[PandasDataset]:
     """Reads csv data and returns a sequence of Datasets.
 
@@ -56,6 +43,14 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     # Check if any column headers contain ':', indicating mode, label or text
     mode_label_text = any(":" in x for x in df_csv.columns)
 
+    # if values in the columns contain ':', split them
+    for col in df_csv.columns[1:]:
+        df_csv[col] = (
+            df_csv[col]
+            .astype(str)
+            .apply(lambda x: x.split(":")[0] if ":" in x else x)
+        )
+
     # Determine the id column based on the SDMX-CSV version
     id_column = "DATAFLOW"
 

pysdmx/io/csv/sdmx10/writer/__init__.py

@@ -2,15 +2,20 @@
 
 from copy import copy
 from pathlib import Path
-from typing import Optional, Sequence, Union
+from typing import Literal, Optional, Sequence, Union
 
 import pandas as pd
 
+from pysdmx.io.csv.__csv_aux_writer import __write_time_period
 from pysdmx.io.pd import PandasDataset
+from pysdmx.model import Schema
+from pysdmx.toolkit.pd._data_utils import format_labels
 
 
 def write(
     datasets: Sequence[PandasDataset],
+    labels: Optional[Literal["id", "both"]] = None,
+    time_format: Optional[Literal["original", "normalized"]] = None,
     output_path: Optional[Union[str, Path]] = None,
 ) -> Optional[str]:
     """Write data to SDMX-CSV 1.0 format.
@@ -20,6 +25,15 @@ def write(
           Must have the same components.
         output_path: Path to write the data to.
           If None, the data is returned as a string.
+        labels: How to write the name of the columns.
+            If None, only the IDs are written.
+            if "id", the names are written as ID only.
+            If "both", the names are witten as id:Name.
+        time_format: How to write the time period.
+            If None, the time period is not modified.
+            If "original", the time period is written as it
+            is in the dataset.
+            "Normalized" is not implemented yet.
 
     Returns:
         SDMX CSV data as a string, if output_path is None.
@@ -31,11 +45,23 @@ def write(
     dataframes = []
     for dataset in datasets:
         df: pd.DataFrame = copy(dataset.data)
-        df.insert(0, "DATAFLOW", dataset.short_urn.split("=")[1])
 
         # Add additional attributes to the dataset
         for k, v in dataset.attributes.items():
             df[k] = v
+        structure_id = dataset.short_urn.split("=")[1]
+        if time_format is not None and time_format != "original":
+            __write_time_period(df, time_format)
+        if labels is not None and isinstance(dataset.structure, Schema):
+            format_labels(df, labels, dataset.structure.components)
+            if labels == "id":
+                df.insert(0, "DATAFLOW", structure_id)
+            else:
+                df.insert(
+                    0, "DATAFLOW", f"{structure_id}:{dataset.structure.name}"
+                )
+        else:
+            df.insert(0, "DATAFLOW", structure_id)
 
         dataframes.append(df)
 

pysdmx/io/csv/sdmx20/__init__.py

@@ -1,10 +1 @@
 """SDMX 2.0 CSV reader and writer."""
-
-from pysdmx.model.dataset import ActionType
-
-SDMX_CSV_ACTION_MAPPER = {
-    ActionType.Append: "A",
-    ActionType.Replace: "R",
-    ActionType.Information: "I",
-    ActionType.Delete: "D",
-}

pysdmx/io/csv/sdmx20/reader/__init__.py

@@ -6,68 +6,8 @@ from typing import Sequence
 import pandas as pd
 
 from pysdmx.errors import Invalid
+from pysdmx.io.csv.__csv_aux_reader import __generate_dataset_from_sdmx_csv
 from pysdmx.io.pd import PandasDataset
-from pysdmx.model.dataset import ActionType
-
-ACTION_SDMX_CSV_MAPPER_READING = {
-    "A": ActionType.Append,
-    "D": ActionType.Delete,
-    "R": ActionType.Replace,
-    "I": ActionType.Information,
-}
-
-
-def __generate_dataset_from_sdmx_csv(data: pd.DataFrame) -> PandasDataset:
-    # Extract Structure type and structure id
-    action = ActionType.Information
-    if "ACTION" in data.columns:
-        unique_values = list(data["ACTION"].unique())
-        if len(unique_values) > 1 and "D" in unique_values:
-            unique_values.remove("D")
-            data = data[data["ACTION"] != "D"]
-        if len(unique_values) == 1:  # If there is only one value, use it
-            action_value = unique_values[0]
-            if action_value not in ACTION_SDMX_CSV_MAPPER_READING:
-                raise Invalid(
-                    "Invalid value on ACTION column",
-                    "Invalid SDMX-CSV 2.0 file. "
-                    "Check the docs for the proper values on ACTION column.",
-                )
-            action = ACTION_SDMX_CSV_MAPPER_READING[action_value]
-            del data["ACTION"]  # Remove ACTION column from DataFrame
-        else:
-            raise Invalid(
-                "Invalid value on ACTION column",
-                "Invalid SDMX-CSV 2.0 file. "
-                "Cannot have more than one value on ACTION column, "
-                "or 2 if D is present",
-            )
-    # For SDMX-CSV version 2, use 'STRUCTURE_ID'
-    # column as the structure id and 'STRUCTURE' as the structure type
-    structure_id = data["STRUCTURE_ID"].iloc[0]
-    structure_type = data["STRUCTURE"].iloc[0]
-    # Drop 'STRUCTURE' and 'STRUCTURE_ID' columns from DataFrame
-    df_csv = data.drop(["STRUCTURE", "STRUCTURE_ID"], axis=1)
-
-    if structure_type == "DataStructure".lower():
-        urn = f"DataStructure={structure_id}"
-    elif structure_type == "Dataflow".lower():
-        urn = f"Dataflow={structure_id}"
-    elif structure_type == "dataprovision":
-        urn = f"ProvisionAgreement={structure_id}"
-    else:
-        raise Invalid(
-            "Invalid value on STRUCTURE column",
-            "Invalid SDMX-CSV 2.0 file. "
-            "Check the docs for the proper values on STRUCTURE column.",
-        )
-
-    # Return a Dataset object with the extracted information
-    return PandasDataset(
-        structure=urn,
-        data=df_csv,
-        action=action,
-    )
 
 
 def read(input_str: str) -> Sequence[PandasDataset]:
@@ -105,6 +45,13 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     df_csv = df_csv.astype(str).replace({"nan": "", "<NA>": ""})
     # Check if any column headers contain ':', indicating mode, label or text
     mode_label_text = any(":" in x for x in df_csv.columns)
+    # if values in the columns contain ':', split them
+    for col in df_csv.columns[2:]:
+        df_csv[col] = (
+            df_csv[col]
+            .astype(str)
+            .apply(lambda x: x.split(":")[0] if ":" in x else x)
+        )
 
     id_column = "STRUCTURE_ID"
     # If mode, label or text is present, modify the DataFrame