pysdmx 1.10.0rc1__py3-none-any.whl → 1.10.0rc2__py3-none-any.whl

pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.10.0rc1"
+__version__ = "1.10.0rc2"

pysdmx/api/fmr/__init__.py

@@ -44,6 +44,7 @@ from pysdmx.model import (
     Metadataflow,
     MetadataProvisionAgreement,
     MetadataReport,
+    MetadataStructure,
     MultiRepresentationMap,
     ProvisionAgreement,
     RepresentationMap,
@@ -767,7 +768,7 @@ class RegistryClient(__BaseRegistryClient):
         agency: str = "*",
         id: str = "*",
         version: str = "+",
-    ) -> Sequence[Dataflow]:
+    ) -> Sequence[MetadataStructure]:
         """Get the metadata structures (MSD) matching the supplied parameters.
 
         Args:
@@ -1341,7 +1342,7 @@ class AsyncRegistryClient(__BaseRegistryClient):
         agency: str = "*",
         id: str = "*",
         version: str = "+",
-    ) -> Sequence[Dataflow]:
+    ) -> Sequence[MetadataStructure]:
         """Get the metadata structures (MSD) matching the supplied parameters.
 
         Args:

pysdmx/io/_pd_utils.py

@@ -0,0 +1,83 @@
+import pandas as pd
+
+from pysdmx.errors import Invalid
+from pysdmx.model.concept import DataType
+from pysdmx.model.dataflow import Schema
+
+NUMERIC_TYPES = {
+    DataType.BIG_INTEGER,
+    DataType.COUNT,
+    DataType.DECIMAL,
+    DataType.DOUBLE,
+    DataType.FLOAT,
+    DataType.INCREMENTAL,
+    DataType.INTEGER,
+    DataType.LONG,
+    DataType.SHORT,
+}
+
+
+def _fill_na_values(data: pd.DataFrame, structure: Schema) -> pd.DataFrame:
+    """Fills missing values in the DataFrame based on the component type.
+
+    Numeric components are filled with "NaN".
+    Other components are filled with "#N/A".
+    If the structure does not have components,
+    all missing values are filled with "".
+
+    Args:
+        data: The DataFrame to fill.
+        structure: The structure definition (´Schema´).
+
+    Returns:
+        The DataFrame with filled missing values.
+
+    Raises:
+        Invalid: If the structure does not have components.
+    """
+    for component in structure.components:
+        if component.id in data.columns:
+            if component.dtype in NUMERIC_TYPES:
+                data[component.id] = (
+                    data[component.id].astype(object).fillna("NaN")
+                )
+            else:
+                data[component.id] = (
+                    data[component.id].astype(object).fillna("#N/A")
+                )
+
+    return data
+
+
+def _validate_explicit_null_values(
+    data: pd.DataFrame, structure: Schema
+) -> None:
+    """Validates that explicit null values are correct for the component type.
+
+    Numeric components must not contain "#N/A".
+    Non-numeric components must not contain "NaN".
+
+    Args:
+        data: The DataFrame to validate.
+        structure: The structure definition (´Schema´).
+
+    Raises:
+        Invalid: If invalid null values are found.
+    """
+    for component in structure.components:
+        if component.id in data.columns:
+            series = data[component.id].astype(str)
+            if component.dtype in NUMERIC_TYPES:
+                # Numeric: #N/A is invalid
+                if series.isin(["#N/A"]).any():
+                    raise Invalid(
+                        f"Invalid null value '#N/A' in numeric component "
+                        f"'{component.id}'."
+                    )
+            else:
+                # Non-numeric: NaN is invalid
+                if series.isin(["NaN"]).any():
+                    raise Invalid(
+                        f"Invalid null value 'NaN' in non-numeric component "
+                        f"'{component.id}'."
+                    )

pysdmx/io/csv/__csv_aux_writer.py

@@ -3,6 +3,8 @@ from typing import List, Literal, Optional, Sequence
 
 import pandas as pd
 
+from pysdmx.errors import Invalid
+from pysdmx.io._pd_utils import _fill_na_values
 from pysdmx.io.pd import PandasDataset
 from pysdmx.model import Schema
 from pysdmx.model.dataset import ActionType
@@ -16,6 +18,25 @@ SDMX_CSV_ACTION_MAPPER = {
 }
 
 
+def _validate_schema_exists(dataset: PandasDataset) -> Schema:
+    """Validates that the dataset has a Schema defined.
+
+    Args:
+        dataset: The dataset to validate.
+
+    Returns:
+        The `Schema` from the dataset.
+
+    Raises:
+        Invalid: If the structure is not a `Schema`.
+    """
+    if not isinstance(dataset.structure, Schema):
+        raise Invalid(
+            "Dataset Structure is not a Schema. Cannot perform operation."
+        )
+    return dataset.structure
+
+
 def __write_time_period(df: pd.DataFrame, time_format: str) -> None:
     # TODO: Correct handle of normalized time format
     raise NotImplementedError("Normalized time format is not implemented yet.")
@@ -70,8 +91,10 @@ def _write_csv_2_aux(
 ) -> List[pd.DataFrame]:
     dataframes = []
     for dataset in datasets:
+        schema = _validate_schema_exists(dataset)
         # Create a copy of the dataset
         df: pd.DataFrame = copy(dataset.data)
+        df = _fill_na_values(df, schema)
         structure_ref, unique_id = dataset.short_urn.split("=", maxsplit=1)
 
         # Add additional attributes to the dataset

pysdmx/io/csv/sdmx10/reader/__init__.py

@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     """
     # Get Dataframe from CSV file
     df_csv = pd.read_csv(
-        StringIO(input_str), keep_default_na=False, na_values=[""]
+        StringIO(input_str), keep_default_na=False, na_values=[]
     )
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")

pysdmx/io/csv/sdmx10/writer/__init__.py

@@ -6,9 +6,12 @@ from typing import Literal, Optional, Sequence, Union
 
 import pandas as pd
 
-from pysdmx.io.csv.__csv_aux_writer import __write_time_period
+from pysdmx.io._pd_utils import _fill_na_values
+from pysdmx.io.csv.__csv_aux_writer import (
+    __write_time_period,
+    _validate_schema_exists,
+)
 from pysdmx.io.pd import PandasDataset
-from pysdmx.model import Schema
 from pysdmx.toolkit.pd._data_utils import format_labels
 
 
@@ -44,22 +47,26 @@ def write(
     # Create a copy of the dataset
     dataframes = []
     for dataset in datasets:
+        # Validate that dataset has a proper Schema
+        schema = _validate_schema_exists(dataset)
+
         df: pd.DataFrame = copy(dataset.data)
 
+        # Fill missing values
+        df = _fill_na_values(df, schema)
+
         # Add additional attributes to the dataset
         for k, v in dataset.attributes.items():
             df[k] = v
         structure_id = dataset.short_urn.split("=")[1]
         if time_format is not None and time_format != "original":
             __write_time_period(df, time_format)
-        if labels is not None and isinstance(dataset.structure, Schema):
-            format_labels(df, labels, dataset.structure.components)
+        if labels is not None:
+            format_labels(df, labels, schema.components)
             if labels == "id":
                 df.insert(0, "DATAFLOW", structure_id)
             else:
-                df.insert(
-                    0, "DATAFLOW", f"{structure_id}:{dataset.structure.name}"
-                )
+                df.insert(0, "DATAFLOW", f"{structure_id}:{schema.name}")
         else:
             df.insert(0, "DATAFLOW", structure_id)
 
@@ -68,8 +75,7 @@ def write(
     # Concatenate the dataframes
     all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
-    # Ensure null values are represented as empty strings
-    all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
+    all_data = all_data.astype(str)
     # If the output path is an empty string we use None
     output_path = (
         None

pysdmx/io/csv/sdmx20/reader/__init__.py

@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     """
     # Get Dataframe from CSV file
     df_csv = pd.read_csv(
-        StringIO(input_str), keep_default_na=False, na_values=[""]
+        StringIO(input_str), keep_default_na=False, na_values=[]
     )
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")

pysdmx/io/csv/sdmx20/writer/__init__.py

@@ -60,7 +60,7 @@ def write(
 
     all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
-    all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
+    all_data = all_data.astype(str)
 
     # If the output path is an empty string we use None
     output_path = (

pysdmx/io/csv/sdmx21/reader/__init__.py

@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     """
     # Get Dataframe from CSV file
     df_csv = pd.read_csv(
-        StringIO(input_str), keep_default_na=False, na_values=[""]
+        StringIO(input_str), keep_default_na=False, na_values=[]
     )
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")

pysdmx/io/csv/sdmx21/writer/__init__.py

@@ -57,7 +57,7 @@ def write(
 
     all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
-    all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
+    all_data = all_data.astype(str)
 
     # If the output path is an empty string we use None
     output_path = (

pysdmx/io/json/sdmxjson2/messages/__init__.py

@@ -12,6 +12,9 @@ from pysdmx.io.json.sdmxjson2.messages.code import (
     JsonHierarchyMessage,
 )
 from pysdmx.io.json.sdmxjson2.messages.concept import JsonConceptSchemeMessage
+from pysdmx.io.json.sdmxjson2.messages.constraint import (
+    JsonDataConstraintMessage,
+)
 from pysdmx.io.json.sdmxjson2.messages.dataflow import (
     JsonDataflowMessage,
     JsonDataflowsMessage,
@@ -50,6 +53,7 @@ __all__ = [
     "JsonCategorySchemeMessage",
     "JsonCodelistMessage",
     "JsonConceptSchemeMessage",
+    "JsonDataConstraintMessage",
     "JsonDataflowMessage",
     "JsonDataflowsMessage",
     "JsonDataStructuresMessage",

pysdmx/io/json/sdmxjson2/messages/code.py

@@ -231,7 +231,7 @@ class JsonCodelists(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for lists of codes."""
 
     codelists: Sequence[JsonCodelist] = ()
-    valuelists: Sequence[JsonValuelist] = ()
+    valueLists: Sequence[JsonValuelist] = ()
 
 
 class JsonCodelistMessage(Struct, frozen=True, omit_defaults=True):
@@ -244,7 +244,7 @@ class JsonCodelistMessage(Struct, frozen=True, omit_defaults=True):
         if self.data.codelists:
             return self.data.codelists[0].to_model()
         else:
-            return self.data.valuelists[0].to_model()
+            return self.data.valueLists[0].to_model()
 
 
 class JsonHierarchicalCode(Struct, frozen=True, omit_defaults=True):
@@ -329,10 +329,10 @@ class JsonHierarchicalCode(Struct, frozen=True, omit_defaults=True):
             code=code.urn,
             validFrom=code.rel_valid_from,
             validTo=code.rel_valid_to,
-            annotations=tuple(annotations),
-            hierarchicalCodes=[
-                JsonHierarchicalCode.from_model(c) for c in code.codes
-            ],
+            annotations=tuple(annotations) if annotations else None,
+            hierarchicalCodes=tuple(
+                [JsonHierarchicalCode.from_model(c) for c in code.codes]
+            ),
         )
 
 
@@ -475,6 +475,15 @@ class JsonHierarchyAssociation(
                 "SDMX-JSON hierarchy associations must reference a context",
                 {"hierarchy_association": ha.id},
             )
+        lnk = (
+            JsonLink(
+                rel="UserDefinedOperator",
+                type="sdmx_artefact",
+                urn=ha.operator,
+            )
+            if ha.operator
+            else None
+        )
         return JsonHierarchyAssociation(
             agency=(
                 ha.agency.id if isinstance(ha.agency, Agency) else ha.agency
@@ -492,6 +501,7 @@ class JsonHierarchyAssociation(
             linkedHierarchy=href,
             linkedObject=ha.component_ref,
             contextObject=ha.context_ref,
+            links=[lnk] if lnk else (),
         )
 
 

pysdmx/io/json/sdmxjson2/messages/constraint.py

@@ -1,16 +1,43 @@
 """Collection of SDMX-JSON schemas for content constraints."""
 
-from typing import Dict, Literal, Optional, Sequence
+from datetime import datetime
+from typing import Optional, Sequence
 
 from msgspec import Struct
 
-from pysdmx.io.json.sdmxjson2.messages.core import MaintainableType
+from pysdmx import errors
+from pysdmx.io.json.sdmxjson2.messages.core import (
+    JsonAnnotation,
+    MaintainableType,
+)
+from pysdmx.model import (
+    Agency,
+    ConstraintAttachment,
+    CubeKeyValue,
+    CubeRegion,
+    CubeValue,
+    DataConstraint,
+    DataKey,
+    DataKeyValue,
+    KeySet,
+)
 
 
 class JsonValue(Struct, frozen=True, omit_defaults=True):
-    """SDMX-JSON payload for an allowed value."""
+    """SDMX-JSON payload for a cube value."""
 
     value: str
+    validFrom: Optional[datetime] = None
+    validTo: Optional[datetime] = None
+
+    def to_model(self) -> CubeValue:
+        """Converts a JsonValue to a CubeValue."""
+        return CubeValue(self.value, self.validFrom, self.validTo)
+
+    @classmethod
+    def from_model(self, cv: CubeValue) -> "JsonValue":
+        """Converts a pysdmx cube value to an SDMX-JSON one."""
+        return JsonValue(cv.value, cv.valid_from, cv.valid_to)
 
 
 class JsonKeyValue(Struct, frozen=True, omit_defaults=True):
@@ -18,36 +45,228 @@ class JsonKeyValue(Struct, frozen=True, omit_defaults=True):
 
     id: str
     values: Sequence[JsonValue]
+    # Additional properties are supported in the model (include,
+    # removePrefix, validFrom, validTo, timeRange) but not by the FMR.
+    # Therefore, they are ignored for now.
+
+    def to_model(self) -> CubeKeyValue:
+        """Converts a JsonKeyValue to a CubeKeyValue."""
+        return CubeKeyValue(self.id, [v.to_model() for v in self.values])
 
-    def to_model(self) -> Sequence[str]:
-        """Returns the requested list of values."""
-        return [v.value for v in self.values]
+    @classmethod
+    def from_model(self, key_value: CubeKeyValue) -> "JsonKeyValue":
+        """Converts a pysdmx cube key value to an SDMX-JSON one."""
+        return JsonKeyValue(
+            key_value.id, [JsonValue.from_model(v) for v in key_value.values]
+        )
 
 
 class JsonCubeRegion(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for a cube region."""
 
+    # The property `components` is ignored as it's not used in the FMR`
     keyValues: Sequence[JsonKeyValue]
+    include: bool = True
 
-    def to_map(self) -> Dict[str, Sequence[str]]:
-        """Gets the list of allowed values for a component."""
-        return {kv.id: kv.to_model() for kv in self.keyValues}
+    def to_model(self) -> CubeRegion:
+        """Converts a JsonCubeRegion to a CubeRegion."""
+        return CubeRegion(
+            [kv.to_model() for kv in self.keyValues], self.include
+        )
+
+    @classmethod
+    def from_model(self, region: CubeRegion) -> "JsonCubeRegion":
+        """Converts a pysdmx cube region to an SDMX-JSON one."""
+        return JsonCubeRegion(
+            [JsonKeyValue.from_model(kv) for kv in region.key_values],
+            region.is_included,
+        )
 
 
 class JsonConstraintAttachment(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for a constraint attachment."""
 
-    dataProvider: Optional[str]
-    simpleDataSources: Optional[Sequence[str]] = None
-    dataStructures: Optional[Sequence[str]] = None
-    dataflows: Optional[Sequence[str]] = None
-    provisionAgreements: Optional[Sequence[str]] = None
-    queryableDataSources: Optional[Sequence[str]] = None
+    dataProvider: Optional[str] = None
+    dataStructures: Sequence[str] = ()
+    dataflows: Sequence[str] = ()
+    provisionAgreements: Sequence[str] = ()
+
+    def to_model(self) -> ConstraintAttachment:
+        """Converts a JsonConstraintAttachment to a ConstraintAttachment."""
+        return ConstraintAttachment(
+            self.dataProvider,
+            self.dataStructures,
+            self.dataflows,
+            self.provisionAgreements,
+        )
+
+    @classmethod
+    def from_model(
+        self, attachment: ConstraintAttachment
+    ) -> "JsonConstraintAttachment":
+        """Converts a pysdmx constraint attachment to an SDMX-JSON one."""
+        ds = attachment.data_structures if attachment.data_structures else ()
+        df = attachment.dataflows if attachment.dataflows else ()
+        pa = (
+            attachment.provision_agreements
+            if attachment.provision_agreements
+            else ()
+        )
+        return JsonConstraintAttachment(attachment.data_provider, ds, df, pa)
+
+
+class JsonDataKeyValue(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for a data key value."""
+
+    id: str
+    value: str
+
+    def to_model(self) -> DataKeyValue:
+        """Converts a JsonDataKeyValue to a DataKeyValue."""
+        return DataKeyValue(self.id, self.value)
+
+    @classmethod
+    def from_model(self, kv: DataKeyValue) -> "JsonDataKeyValue":
+        """Converts a pysdmx key value to an SDMX-JSON one."""
+        return JsonDataKeyValue(kv.id, kv.value)
+
+
+class JsonDataKey(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for a data key."""
+
+    keyValues: Sequence[JsonDataKeyValue]
+    validFrom: Optional[datetime] = None
+    validTo: Optional[datetime] = None
+
+    def to_model(self) -> DataKey:
+        """Converts a JsonDataKey to a DataKey."""
+        return DataKey(
+            [kv.to_model() for kv in self.keyValues],
+            self.validFrom,
+            self.validTo,
+        )
+
+    @classmethod
+    def from_model(self, kv: DataKey) -> "JsonDataKey":
+        """Converts a pysdmx key constraint to an SDMX-JSON one."""
+        return JsonDataKey(
+            [JsonDataKeyValue.from_model(val) for val in kv.keys_values],
+            kv.valid_from,
+            kv.valid_to,
+        )
+
+
+class JsonKeySet(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for a keyset."""
+
+    keys: Sequence[JsonDataKey]
+    isIncluded: bool
+
+    def to_model(self) -> KeySet:
+        """Converts a JsonKeySet to a KeySet."""
+        return KeySet([k.to_model() for k in self.keys], self.isIncluded)
+
+    @classmethod
+    def from_model(self, ks: KeySet) -> "JsonKeySet":
+        """Converts a pysdmx key set constraint to an SDMX-JSON one."""
+        return JsonKeySet(
+            [JsonDataKey.from_model(k) for k in ks.keys], ks.is_included
+        )
 
 
 class JsonDataConstraint(MaintainableType, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for a content constraint."""
 
-    role: Optional[Literal["Allowed", "Actual"]] = None
     constraintAttachment: Optional[JsonConstraintAttachment] = None
     cubeRegions: Optional[Sequence[JsonCubeRegion]] = None
+    dataKeySets: Optional[Sequence[JsonKeySet]] = None
+
+    def to_model(self) -> DataConstraint:
+        """Converts a JsonDataConstraint to a pysdmx Data Constraint."""
+        at = self.constraintAttachment.to_model()  # type: ignore[union-attr]
+        return DataConstraint(
+            id=self.id,
+            name=self.name,
+            agency=self.agency,
+            description=self.description,
+            version=self.version,
+            annotations=tuple([a.to_model() for a in self.annotations]),
+            is_external_reference=self.isExternalReference,
+            valid_from=self.validFrom,
+            valid_to=self.validTo,
+            constraint_attachment=at,
+            cube_regions=[r.to_model() for r in self.cubeRegions]
+            if self.cubeRegions
+            else (),
+            key_sets=[s.to_model() for s in self.dataKeySets]
+            if self.dataKeySets
+            else (),
+        )
+
+    @classmethod
+    def from_model(self, cons: DataConstraint) -> "JsonDataConstraint":
+        """Converts a pysdmx constraint to an SDMX-JSON one."""
+        crs = (
+            [JsonCubeRegion.from_model(r) for r in cons.cube_regions]
+            if cons.cube_regions
+            else None
+        )
+        dks = (
+            [JsonKeySet.from_model(s) for s in cons.key_sets]
+            if cons.key_sets
+            else None
+        )
+        if not cons.name:
+            raise errors.Invalid(
+                "Invalid input",
+                "SDMX-JSON data constraints must have a name",
+                {"data_constraint": cons.id},
+            )
+        if not cons.constraint_attachment:
+            raise errors.Invalid(
+                "Invalid input",
+                "SDMX-JSON data constraints must have a constraint attachment",
+                {"data_constraint": cons.id},
+            )
+        return JsonDataConstraint(
+            id=cons.id,
+            name=cons.name,
+            agency=(
+                cons.agency.id
+                if isinstance(cons.agency, Agency)
+                else cons.agency
+            ),
+            description=cons.description,
+            version=cons.version,
+            annotations=tuple(
+                [JsonAnnotation.from_model(a) for a in cons.annotations]
+            ),
+            isExternalReference=cons.is_external_reference,
+            validFrom=cons.valid_from,
+            validTo=cons.valid_to,
+            constraintAttachment=JsonConstraintAttachment.from_model(
+                cons.constraint_attachment
+            ),
+            cubeRegions=crs,
+            dataKeySets=dks,
+        )
+
+
+class JsonDataConstraints(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for data constraints."""
+
+    dataConstraints: Sequence[JsonDataConstraint] = ()
+
+    def to_model(self) -> Sequence[DataConstraint]:
+        """Returns the requested data constraints."""
+        return [cc.to_model() for cc in self.dataConstraints]
+
+
+class JsonDataConstraintMessage(Struct, frozen=True, omit_defaults=True):
+    """SDMX-JSON payload for /dataconstraint queries."""
+
+    data: JsonDataConstraints
+
+    def to_model(self) -> Sequence[DataConstraint]:
+        """Returns the requested data constraints."""
+        return self.data.to_model()