pysdmx 1.10.0rc2__tar.gz → 1.11.0__tar.gz

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pysdmx
-Version: 1.10.0rc2
+Version: 1.11.0
 Summary: Your opinionated Python SDMX library
 License: Apache-2.0
 License-File: LICENSE
@@ -24,6 +24,12 @@ Requires-Dist: jsonschema (>=4.10) ; extra == "json"
 Requires-Dist: lxml (>=5.2) ; extra == "all"
 Requires-Dist: lxml (>=5.2) ; extra == "xml"
 Requires-Dist: msgspec (>=0)
+Requires-Dist: numpy (>=2.0.2,<2.1) ; (python_version < "3.13") and (extra == "all")
+Requires-Dist: numpy (>=2.0.2,<2.1) ; (python_version < "3.13") and (extra == "data")
+Requires-Dist: numpy (>=2.0.2,<2.1) ; (python_version < "3.13") and (extra == "vtl")
+Requires-Dist: numpy (>=2.1.0,<2.3) ; (python_version >= "3.13") and (extra == "all")
+Requires-Dist: numpy (>=2.1.0,<2.3) ; (python_version >= "3.13") and (extra == "data")
+Requires-Dist: numpy (>=2.1.0,<2.3) ; (python_version >= "3.13") and (extra == "vtl")
 Requires-Dist: pandas (>=2.1.4) ; extra == "all"
 Requires-Dist: pandas (>=2.1.4) ; extra == "data"
 Requires-Dist: parsy (>=2.1)

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pysdmx"
-version = "1.10.0rc2"
+version = "1.11.0"
 description = "Your opinionated Python SDMX library"
 license = { text = "Apache-2.0" }
 readme = "README.rst"
@@ -10,7 +10,9 @@ authors = [
     { name = "Stratos Nikoloutsos", email = "<stratos.nikoloutsos@bis.org>" },
     { name = "Francisco Javier Hernandez del Caño", email = "<javier.hernandez@meaningfuldata.eu>" },
     { name = "Mateo de Lorenzo Argelés", email = "<mateo.delorenzo@meaningfuldata.eu>" },
-    { name = "Jose Antonio Franco Martin", email = "<jose.franco@meaningfuldata.eu>" }
+    { name = "Jose Antonio Franco Martin", email = "<jose.franco@meaningfuldata.eu>" },
+    { name = "Mario Cortés Ruiz", email = "<mario.cortes@meaningfuldata.eu>" }
+
 ]
 keywords = ["sdmx", "data discovery", "data retrieval", "metadata", "fmr"]
 classifiers = [
@@ -33,12 +35,13 @@ documentation = "https://bis-med-it.github.io/pysdmx"
 "Bug Tracker" = "https://bis-med-it.github.io/pysdmx/issues"
 
 [project.optional-dependencies]
-data = ["pandas>=2.1.4"]
+data = ["pandas>=2.1.4", "numpy>=2.0.2,<2.1; python_version < '3.13'", "numpy>=2.1.0,<2.3; python_version >= '3.13'"]
 dc = ["python-dateutil>=2.8.2"]
-vtl = ["vtlengine>=1.2,<2.0"]
+vtl = ["vtlengine>=1.2,<2.0", "numpy>=2.0.2,<2.1; python_version < '3.13'", "numpy>=2.1.0,<2.3; python_version >= '3.13'"]
 json = ["sdmxschemas>=1.0.0", "jsonschema>=4.10"]
 xml = ["lxml>=5.2", "xmltodict>=0.13", "sdmxschemas>=1.0.0"]
-all = ["lxml>=5.2", "xmltodict>=0.13", "sdmxschemas>=1.0.0", "pandas>=2.1.4", "python-dateutil>=2.8.2", "vtlengine>=1.2,<2.0"]
+all = ["lxml>=5.2", "xmltodict>=0.13", "sdmxschemas>=1.0.0", "pandas>=2.1.4", "python-dateutil>=2.8.2", "vtlengine>=1.2,<2.0", 
+       "numpy>=2.0.2,<2.1; python_version < '3.13'", "numpy>=2.1.0,<2.3; python_version >= '3.13'"]
 
 [tool.poetry]
 requires-poetry = ">=2.0"
@@ -47,6 +50,7 @@ requires-poetry = ">=2.0"
 python = ">=3.9,<4.0"
 
 [tool.poetry.group.dev.dependencies]
+coverage = ">=7.0,!=7.13.0,<8.0"
 ruff = ">=0.13.2"
 mypy = "^1.18.2"
 pytest = "^8.3.2"
@@ -57,7 +61,10 @@ pyroma = "^4.2"
 lxml-stubs = "^0.5.1"
 types-xmltodict = "^0.13.0.3"
 types-python-dateutil = "^2.9.0.20240316"
-pandas-stubs = "^2.1.4.231227"
+pandas-stubs = [
+    { version = ">2.1.4.231227,<2.2.3.240901", python = "<3.13" },
+    { version = ">=2.2.3.241009", python = ">=3.13" }
+]
 types-jsonschema = "^4.25.1.20251009"
 
 [tool.poetry.group.docs.dependencies]

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.10.0rc2"
+__version__ = "1.11.0"

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/api/fmr/__init__.py

@@ -39,6 +39,7 @@ from pysdmx.model import (
     Dataflow,
     DataflowInfo,
     DataProvider,
+    DataStructureDefinition,
     Hierarchy,
     HierarchyAssociation,
     Metadataflow,
@@ -679,7 +680,7 @@ class RegistryClient(__BaseRegistryClient):
         agency: str = "*",
         id: str = "*",
         version: str = "+",
-    ) -> Sequence[Dataflow]:
+    ) -> Sequence[DataStructureDefinition]:
         """Get the data structures(s) matching the supplied parameters.
 
         Args:
@@ -1253,7 +1254,7 @@ class AsyncRegistryClient(__BaseRegistryClient):
         agency: str = "*",
         id: str = "*",
         version: str = "+",
-    ) -> Sequence[Dataflow]:
+    ) -> Sequence[DataStructureDefinition]:
         """Get the data structures(s) matching the supplied parameters.
 
         Args:

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/api/fmr/maintenance.py

@@ -64,9 +64,7 @@ class RegistryMaintenanceClient:
             timeout: The maximum number of seconds to wait before considering
                 that a request timed out. Defaults to 10 seconds.
         """
-        if api_endpoint.endswith("/"):
-            api_endpoint = api_endpoint[0:-1]
-        self._api_endpoint = f"{api_endpoint}"
+        self._api_endpoint = self.__sanitize_endpoint(api_endpoint)
         self._user = user
         self._password = password
         self._timeout = timeout
@@ -87,7 +85,6 @@ class RegistryMaintenanceClient:
     ) -> None:
         with httpx.Client(verify=self._ssl_context) as client:
             try:
-                url = f"{endpoint}"
                 auth = httpx.BasicAuth(self._user, self._password)
                 headers = {
                     "Content-Type": "application/text",
@@ -99,7 +96,7 @@ class RegistryMaintenanceClient:
                     serializer = serializers.structure_message
                 bodyjs = self._encoder.encode(serializer.from_model(message))
                 r = client.post(
-                    url,
+                    endpoint,
                     headers=headers,
                     content=bodyjs,
                     timeout=self._timeout,
@@ -156,3 +153,11 @@ class RegistryMaintenanceClient:
         message = MetadataMessage(header=header, reports=reports)
         endpoint = f"{self._api_endpoint}/ws/secure/sdmx/v2/metadata"
         return self.__post(message, action, endpoint)
+
+    def __sanitize_endpoint(self, endpoint: str) -> str:
+        if endpoint.endswith("/"):
+            endpoint = endpoint[0:-1]
+        endpoint = endpoint.replace("/ws/secure/sdmx/v2/metadata", "")
+        endpoint = endpoint.replace("/sdmx/v2", "")
+        endpoint = endpoint.replace("/ws/secure/sdmxapi/rest", "")
+        return endpoint

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/api/qb/data.py

@@ -329,6 +329,8 @@ class DataQuery(_CoreDataQuery, frozen=True, omit_defaults=True):
 
     def __get_short_v2_qs(self, api_version: ApiVersion) -> str:
         qs = ""
+        if self.components:
+            qs += self._create_component_filters(self.components)
         if self.updated_after:
             qs = super()._append_qs_param(
                 qs,

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/__csv_aux_writer.py

@@ -3,8 +3,6 @@ from typing import List, Literal, Optional, Sequence
 
 import pandas as pd
 
-from pysdmx.errors import Invalid
-from pysdmx.io._pd_utils import _fill_na_values
 from pysdmx.io.pd import PandasDataset
 from pysdmx.model import Schema
 from pysdmx.model.dataset import ActionType
@@ -18,25 +16,6 @@ SDMX_CSV_ACTION_MAPPER = {
 }
 
 
-def _validate_schema_exists(dataset: PandasDataset) -> Schema:
-    """Validates that the dataset has a Schema defined.
-
-    Args:
-        dataset: The dataset to validate.
-
-    Returns:
-        The `Schema` from the dataset.
-
-    Raises:
-        Invalid: If the structure is not a `Schema`.
-    """
-    if not isinstance(dataset.structure, Schema):
-        raise Invalid(
-            "Dataset Structure is not a Schema. Cannot perform operation."
-        )
-    return dataset.structure
-
-
 def __write_time_period(df: pd.DataFrame, time_format: str) -> None:
     # TODO: Correct handle of normalized time format
     raise NotImplementedError("Normalized time format is not implemented yet.")
@@ -91,10 +70,8 @@ def _write_csv_2_aux(
 ) -> List[pd.DataFrame]:
     dataframes = []
     for dataset in datasets:
-        schema = _validate_schema_exists(dataset)
         # Create a copy of the dataset
         df: pd.DataFrame = copy(dataset.data)
-        df = _fill_na_values(df, schema)
         structure_ref, unique_id = dataset.short_urn.split("=", maxsplit=1)
 
         # Add additional attributes to the dataset

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/sdmx10/reader/__init__.py

@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     """
     # Get Dataframe from CSV file
     df_csv = pd.read_csv(
-        StringIO(input_str), keep_default_na=False, na_values=[]
+        StringIO(input_str), keep_default_na=False, na_values=[""]
     )
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/sdmx10/writer/__init__.py

@@ -6,12 +6,9 @@ from typing import Literal, Optional, Sequence, Union
 
 import pandas as pd
 
-from pysdmx.io._pd_utils import _fill_na_values
-from pysdmx.io.csv.__csv_aux_writer import (
-    __write_time_period,
-    _validate_schema_exists,
-)
+from pysdmx.io.csv.__csv_aux_writer import __write_time_period
 from pysdmx.io.pd import PandasDataset
+from pysdmx.model import Schema
 from pysdmx.toolkit.pd._data_utils import format_labels
 
 
@@ -47,26 +44,22 @@ def write(
     # Create a copy of the dataset
     dataframes = []
     for dataset in datasets:
-        # Validate that dataset has a proper Schema
-        schema = _validate_schema_exists(dataset)
-
         df: pd.DataFrame = copy(dataset.data)
 
-        # Fill missing values
-        df = _fill_na_values(df, schema)
-
         # Add additional attributes to the dataset
         for k, v in dataset.attributes.items():
             df[k] = v
         structure_id = dataset.short_urn.split("=")[1]
         if time_format is not None and time_format != "original":
             __write_time_period(df, time_format)
-        if labels is not None:
-            format_labels(df, labels, schema.components)
+        if labels is not None and isinstance(dataset.structure, Schema):
+            format_labels(df, labels, dataset.structure.components)
             if labels == "id":
                 df.insert(0, "DATAFLOW", structure_id)
             else:
-                df.insert(0, "DATAFLOW", f"{structure_id}:{schema.name}")
+                df.insert(
+                    0, "DATAFLOW", f"{structure_id}:{dataset.structure.name}"
+                )
         else:
             df.insert(0, "DATAFLOW", structure_id)
 
@@ -75,7 +68,8 @@ def write(
     # Concatenate the dataframes
     all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
-    all_data = all_data.astype(str)
+    # Ensure null values are represented as empty strings
+    all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
     # If the output path is an empty string we use None
     output_path = (
         None

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/sdmx20/reader/__init__.py

@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     """
     # Get Dataframe from CSV file
     df_csv = pd.read_csv(
-        StringIO(input_str), keep_default_na=False, na_values=[]
+        StringIO(input_str), keep_default_na=False, na_values=[""]
     )
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/sdmx20/writer/__init__.py

@@ -60,7 +60,7 @@ def write(
 
     all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
-    all_data = all_data.astype(str)
+    all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
 
     # If the output path is an empty string we use None
     output_path = (

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/sdmx21/reader/__init__.py

@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
     """
     # Get Dataframe from CSV file
     df_csv = pd.read_csv(
-        StringIO(input_str), keep_default_na=False, na_values=[]
+        StringIO(input_str), keep_default_na=False, na_values=[""]
     )
     # Drop empty columns
     df_csv = df_csv.dropna(axis=1, how="all")

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/csv/sdmx21/writer/__init__.py

@@ -57,7 +57,7 @@ def write(
 
     all_data = pd.concat(dataframes, ignore_index=True, axis=0)
 
-    all_data = all_data.astype(str)
+    all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
 
     # If the output path is an empty string we use None
     output_path = (

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/input_processor.py

@@ -97,6 +97,10 @@ def __get_sdmx_json_flavour(input_str: str) -> Tuple[str, Format]:
         return input_str, Format.STRUCTURE_SDMX_JSON_2_0_0
     elif "2.0.0/sdmx-json-metadata-schema.json" in flavour_check:
         return input_str, Format.REFMETA_SDMX_JSON_2_0_0
+    elif "2.1/sdmx-json-structure-schema.json" in flavour_check:
+        return input_str, Format.STRUCTURE_SDMX_JSON_2_1_0
+    elif "2.1/sdmx-json-metadata-schema.json" in flavour_check:
+        return input_str, Format.REFMETA_SDMX_JSON_2_1_0
     elif "sdmx-json" in flavour_check:
         raise NotImplemented(
             "Unsupported format", "This flavour of SDMX-JSON is not supported."

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/json/fusion/messages/dsd.py

@@ -20,6 +20,7 @@ from pysdmx.model import (
     Codelist,
     Component,
     Components,
+    Concept,
     DataStructureDefinition,
     DataType,
     Facets,
@@ -32,14 +33,15 @@ from pysdmx.util import parse_item_urn
 def _find_concept(
     cs: Sequence[FusionConceptScheme],
     urn: str,
-) -> FusionConcept:
+) -> Optional[FusionConcept]:
     r = parse_item_urn(urn)
     f = [
         m
         for m in cs
         if (m.agency == r.agency and m.id == r.id and m.version == r.version)
     ]
-    return [c for c in f[0].items if c.id == r.item_id][0]
+    m = [c for c in f[0].items if c.id == r.item_id]
+    return m[0] if m else None
 
 
 def _get_representation(
@@ -80,8 +82,14 @@ class FusionAttribute(Struct, frozen=True):
     measureReferences: Optional[Sequence[str]] = None
 
     def __derive_level(self, groups: Sequence[FusionGroup]) -> str:
-        if self.attachmentLevel == "OBSERVATION":
-            return "O"
+        if self.measureReferences:
+            if (
+                len(self.measureReferences) == 1
+                and self.measureReferences[0] == "OBS_VALUE"
+            ):
+                return "O"
+            else:
+                return ",".join(self.measureReferences)
         elif self.attachmentLevel == "DATA_SET":
             return "D"
         elif self.attachmentLevel == "GROUP":
@@ -106,12 +114,13 @@ class FusionAttribute(Struct, frozen=True):
         groups: Sequence[FusionGroup],
     ) -> Component:
         """Returns an attribute."""
-        c = _find_concept(cs, self.concept)
+        m = _find_concept(cs, self.concept) if cs else None
+        c = m.to_model(cls) if m else parse_item_urn(self.concept)
         dt, facets, codes, ab = _get_representation(
             self.id, self.representation, cls, cons
         )
         lvl = self.__derive_level(groups)
-        desc = c.descriptions[0].value if c.descriptions else None
+        desc = c.description if isinstance(c, Concept) else None
         if self.representation and self.representation.representation:
             local_enum_ref = self.representation.representation
         else:
@@ -120,10 +129,10 @@ class FusionAttribute(Struct, frozen=True):
             id=self.id,
             required=self.mandatory,
             role=Role.ATTRIBUTE,
-            concept=c.to_model(cls),
+            concept=c,
             local_dtype=dt,
             local_facets=facets,
-            name=c.names[0].value,
+            name=c.name if isinstance(c, Concept) else None,
             description=desc,
             local_codes=codes,
             attachment_level=lvl,
@@ -162,11 +171,12 @@ class FusionDimension(Struct, frozen=True):
         cons: Dict[str, Sequence[str]],
     ) -> Component:
         """Returns a dimension."""
-        c = _find_concept(cs, self.concept)
+        m = _find_concept(cs, self.concept) if cs else None
+        c = m.to_model(cls) if m else parse_item_urn(self.concept)
         dt, facets, codes, ab = _get_representation(
             self.id, self.representation, cls, cons
         )
-        desc = c.descriptions[0].value if c.descriptions else None
+        desc = c.description if isinstance(c, Concept) else None
         if self.representation and self.representation.representation:
             local_enum_ref = self.representation.representation
         else:
@@ -175,10 +185,10 @@ class FusionDimension(Struct, frozen=True):
             id=self.id,
             required=True,
             role=Role.DIMENSION,
-            concept=c.to_model(cls),
+            concept=c,
             local_dtype=dt,
             local_facets=facets,
-            name=c.names[0].value,
+            name=c.name if isinstance(c, Concept) else None,
             description=desc,
             local_codes=codes,
             array_def=ab,
@@ -216,11 +226,12 @@ class FusionMeasure(Struct, frozen=True):
         cons: Dict[str, Sequence[str]],
     ) -> Component:
         """Returns a measure."""
-        c = _find_concept(cs, self.concept)
+        m = _find_concept(cs, self.concept) if cs else None
+        c = m.to_model(cls) if m else parse_item_urn(self.concept)
         dt, facets, codes, ab = _get_representation(
             self.id, self.representation, cls, cons
         )
-        desc = c.descriptions[0].value if c.descriptions else None
+        desc = c.description if isinstance(c, Concept) else None
         if self.representation and self.representation.representation:
             local_enum_ref = self.representation.representation
         else:
@@ -229,10 +240,10 @@ class FusionMeasure(Struct, frozen=True):
             id=self.id,
             required=self.mandatory,
             role=Role.MEASURE,
-            concept=c.to_model(cls),
+            concept=c,
             local_dtype=dt,
             local_facets=facets,
-            name=c.names[0].value,
+            name=c.name if isinstance(c, Concept) else None,
             description=desc,
             local_codes=codes,
             array_def=ab,

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/json/fusion/messages/msd.py

@@ -14,13 +14,9 @@ from pysdmx.io.json.fusion.messages.dsd import (
     _find_concept,
     _get_representation,
 )
-from pysdmx.model import (
-    ArrayBoundaries,
-    MetadataComponent,
-)
-from pysdmx.model import (
-    MetadataStructure as MSD,
-)
+from pysdmx.model import ArrayBoundaries, MetadataComponent
+from pysdmx.model import MetadataStructure as MSD
+from pysdmx.util import parse_item_urn
 
 
 class FusionMetadataAttribute(Struct, frozen=True):
@@ -40,7 +36,8 @@ class FusionMetadataAttribute(Struct, frozen=True):
         cls: Sequence[FusionCodelist],
     ) -> MetadataComponent:
         """Returns an attribute."""
-        c = _find_concept(cs, self.concept)
+        m = _find_concept(cs, self.concept) if cs else None
+        c = m.to_model(cls) if m else parse_item_urn(self.concept)
         dt, facets, codes, _ = _get_representation(
             self.id, self.representation, cls, {}
         )
@@ -60,7 +57,7 @@ class FusionMetadataAttribute(Struct, frozen=True):
         return MetadataComponent(
             self.id,
             is_presentational=self.presentational,  # type: ignore[arg-type]
-            concept=c.to_model(cls),
+            concept=c,
             local_dtype=dt,
             local_facets=facets,
             local_codes=codes,

{pysdmx-1.10.0rc2 → pysdmx-1.11.0}/src/pysdmx/io/json/sdmxjson2/messages/core.py

@@ -307,17 +307,24 @@ class JsonHeader(msgspec.Struct, frozen=True, omit_defaults=True):
         self,
         header: Header,
         msg_type: Literal["structure", "metadata"] = "structure",
+        msg_version: Literal["2.0.0", "2.1"] = "2.0.0",
     ) -> "JsonHeader":
         """Create an SDMX-JSON header from a pysdmx Header."""
+        if msg_version == "2.0.0":
+            schema = (
+                "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/"
+                f"develop/structure-message/tools/schemas/{msg_version}/"
+                f"sdmx-json-{msg_type}-schema.json"
+            )
+        else:
+            schema = (
+                f"https://json.sdmx.org/2.1/sdmx-json-{msg_type}-schema.json"
+            )
         return JsonHeader(
             header.id,
             header.prepared,
             header.sender,
             header.test,
             receivers=header.receiver if header.receiver else None,
-            schema=(
-                "https://raw.githubusercontent.com/sdmx-twg/sdmx-json/"
-                "develop/structure-message/tools/schemas/2.0.0/"
-                f"sdmx-json-{msg_type}-schema.json"
-            ),
+            schema=schema,
         )