pysdmx 1.8.1__py3-none-any.whl → 1.10.0__py3-none-any.whl

pysdmx/__extras_check.py

@@ -46,7 +46,7 @@ def __check_xml_extra() -> None:
 
 def __check_vtl_extra() -> None:
     try:
-        import vtlengine  # type: ignore[import-untyped]  # noqa: F401
+        import vtlengine  # noqa: F401
     except ImportError:
         raise ImportError(
             ERROR_MESSAGE.format(
@@ -55,3 +55,17 @@ def __check_vtl_extra() -> None:
                 " and prettify",
             )
         ) from None
+
+
+def __check_json_extra() -> None:
+    try:
+        import jsonschema  # noqa: F401
+        import sdmxschemas  # noqa: F401
+    except ImportError:
+        raise ImportError(
+            ERROR_MESSAGE.format(
+                extra_name="json",
+                extra_desc="the validation of SDMX-JSON Structure Messages "
+                "(hint, use validate=False if you don't need validation)",
+            )
+        ) from None

pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.8.1"
+__version__ = "1.10.0"

pysdmx/api/fmr/__init__.py

@@ -44,6 +44,7 @@ from pysdmx.model import (
     Metadataflow,
     MetadataProvisionAgreement,
     MetadataReport,
+    MetadataStructure,
     MultiRepresentationMap,
     ProvisionAgreement,
     RepresentationMap,
@@ -767,7 +768,7 @@ class RegistryClient(__BaseRegistryClient):
         agency: str = "*",
         id: str = "*",
         version: str = "+",
-    ) -> Sequence[Dataflow]:
+    ) -> Sequence[MetadataStructure]:
         """Get the metadata structures (MSD) matching the supplied parameters.
 
         Args:
@@ -1341,7 +1342,7 @@ class AsyncRegistryClient(__BaseRegistryClient):
         agency: str = "*",
         id: str = "*",
         version: str = "+",
-    ) -> Sequence[Dataflow]:
+    ) -> Sequence[MetadataStructure]:
         """Get the metadata structures (MSD) matching the supplied parameters.
 
         Args:

pysdmx/io/input_processor.py

@@ -29,16 +29,19 @@ def __check_xml(input_str: str) -> bool:
 
 def __check_csv(input_str: str) -> bool:
     try:
-        max_length = min(2048, len(input_str))
-        dialect = csv.Sniffer().sniff(input_str[:max_length])
+        lines = input_str.splitlines()
+
+        # Use the first N complete lines
+        # (1 should be enough)
+        max_lines = 1
+        sample = "\n".join(lines[:max_lines])
+
+        dialect = csv.Sniffer().sniff(sample)
         control_csv_format = (
             dialect.delimiter == "," and dialect.quotechar == '"'
         )
         # Check we can access the data and it is not empty
-        if (
-            len(input_str.splitlines()) > 1
-            or input_str.splitlines()[0].count(",") > 1
-        ) and control_csv_format:
+        if (len(lines) > 1 or lines[0].count(",") > 1) and control_csv_format:
             return True
     except Exception:
         return False

pysdmx/io/json/fusion/messages/category.py

@@ -1,7 +1,7 @@
 """Collection of Fusion-JSON schemas for categories and category schemes."""
 
 from collections import defaultdict
-from typing import Dict, Optional, Sequence
+from typing import Dict, Optional, Sequence, Tuple, Union
 
 from msgspec import Struct
 
@@ -11,6 +11,8 @@ from pysdmx.model import (
     Agency,
     Category,
     DataflowRef,
+    ItemReference,
+    Reference,
 )
 from pysdmx.model import (
     Categorisation as CT,
@@ -21,7 +23,7 @@ from pysdmx.model import (
 from pysdmx.model import (
     Dataflow as DF,
 )
-from pysdmx.util import find_by_urn
+from pysdmx.util import find_by_urn, parse_urn
 
 
 class FusionCategorisation(Struct, frozen=True, rename={"agency": "agencyId"}):
@@ -57,14 +59,46 @@ class FusionCategory(Struct, frozen=True):
     descriptions: Optional[Sequence[FusionString]] = None
     items: Sequence["FusionCategory"] = ()
 
-    def to_model(self) -> Category:
+    def __add_flows(
+        self, cni: str, cf: Dict[str, list[DF]]
+    ) -> Sequence[DataflowRef]:
+        if cni in cf:
+            return [
+                DataflowRef(
+                    (
+                        df.agency.id
+                        if isinstance(df.agency, Agency)
+                        else df.agency
+                    ),
+                    df.id,
+                    df.version,
+                    df.name,
+                )
+                for df in cf[cni]
+            ]
+        else:
+            return ()
+
+    def to_model(
+        self,
+        cat_flows: dict[str, list[DF]],
+        cat_other: dict[str, list[Union[ItemReference, Reference]]],
+        parent_id: Optional[str] = None,
+    ) -> Category:
         """Converts a FusionCode to a standard code."""
         description = self.descriptions[0].value if self.descriptions else None
+        cni = f"{parent_id}.{self.id}" if parent_id else self.id
+        dataflows = self.__add_flows(cni, cat_flows)
+        others = cat_other.get(cni, ())
         return Category(
             id=self.id,
             name=self.names[0].value,
             description=description,
-            categories=[c.to_model() for c in self.items],
+            categories=[
+                c.to_model(cat_flows, cat_other, cni) for c in self.items
+            ],
+            dataflows=dataflows,
+            other_references=others,
         )
 
 
@@ -78,16 +112,42 @@ class FusionCategoryScheme(Struct, frozen=True, rename={"agency": "agencyId"}):
     version: str = "1.0"
     items: Sequence[FusionCategory] = ()
 
-    def to_model(self) -> CS:
+    def __group_refs(
+        self,
+        categorisations: Sequence[FusionCategorisation],
+        dataflows: Sequence[FusionDataflow],
+    ) -> Tuple[
+        dict[str, list[DF]], dict[str, list[Union[ItemReference, Reference]]]
+    ]:
+        flows: defaultdict[str, list[DF]] = defaultdict(list)
+        other: defaultdict[str, list[Union[ItemReference, Reference]]] = (
+            defaultdict(list)
+        )
+        for c in categorisations:
+            ref = parse_urn(c.structureReference)
+            src = c.categoryReference[c.categoryReference.find(")") + 2 :]
+            if ref.sdmx_type == "Dataflow":
+                d = find_by_urn(dataflows, c.structureReference)
+                flows[src].append(d.to_model())
+            else:
+                other[src].append(ref)
+        return (flows, other)
+
+    def to_model(
+        self,
+        categorisations: Sequence[FusionCategorisation] = (),
+        dataflows: Sequence[FusionDataflow] = (),
+    ) -> CS:
         """Converts a JsonCodelist to a standard codelist."""
         description = self.descriptions[0].value if self.descriptions else None
+        cat_flows, cat_others = self.__group_refs(categorisations, dataflows)
         return CS(
             id=self.id,
             name=self.names[0].value,
             agency=self.agency,
             description=description,
             version=self.version,
-            items=[c.to_model() for c in self.items],
+            items=[c.to_model(cat_flows, cat_others) for c in self.items],
         )
 
 
@@ -98,43 +158,11 @@ class FusionCategorySchemeMessage(Struct, frozen=True):
     Categorisation: Sequence[FusionCategorisation] = ()
     Dataflow: Sequence[FusionDataflow] = ()
 
-    def __group_flows(self) -> defaultdict[str, list[DF]]:
-        out: defaultdict[str, list[DF]] = defaultdict(list)
-        for c in self.Categorisation:
-            d = find_by_urn(self.Dataflow, c.structureReference)
-            src = c.categoryReference[c.categoryReference.find(")") + 2 :]
-            out[src].append(d.to_model())
-        return out
-
-    def __add_flows(
-        self, cat: Category, cni: str, cf: Dict[str, list[DF]]
-    ) -> None:
-        if cat.categories:
-            for c in cat.categories:
-                self.__add_flows(c, f"{cni}.{c.id}", cf)
-        if cni in cf:
-            dfrefs = [
-                DataflowRef(
-                    (
-                        df.agency.id
-                        if isinstance(df.agency, Agency)
-                        else df.agency
-                    ),
-                    df.id,
-                    df.version,
-                    df.name,
-                )
-                for df in cf[cni]
-            ]
-            cat.dataflows = dfrefs
-
     def to_model(self) -> CS:
         """Returns the requested category scheme."""
-        cf = self.__group_flows()
-        cs = self.CategoryScheme[0].to_model()
-        for c in cs:
-            self.__add_flows(c, c.id, cf)
-        return cs
+        return self.CategoryScheme[0].to_model(
+            self.Categorisation, self.Dataflow
+        )
 
 
 class FusionCategorisationMessage(Struct, frozen=True):

pysdmx/io/json/sdmxjson2/messages/__init__.py

@@ -12,6 +12,9 @@ from pysdmx.io.json.sdmxjson2.messages.code import (
     JsonHierarchyMessage,
 )
 from pysdmx.io.json.sdmxjson2.messages.concept import JsonConceptSchemeMessage
+from pysdmx.io.json.sdmxjson2.messages.constraint import (
+    JsonDataConstraintMessage,
+)
 from pysdmx.io.json.sdmxjson2.messages.dataflow import (
     JsonDataflowMessage,
     JsonDataflowsMessage,
@@ -50,6 +53,7 @@ __all__ = [
     "JsonCategorySchemeMessage",
     "JsonCodelistMessage",
     "JsonConceptSchemeMessage",
+    "JsonDataConstraintMessage",
     "JsonDataflowMessage",
     "JsonDataflowsMessage",
     "JsonDataStructuresMessage",

pysdmx/io/json/sdmxjson2/messages/category.py

@@ -1,7 +1,7 @@
 """Collection of SDMX-JSON schemas for categories and category schemes."""
 
 from collections import defaultdict
-from typing import Dict, Sequence
+from typing import Dict, Optional, Sequence, Tuple, Union
 
 from msgspec import Struct
 
@@ -20,8 +20,10 @@ from pysdmx.model import (
     CategoryScheme,
     Dataflow,
     DataflowRef,
+    ItemReference,
+    Reference,
 )
-from pysdmx.util import find_by_urn
+from pysdmx.util import find_by_urn, parse_urn
 
 
 class JsonCategorisation(
@@ -84,14 +86,46 @@ class JsonCategory(NameableType, frozen=True, omit_defaults=True):
 
     categories: Sequence["JsonCategory"] = ()
 
-    def to_model(self) -> Category:
+    def __add_flows(
+        self, cni: str, cf: Dict[str, list[Dataflow]]
+    ) -> Sequence[DataflowRef]:
+        if cni in cf:
+            return [
+                DataflowRef(
+                    (
+                        df.agency.id
+                        if isinstance(df.agency, Agency)
+                        else df.agency
+                    ),
+                    df.id,
+                    df.version,
+                    df.name,
+                )
+                for df in cf[cni]
+            ]
+        else:
+            return ()
+
+    def to_model(
+        self,
+        cat_flows: dict[str, list[Dataflow]],
+        cat_other: dict[str, list[Union[ItemReference, Reference]]],
+        parent_id: Optional[str] = None,
+    ) -> Category:
         """Converts a FusionCode to a standard code."""
+        cni = f"{parent_id}.{self.id}" if parent_id else self.id
+        dataflows = self.__add_flows(cni, cat_flows)
+        others = cat_other.get(cni, ())
         return Category(
             id=self.id,
             name=self.name,
             description=self.description,
-            categories=[c.to_model() for c in self.categories],
+            categories=[
+                c.to_model(cat_flows, cat_other, cni) for c in self.categories
+            ],
             annotations=[a.to_model() for a in self.annotations],
+            dataflows=dataflows,
+            other_references=others,
         )
 
     @classmethod
@@ -126,15 +160,42 @@ class JsonCategoryScheme(
 
     categories: Sequence[JsonCategory] = ()
 
-    def to_model(self) -> CategoryScheme:
-        """Converts a JsonCodelist to a standard codelist."""
+    def __group_refs(
+        self,
+        categorisations: Sequence[JsonCategorisation] = (),
+        dataflows: Sequence[JsonDataflow] = (),
+    ) -> Tuple[
+        dict[str, list[Dataflow]],
+        dict[str, list[Union[ItemReference, Reference]]],
+    ]:
+        flows: defaultdict[str, list[Dataflow]] = defaultdict(list)
+        other: defaultdict[str, list[Union[ItemReference, Reference]]] = (
+            defaultdict(list)
+        )
+        for c in categorisations:
+            ref = parse_urn(c.source)
+            src = c.target[c.target.find(")") + 2 :]
+            if ref.sdmx_type == "Dataflow":
+                d = find_by_urn(dataflows, c.source)
+                flows[src].append(d.to_model())
+            else:
+                other[src].append(ref)
+        return (flows, other)
+
+    def to_model(
+        self,
+        categorisations: Sequence[JsonCategorisation] = (),
+        dataflows: Sequence[JsonDataflow] = (),
+    ) -> CategoryScheme:
+        """Converts a JsonCategoryScheme to a standard one."""
+        cat_flows, cat_other = self.__group_refs(categorisations, dataflows)
         return CategoryScheme(
             id=self.id,
             name=self.name,
             agency=self.agency,
             description=self.description,
             version=self.version,
-            items=[c.to_model() for c in self.categories],
+            items=[c.to_model(cat_flows, cat_other) for c in self.categories],
             is_external_reference=self.isExternalReference,
             is_partial=self.isPartial,
             valid_from=self.validFrom,
@@ -179,49 +240,21 @@ class JsonCategorySchemes(Struct, frozen=True, omit_defaults=True):
     categorisations: Sequence[JsonCategorisation] = ()
     dataflows: Sequence[JsonDataflow] = ()
 
+    def to_model(self) -> CategoryScheme:
+        """Returns the requested codelist."""
+        return self.categorySchemes[0].to_model(
+            self.categorisations, self.dataflows
+        )
+
 
 class JsonCategorySchemeMessage(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for /categoryscheme queries."""
 
     data: JsonCategorySchemes
 
-    def __group_flows(self) -> defaultdict[str, list[Dataflow]]:
-        out: defaultdict[str, list[Dataflow]] = defaultdict(list)
-        for c in self.data.categorisations:
-            d = find_by_urn(self.data.dataflows, c.source)
-            src = c.target[c.target.find(")") + 2 :]
-            out[src].append(d.to_model())
-        return out
-
-    def __add_flows(
-        self, cat: Category, cni: str, cf: Dict[str, list[Dataflow]]
-    ) -> None:
-        if cat.categories:
-            for c in cat.categories:
-                self.__add_flows(c, f"{cni}.{c.id}", cf)
-        if cni in cf:
-            dfrefs = [
-                DataflowRef(
-                    (
-                        df.agency.id
-                        if isinstance(df.agency, Agency)
-                        else df.agency
-                    ),
-                    df.id,
-                    df.version,
-                    df.name,
-                )
-                for df in cf[cni]
-            ]
-            cat.dataflows = dfrefs
-
     def to_model(self) -> CategoryScheme:
-        """Returns the requested codelist."""
-        cf = self.__group_flows()
-        cs = self.data.categorySchemes[0].to_model()
-        for c in cs:
-            self.__add_flows(c, c.id, cf)
-        return cs
+        """Returns the requested category scheme."""
+        return self.data.to_model()
 
 
 class JsonCategorisations(Struct, frozen=True, omit_defaults=True):

pysdmx/io/json/sdmxjson2/messages/code.py

@@ -231,7 +231,7 @@ class JsonCodelists(Struct, frozen=True, omit_defaults=True):
     """SDMX-JSON payload for lists of codes."""
 
     codelists: Sequence[JsonCodelist] = ()
-    valuelists: Sequence[JsonValuelist] = ()
+    valueLists: Sequence[JsonValuelist] = ()
 
 
 class JsonCodelistMessage(Struct, frozen=True, omit_defaults=True):
@@ -244,7 +244,7 @@ class JsonCodelistMessage(Struct, frozen=True, omit_defaults=True):
         if self.data.codelists:
             return self.data.codelists[0].to_model()
         else:
-            return self.data.valuelists[0].to_model()
+            return self.data.valueLists[0].to_model()
 
 
 class JsonHierarchicalCode(Struct, frozen=True, omit_defaults=True):
@@ -329,10 +329,10 @@ class JsonHierarchicalCode(Struct, frozen=True, omit_defaults=True):
             code=code.urn,
             validFrom=code.rel_valid_from,
             validTo=code.rel_valid_to,
-            annotations=tuple(annotations),
-            hierarchicalCodes=[
-                JsonHierarchicalCode.from_model(c) for c in code.codes
-            ],
+            annotations=tuple(annotations) if annotations else None,
+            hierarchicalCodes=tuple(
+                [JsonHierarchicalCode.from_model(c) for c in code.codes]
+            ),
         )
 
 
@@ -475,6 +475,15 @@ class JsonHierarchyAssociation(
                 "SDMX-JSON hierarchy associations must reference a context",
                 {"hierarchy_association": ha.id},
             )
+        lnk = (
+            JsonLink(
+                rel="UserDefinedOperator",
+                type="sdmx_artefact",
+                urn=ha.operator,
+            )
+            if ha.operator
+            else None
+        )
         return JsonHierarchyAssociation(
             agency=(
                 ha.agency.id if isinstance(ha.agency, Agency) else ha.agency
@@ -492,6 +501,7 @@ class JsonHierarchyAssociation(
             linkedHierarchy=href,
             linkedObject=ha.component_ref,
             contextObject=ha.context_ref,
+            links=[lnk] if lnk else (),
         )