pysdmx 1.6.0__tar.gz → 1.7.0__tar.gz

{pysdmx-1.6.0 → pysdmx-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pysdmx
-Version: 1.6.0
+Version: 1.7.0
 Summary: Your opinionated Python SDMX library
 License: Apache-2.0
 License-File: LICENSE
@@ -18,7 +18,7 @@ Provides-Extra: data
 Provides-Extra: dc
 Provides-Extra: vtl
 Provides-Extra: xml
-Requires-Dist: httpx (>=0)
+Requires-Dist: httpx[http2] (>=0)
 Requires-Dist: lxml (>=5.2) ; extra == "all"
 Requires-Dist: lxml (>=5.2) ; extra == "xml"
 Requires-Dist: msgspec (>=0)

{pysdmx-1.6.0 → pysdmx-1.7.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pysdmx"
-version = "1.6.0"
+version = "1.7.0"
 description = "Your opinionated Python SDMX library"
 license = { text = "Apache-2.0" }
 readme = "README.rst"
@@ -21,7 +21,7 @@ classifiers = [
     "Typing :: Typed"
 ]
 dependencies = [
-    "httpx>=0.*",
+    "httpx[http2]>=0.*",
     "msgspec>=0.*",
     "parsy>=2.1",
 ]
@@ -46,7 +46,7 @@ requires-poetry = ">=2.0"
 python = ">=3.9,<4.0"
 
 [tool.poetry.group.dev.dependencies]
-ruff = "^0.8.1"
+ruff = ">=0.13.2"
 mypy = "^1.1.1"
 pytest = "^8.3.2"
 pytest-asyncio = "^0.21.1"
@@ -70,16 +70,30 @@ build-backend = "poetry.core.masonry.api"
 [tool.ruff]
 line-length = 79
 lint.select = [
-    "ASYNC", "B", "C4", "C90", "D", "E", "F", "I", "LOG", "PT", "S", "SIM", "W"
-]
-lint.ignore = [
-    "E203", "D411", "F901", "S320"
+    "ASYNC", # flake8-async: checks for async/await best practices
+    "B",     # flake8-bugbear: detect bugs and design problems
+    "C4",    # flake8-comprehensions: better list/dict/set comprehensions
+    "C90",   # mccabe: code complexity checker
+    "D",     # pydocstyle: enforce docstring conventions
+    "DTZ",   # flake8-datetimez: datetime best practices and timezone issues
+    "E",     # pycodestyle errors
+    "ERA",   # eradicate: Find commented-out code that should be removed
+    "F",     # pyflakes: detect various errors
+    "FURB",  # refurb: functional Python improvements and refactoring
+    "I",     # isort: import sorting
+    "LOG",   # flake8-logging: logging best practices
+    "PERF",  # perflint: performance optimizations
+    "PT",    # flake8-pytest-style: pytest best practices
+    "S",     # flake8-bandit: security issues
+    "SIM",   # flake8-simplify: code simplification suggestions
+    "W"      # pycodestyle warnings    
 ]
+lint.ignore = ["D411", "E203", "F901"]
 lint.mccabe.max-complexity = 10
 lint.pydocstyle.convention = "google"
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*" = ["S101", "S311", "D100", "D103", "D104"]
+"tests/*" = ["DTZ", "D100", "D103", "D104", "PERF", "S101", "S311"]
 
 [tool.mypy]
 files = "src"

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/__init__.py

@@ -1,3 +1,3 @@
 """Your opinionated Python SDMX library."""
 
-__version__ = "1.6.0"
+__version__ = "1.7.0"

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/input_processor.py

@@ -3,11 +3,11 @@
 import csv
 import os.path
 from io import BytesIO, StringIO, TextIOWrapper
-from json import JSONDecodeError, loads
 from os import PathLike
 from pathlib import Path
 from typing import Optional, Tuple, Union
 
+import msgspec
 from httpx import Client as httpx_Client
 from httpx import HTTPStatusError, create_ssl_context
 
@@ -29,7 +29,7 @@ def __check_xml(input_str: str) -> bool:
 
 def __check_csv(input_str: str) -> bool:
     try:
-        max_length = len(input_str) if len(input_str) < 2048 else 2048
+        max_length = min(2048, len(input_str))
         dialect = csv.Sniffer().sniff(input_str[:max_length])
         control_csv_format = (
             dialect.delimiter == "," and dialect.quotechar == '"'
@@ -47,9 +47,9 @@ def __check_csv(input_str: str) -> bool:
 
 def __check_json(input_str: str) -> bool:
     try:
-        loads(input_str)
+        msgspec.json.decode(input_str)
         return True
-    except JSONDecodeError:
+    except msgspec.DecodeError:
         return False
 
 

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/json/fusion/messages/concept.py

@@ -17,6 +17,7 @@ class FusionConcept(msgspec.Struct, frozen=True):
     """Fusion-JSON payload for concepts."""
 
     id: str
+    urn: str
     names: Sequence[FusionString]
     representation: Optional[FusionRepresentation] = None
     descriptions: Optional[Sequence[FusionString]] = None
@@ -48,6 +49,7 @@ class FusionConcept(msgspec.Struct, frozen=True):
             description=d,
             codes=c,
             enum_ref=cl_ref,
+            urn=self.urn,
         )
 
 
@@ -63,18 +65,10 @@ class FusionConceptScheme(
     version: str = "1.0"
     items: Sequence[FusionConcept] = ()
 
-    def __set_urn(self, concept: Concept) -> Concept:
-        urn = (
-            "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept="
-            f"{self.agency}:{self.id}({self.version}).{concept.id}"
-        )
-        return msgspec.structs.replace(concept, urn=urn)
-
     def to_model(self, codelists: Sequence[FusionCodelist]) -> CS:
         """Converts a FusionConceptScheme to a standard concept scheme."""
         d = self.descriptions[0].value if self.descriptions else None
         concepts = [c.to_model(codelists) for c in self.items]
-        concepts = [self.__set_urn(c) for c in concepts]
         return CS(
             id=self.id,
             name=self.names[0].value,

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/json/sdmxjson2/messages/code.py

@@ -34,7 +34,7 @@ class JsonCode(NameableType, frozen=True, omit_defaults=True):
     parent: Optional[str] = None
 
     def __handle_date(self, datestr: str) -> datetime:
-        return datetime.strptime(datestr, _VAL_FMT)
+        return datetime.strptime(datestr, _VAL_FMT)  # noqa
 
     def __get_val(
         self, a: JsonAnnotation
@@ -49,26 +49,38 @@ class JsonCode(NameableType, frozen=True, omit_defaults=True):
 
     def to_model(self) -> Code:
         """Converts a JsonCode to a standard code."""
+        # Pre-filter annotations once outside the tuple creation
+        vf, vt = None, None
+
         if self.annotations:
-            vp = [
-                a for a in self.annotations if a.type == "FR_VALIDITY_PERIOD"
+            # Get validity period info
+            vp = next(
+                (
+                    a
+                    for a in self.annotations
+                    if a.type == "FR_VALIDITY_PERIOD"
+                ),
+                None,
+            )
+            if vp:
+                vf, vt = self.__get_val(vp)
+
+            # Pre-filter non-validity period annotations
+            filtered_annotations = [
+                a.to_model()
+                for a in self.annotations
+                if a.type != "FR_VALIDITY_PERIOD"
             ]
         else:
-            vp = None
-        vf, vt = self.__get_val(vp[0]) if vp else (None, None)
+            filtered_annotations = []
+
         return Code(
             id=self.id,
             name=self.name,
             description=self.description,
             valid_from=vf,
             valid_to=vt,
-            annotations=tuple(
-                [
-                    a.to_model()
-                    for a in self.annotations
-                    if a.type != "FR_VALIDITY_PERIOD"
-                ]
-            ),
+            annotations=tuple(filtered_annotations),
         )
 
     @classmethod
@@ -113,13 +125,23 @@ class JsonCodelist(ItemSchemeType, frozen=True, omit_defaults=True):
 
     def to_model(self) -> Codelist:
         """Converts a JsonCodelist to a standard codelist."""
+        # Process codes in batches to reduce memory pressure
+        batch_size = 10000
+        all_codes = []
+
+        # Process in batches
+        for i in range(0, len(self.codes), batch_size):
+            batch = self.codes[i : i + batch_size]
+            batch_models = [code.to_model() for code in batch]
+            all_codes.extend(batch_models)
+
         return Codelist(
             id=self.id,
             name=self.name,
             agency=self.agency,
             description=self.description,
             version=self.version,
-            items=tuple([i.to_model() for i in self.codes]),
+            items=tuple(all_codes),
             annotations=tuple([a.to_model() for a in self.annotations]),
             is_external_reference=self.isExternalReference,
             is_partial=self.isPartial,

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/json/sdmxjson2/messages/concept.py

@@ -9,6 +9,7 @@ from pysdmx.io.json.sdmxjson2.messages.code import JsonCodelist
 from pysdmx.io.json.sdmxjson2.messages.core import (
     ItemSchemeType,
     JsonAnnotation,
+    JsonLink,
     JsonRepresentation,
     NameableType,
 )
@@ -29,6 +30,7 @@ class JsonConcept(NameableType, frozen=True, omit_defaults=True):
     coreRepresentation: Optional[JsonRepresentation] = None
     parent: Optional[str] = None
     isoConceptReference: Optional[IsoConceptReference] = None
+    links: Sequence[JsonLink] = ()
 
     def to_model(self, codelists: Sequence[Codelist]) -> Concept:
         """Converts a JsonConcept to a standard concept."""
@@ -48,6 +50,8 @@ class JsonConcept(NameableType, frozen=True, omit_defaults=True):
             facets = None
             codes = None
             cl_ref = None
+        urn_lnk = [lnk for lnk in self.links if lnk.rel == "self"]
+        c_urn = urn_lnk[0].urn if len(urn_lnk) > 0 else None
         return Concept(
             id=self.id,
             dtype=dt,
@@ -56,6 +60,7 @@ class JsonConcept(NameableType, frozen=True, omit_defaults=True):
             description=self.description,
             codes=codes,
             enum_ref=cl_ref,
+            urn=c_urn,
         )
 
     @classmethod
@@ -95,18 +100,10 @@ class JsonConceptScheme(ItemSchemeType, frozen=True, omit_defaults=True):
 
     concepts: Sequence[JsonConcept] = ()
 
-    def __set_urn(self, concept: Concept) -> Concept:
-        urn = (
-            "urn:sdmx:org.sdmx.infomodel.conceptscheme.Concept="
-            f"{self.agency}:{self.id}({self.version}).{concept.id}"
-        )
-        return msgspec.structs.replace(concept, urn=urn)
-
     def to_model(self, codelists: Sequence[JsonCodelist]) -> ConceptScheme:
         """Converts a JsonConceptScheme to a standard concept scheme."""
         cls = [cl.to_model() for cl in codelists]
         concepts = [c.to_model(cls) for c in self.concepts]
-        concepts = [self.__set_urn(c) for c in concepts]
         return ConceptScheme(
             id=self.id,
             name=self.name,

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/__ss_aux_reader.py

@@ -30,8 +30,7 @@ def _reading_str_series(dataset: Dict[str, Any]) -> pd.DataFrame:
         if OBS in data:
             del keys[OBS]
             data[OBS] = add_list(data[OBS])
-            for j in data[OBS]:
-                test_list.append({**keys, **j})
+            test_list.extend([{**keys, **j} for j in data[OBS]])
         else:
             test_list.append(keys)
         test_list, df = __process_df(test_list, df)

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/__structure_aux_reader.py

@@ -397,7 +397,7 @@ class StructureParser(Struct):
         """
         if json_fac is None:
             return
-        for key, _value in json_fac.items():
+        for key in json_fac:
             if key == TEXT_TYPE and json_fac[TEXT_TYPE] in list(DataType):
                 json_obj["dtype"] = DataType(json_fac[TEXT_TYPE])
 
@@ -907,9 +907,9 @@ class StructureParser(Struct):
         item_json_info = self.__format_name_description(item_json_info)
         if CONTACT in item_json_info and item_name_class == AGENCY:
             item_json_info[CONTACT] = add_list(item_json_info[CONTACT])
-            contacts = []
-            for e in item_json_info[CONTACT]:
-                contacts.append(self.__format_contact(e))
+            contacts = [
+                self.__format_contact(e) for e in item_json_info[CONTACT]
+            ]
             item_json_info[CONTACT.lower() + "s"] = contacts
             del item_json_info[CONTACT]
 
@@ -949,9 +949,11 @@ class StructureParser(Struct):
                         group_dimensions = [group_dimensions]
 
                     group["dimensions"] = [
-                        d[DIM_REF]
-                        if isinstance(d[DIM_REF], str)
-                        else d[DIM_REF][REF][ID]
+                        (
+                            d[DIM_REF]
+                            if isinstance(d[DIM_REF], str)
+                            else d[DIM_REF][REF][ID]
+                        )
                         for d in group_dimensions
                     ]
 
@@ -994,9 +996,12 @@ class StructureParser(Struct):
             items = []
             if item in element:
                 element[item] = add_list(element[item])
-                for item_elem in element[item]:
-                    # Dynamic
-                    items.append(self.__format_item(item_elem, item))
+                items.extend(
+                    [
+                        self.__format_item(item_elem, item)
+                        for item_elem in element[item]
+                    ]
+                )
                 del element[item]
             element["items"] = items
             element = self.__format_agency(element)

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/__structure_aux_writer.py

@@ -310,9 +310,9 @@ def __write_maintainable(
         f"{str(maintainable.is_external_reference).lower()!r}"
     )
     if not references_30 and not (isinstance(maintainable, AgencyScheme)):
-        outfile["Attributes"] += (
-            f" isFinal={str(maintainable.is_final).lower()!r}"
-        )
+        outfile[
+            "Attributes"
+        ] += f" isFinal={str(maintainable.is_final).lower()!r}"
 
     if isinstance(maintainable.agency, str):
         outfile["Attributes"] += f" agencyID={maintainable.agency!r}"
@@ -462,7 +462,7 @@ def __write_components(  # noqa: C901
         )
 
     position = 1
-    for _, comps in components.items():
+    for comps in components.values():
         if comps:
             role_name = ROLE_MAPPING[comps[0].role]
             if role_name == MEASURE:
@@ -809,9 +809,9 @@ def __write_scheme(  # noqa: C901
         DSD,
         DFW,
     ]:
-        data["Attributes"] += (
-            f" isPartial={str(item_scheme.is_partial).lower()!r}"
-        )
+        data[
+            "Attributes"
+        ] += f" isPartial={str(item_scheme.is_partial).lower()!r}"
     if scheme in [
         RULE_SCHEME,
         UDO_SCHEME,
@@ -820,9 +820,9 @@ def __write_scheme(  # noqa: C901
         CUSTOM_TYPE_SCHEME,
         NAME_PER_SCHEME,
     ]:
-        data["Attributes"] += (
-            f" {_write_vtl(item_scheme, indent, references_30)}"
-        )
+        data[
+            "Attributes"
+        ] += f" {_write_vtl(item_scheme, indent, references_30)}"
 
     outfile = ""
 
@@ -1129,9 +1129,11 @@ def _write_vtl(  # noqa: C901
             ref_codelist = (
                 item_or_scheme.codelist
                 if isinstance(item_or_scheme.codelist, Reference)
-                else parse_urn(item_or_scheme.codelist)
-                if isinstance(item_or_scheme.codelist, str)
-                else parse_short_urn(item_or_scheme.codelist.short_urn)
+                else (
+                    parse_urn(item_or_scheme.codelist)
+                    if isinstance(item_or_scheme.codelist, str)
+                    else parse_short_urn(item_or_scheme.codelist.short_urn)
+                )
             )
             if references_30:
                 data += (

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/__write_data_aux.py

@@ -54,13 +54,15 @@ def writing_validation(dataset: PandasDataset) -> None:
         for comp in dataset.structure.components
         if comp.role in (Role.DIMENSION, Role.MEASURE)
     ]
-    for att in dataset.structure.components.attributes:
+    required_components.extend(
+        att.id
+        for att in dataset.structure.components.attributes
         if (
             att.required
             and att.attachment_level is not None
             and att.attachment_level != "D"
-        ):
-            required_components.append(att.id)
+        )
+    )
     non_required = [
         comp.id
         for comp in dataset.structure.components

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/__write_structure_specific_aux.py

@@ -198,8 +198,12 @@ def __group_processing(
             .to_dict(orient="records")
         )
 
-        for record in grouped_data:
-            out_list.append(__format_group_str(record, group["group_id"]))
+        out_list.extend(
+            [
+                __format_group_str(record, group["group_id"])
+                for record in grouped_data
+            ]
+        )
 
     return "".join(out_list)
 

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/doc_validation.py

@@ -39,9 +39,7 @@ def validate_doc(input_str: str) -> None:
     doc = etree.parse(bytes_infile, parser=parser)
     if not xmlschema.validate(doc):
         log_errors = list(xmlschema.error_log)  # type: ignore[call-overload]
-        unhandled_errors = []
-        for e in log_errors:
-            unhandled_errors.append(e.message)
+        unhandled_errors = [e.message for e in log_errors]
         severe_errors = unhandled_errors.copy()
         for e in unhandled_errors:
             for allowed_error in ALLOWED_ERRORS_CONTENT:

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/io/xml/sdmx21/writer/generic.py

@@ -327,10 +327,12 @@ def __group_processing(
             .to_dict(orient="records")
         )
 
-        for record in grouped_data:
-            out_list.append(
+        out_list.extend(
+            [
                 __format_group_str(record, group_id, dimensions, attribute)
-            )
+                for record in grouped_data
+            ]
+        )
 
     return "".join(out_list)
 

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/model/__init__.py

@@ -131,9 +131,7 @@ def decoders(type: Type, obj: Any) -> Any:  # type: ignore[type-arg]
             target types
     """
     if type is Components:
-        comps = []
-        for item in obj:
-            comps.append(msgspec.convert(item, Component))
+        comps = [msgspec.convert(item, Component) for item in obj]
         return Components(comps)
     else:
         raise NotImplementedError(f"Objects of type {type} are not supported")

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/model/map.py

@@ -486,9 +486,7 @@ class StructureMap(MaintainableArtefact, frozen=True, omit_defaults=True):
         """Return the number of mapping rules in the structure map."""
         return len(self.maps)
 
-    def __getitem__(
-        self, id_: str
-    ) -> Optional[
+    def __getitem__(self, id_: str) -> Optional[
         Sequence[
             Union[
                 ComponentMap,
@@ -500,12 +498,12 @@ class StructureMap(MaintainableArtefact, frozen=True, omit_defaults=True):
         ]
     ]:
         """Return the mapping rules for the supplied component."""
-        out = []
-        for m in self.maps:
-            if (
-                hasattr(m, "source") and (m.source == id_ or id_ in m.source)
-            ) or (isinstance(m, FixedValueMap) and m.target == id_):
-                out.append(m)
+        out = [
+            m
+            for m in self.maps
+            if (hasattr(m, "source") and (m.source == id_ or id_ in m.source))
+            or (isinstance(m, FixedValueMap) and m.target == id_)
+        ]
         if len(out) == 0:
             return None
         else:

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/model/message.py

@@ -168,10 +168,7 @@ class StructureMessage(Struct, repr_omit_defaults=True, frozen=True):
             raise NotFound(
                 f"No {type_.__name__} found in message.",
             )
-        structures = []
-        for element in self.structures:
-            if isinstance(element, type_):
-                structures.append(element)
+        structures = [e for e in self.structures if isinstance(e, type_)]
         return structures
 
     def __get_enumerations(

{pysdmx-1.6.0 → pysdmx-1.7.0}/src/pysdmx/toolkit/pd/_data_utils.py

@@ -53,15 +53,14 @@ def get_codes(
     dimension_code: str, structure: Schema, data: pd.DataFrame
 ) -> Tuple[List[str], List[str], List[Dict[str, Any]]]:
     """This function divides the components in Series and Obs."""
-    series_codes = []
     groups = structure.groups
     group_codes = []
     obs_codes = [dimension_code, structure.components.measures[0].id]
 
     # Getting the series and obs codes
-    for dim in structure.components.dimensions:
-        if dim.id != dimension_code:
-            series_codes.append(dim.id)
+    series_codes = [
+        d.id for d in structure.components.dimensions if d.id != dimension_code
+    ]
 
     # Adding the attributes based on the attachment level
     for att in structure.components.attributes: