eba-xbridge 1.5.0rc4__tar.gz → 1.5.0rc6__tar.gz

{eba_xbridge-1.5.0rc4 → eba_xbridge-1.5.0rc6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eba-xbridge
-Version: 1.5.0rc4
+Version: 1.5.0rc6
 Summary: XBRL-XML to XBRL-CSV converter for EBA Taxonomy (version 4.1)
 License: Apache 2.0
 License-File: LICENSE
@@ -154,6 +154,55 @@ Customize the conversion with additional parameters:
         strict_validation=False,  # Emit warnings instead of errors for orphaned facts
     )
 
+Python API - Handling Warnings
+------------------------------
+
+XBridge emits structured warnings that can be filtered or turned into errors from your code.
+The most common ones are:
+
+* ``IdentifierPrefixWarning``: Unknown entity identifier prefix; XBridge falls back to ``rs``.
+* ``FilingIndicatorWarning``: Filing indicator inconsistencies; some facts are excluded.
+
+To capture these warnings when using ``convert_instance``:
+
+.. code-block:: python
+
+    import warnings
+    from xbridge.api import convert_instance
+    from xbridge.exceptions import XbridgeWarning, FilingIndicatorWarning
+
+    input_path = "path/to/instance.xbrl"
+    output_path = "path/to/output"
+
+    with warnings.catch_warnings(record=True) as caught:
+        # Ensure all xbridge warnings are captured
+        warnings.simplefilter("always", XbridgeWarning)
+
+        zip_path = convert_instance(
+            instance_path=input_path,
+            output_path=output_path,
+            validate_filing_indicators=True,
+            strict_validation=False,  # Warnings instead of errors for orphaned facts
+        )
+
+    filing_warnings = [
+        w for w in caught if issubclass(w.category, FilingIndicatorWarning)
+    ]
+    for w in filing_warnings:
+        print(f"Filing indicator warning: {w.message}")
+
+To treat all XBridge warnings as errors:
+
+.. code-block:: python
+
+    import warnings
+    from xbridge.api import convert_instance
+    from xbridge.exceptions import XbridgeWarning
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", XbridgeWarning)
+        convert_instance("path/to/instance.xbrl", "path/to/output")
+
 Loading an Instance
 -------------------
 

{eba_xbridge-1.5.0rc4 → eba_xbridge-1.5.0rc6}/README.rst

@@ -126,6 +126,55 @@ Customize the conversion with additional parameters:
         strict_validation=False,  # Emit warnings instead of errors for orphaned facts
     )
 
+Python API - Handling Warnings
+------------------------------
+
+XBridge emits structured warnings that can be filtered or turned into errors from your code.
+The most common ones are:
+
+* ``IdentifierPrefixWarning``: Unknown entity identifier prefix; XBridge falls back to ``rs``.
+* ``FilingIndicatorWarning``: Filing indicator inconsistencies; some facts are excluded.
+
+To capture these warnings when using ``convert_instance``:
+
+.. code-block:: python
+
+    import warnings
+    from xbridge.api import convert_instance
+    from xbridge.exceptions import XbridgeWarning, FilingIndicatorWarning
+
+    input_path = "path/to/instance.xbrl"
+    output_path = "path/to/output"
+
+    with warnings.catch_warnings(record=True) as caught:
+        # Ensure all xbridge warnings are captured
+        warnings.simplefilter("always", XbridgeWarning)
+
+        zip_path = convert_instance(
+            instance_path=input_path,
+            output_path=output_path,
+            validate_filing_indicators=True,
+            strict_validation=False,  # Warnings instead of errors for orphaned facts
+        )
+
+    filing_warnings = [
+        w for w in caught if issubclass(w.category, FilingIndicatorWarning)
+    ]
+    for w in filing_warnings:
+        print(f"Filing indicator warning: {w.message}")
+
+To treat all XBridge warnings as errors:
+
+.. code-block:: python
+
+    import warnings
+    from xbridge.api import convert_instance
+    from xbridge.exceptions import XbridgeWarning
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", XbridgeWarning)
+        convert_instance("path/to/instance.xbrl", "path/to/output")
+
 Loading an Instance
 -------------------
 

{eba_xbridge-1.5.0rc4 → eba_xbridge-1.5.0rc6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eba-xbridge"
-version = "1.5.0rc4"
+version = "1.5.0rc6"
 description = "XBRL-XML to XBRL-CSV converter for EBA Taxonomy (version 4.1)"
 license = 'Apache 2.0'
 readme = "README.rst"

{eba_xbridge-1.5.0rc4 → eba_xbridge-1.5.0rc6}/src/xbridge/__init__.py

@@ -2,4 +2,4 @@
 Init file for eba-xbridge library
 """
 
-__version__ = "1.5.0rc4"
+__version__ = "1.5.0rc6"

{eba_xbridge-1.5.0rc4 → eba_xbridge-1.5.0rc6}/src/xbridge/converter.py

@@ -15,6 +15,7 @@ from zipfile import ZipFile
 
 import pandas as pd
 
+from xbridge.exceptions import DecimalValueError, FilingIndicatorWarning
 from xbridge.instance import CsvInstance, Instance, XmlInstance
 from xbridge.modules import Module, Table
 
@@ -300,17 +301,27 @@ class Converter:
         if instance_df.empty or table.variable_df is None:
             return set()
 
+        variable_columns = set(table.variable_columns or [])
         open_keys = set(table.open_keys)
 
+        # self.instance.instance_df is guaranteed to not be None due to check at line 296
+        instance_columns = set(self.instance.instance_df.columns)
+
         datapoint_df = table.variable_df.copy()
 
-        # For validation we match minimally on metric (concept) and any open keys present
-        merge_cols: list[str] = []
-        if "metric" in datapoint_df.columns and "metric" in instance_df.columns:
-            merge_cols.append("metric")
-        merge_cols.extend(
-            [key for key in open_keys if key in datapoint_df.columns and key in instance_df.columns]
-        )
+        # Handle missing columns by filtering datapoint_df (same as _variable_generator)
+        # This prevents requiring dimensions that don't exist in the instance
+        missing_cols = list(variable_columns - instance_columns)
+        if "data_type" in missing_cols:
+            missing_cols.remove("data_type")
+        if missing_cols:
+            mask = datapoint_df[missing_cols].isnull().all(axis=1)
+            datapoint_df = datapoint_df.loc[mask]
+            datapoint_df = datapoint_df.drop(columns=missing_cols)
+
+        # Match on all variable columns (dimensions) to avoid Cartesian product
+        # explosion. Consistent with _variable_generator() to prevent OOM.
+        merge_cols = list(variable_columns & instance_columns)
 
         instance_df = instance_df.copy()
         instance_df["_idx"] = instance_df.index
@@ -366,31 +377,23 @@ class Converter:
 
                 data_type = row["data_type"][1:]
                 decimals = row["decimals"]
+                normalized_decimals = self._normalize_decimals_value(decimals)
 
                 if data_type not in self._decimals_parameters:
-                    self._decimals_parameters[data_type] = (
-                        int(decimals) if decimals not in {"INF", "#none"} else decimals
-                    )
+                    self._decimals_parameters[data_type] = normalized_decimals
                 else:
-                    # If new value is a special value, skip it (prefer numeric values)
-                    if decimals in {"INF", "#none"}:
-                        pass
-                    # If new value is numeric
-                    else:
-                        try:
-                            decimals = int(decimals)
-                        except ValueError:
-                            raise ValueError(
-                                f"Invalid decimals value: {decimals}, "
-                                "should be integer, 'INF' or '#none'"
-                            )
-
-                        # If existing value is special, replace with numeric
-                        if (
-                            self._decimals_parameters[data_type] in {"INF", "#none"}
-                            or decimals < self._decimals_parameters[data_type]
-                        ):
-                            self._decimals_parameters[data_type] = decimals
+                    # Skip special values when we already have an entry,
+                    # as numeric values take precedence.
+                    if normalized_decimals in {"INF", "#none"}:
+                        continue
+
+                    existing_value = self._decimals_parameters[data_type]
+                    if existing_value in {"INF", "#none"} or (
+                        isinstance(existing_value, int)
+                        and isinstance(normalized_decimals, int)
+                        and normalized_decimals < existing_value
+                    ):
+                        self._decimals_parameters[data_type] = normalized_decimals
 
             drop_columns = merge_cols + ["data_type", "decimals"]
         else:
@@ -412,6 +415,23 @@ class Converter:
 
         return table_df
 
+    def _normalize_decimals_value(self, decimals: Any) -> Union[int, str]:
+        """Return a validated decimals value or raise a DecimalValueError."""
+        candidate = decimals
+        if isinstance(candidate, str):
+            candidate = candidate.strip()
+
+        if candidate in {"INF", "#none"}:
+            return candidate
+
+        try:
+            return int(candidate)
+        except (TypeError, ValueError) as exc:
+            raise DecimalValueError(
+                f"Invalid decimals value: {decimals}, should be integer, 'INF' or '#none'",
+                offending_value=decimals,
+            ) from exc
+
     def _convert_tables(
         self,
         temp_dir_path: Path,
@@ -547,7 +567,11 @@ class Converter:
                 "Consider setting filed=true for the relevant tables "
                 "or removing these facts from the XML."
             )
-            warnings.warn(error_msg)
+            warnings.warn(
+                error_msg,
+                category=FilingIndicatorWarning,
+                stacklevel=2,
+            )
 
     def _convert_parameters(self, temp_dir_path: Path) -> None:
         # Workaround;

eba_xbridge-1.5.0rc6/src/xbridge/exceptions.py

@@ -0,0 +1,33 @@
+"""Custom exception types for the xbridge package."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+class SchemaRefValueError(ValueError):
+    """Raised when schemaRef validation fails in an XBRL instance."""
+
+    def __init__(self, error_message: str, offending_value: Optional[Any] = None) -> None:
+        super().__init__(error_message)
+        self.offending_value = offending_value
+
+
+class DecimalValueError(ValueError):
+    """Raised when decimals metadata contains unsupported values."""
+
+    def __init__(self, error_message: str, offending_value: Optional[Any] = None) -> None:
+        super().__init__(error_message)
+        self.offending_value = offending_value
+
+
+class XbridgeWarning(Warning):
+    """Base warning for the xbridge library."""
+
+
+class IdentifierPrefixWarning(XbridgeWarning):
+    """Unknown identifier prefix; defaulting to 'rs'."""
+
+
+class FilingIndicatorWarning(XbridgeWarning):
+    """Facts orphaned by filing indicators; some are excluded."""

{eba_xbridge-1.5.0rc4 → eba_xbridge-1.5.0rc6}/src/xbridge/instance.py

@@ -13,6 +13,8 @@ from zipfile import ZipFile
 import pandas as pd
 from lxml import etree
 
+from xbridge.exceptions import IdentifierPrefixWarning, SchemaRefValueError
+
 # Cache namespace → CSV prefix derivations to avoid repeated string work during parse
 _namespace_prefix_cache: Dict[str, str] = {}
 
@@ -40,6 +42,49 @@ def _derive_csv_prefix(namespace_uri: str) -> Optional[str]:
     return prefix
 
 
+def _derive_metric_prefix(namespace_uri: str) -> Optional[str]:
+    """
+    Derive the CSV prefix for metrics from a namespace URI.
+
+    For metrics, we preserve version suffixes in the prefix:
+    - http://www.eba.europa.eu/xbrl/crr/dict/met -> eba_met
+    - http://www.eba.europa.eu/xbrl/crr/dict/met/3.5 -> eba_met_3.5
+    - http://www.eba.europa.eu/xbrl/crr/dict/met/4.0 -> eba_met_4.0
+    """
+    if not namespace_uri:
+        return None
+
+    cached = _namespace_prefix_cache.get(f"metric:{namespace_uri}")
+    if cached is not None:
+        return cached
+
+    cleaned = namespace_uri.rstrip("#/")
+
+    # Split the URI into path segments
+    segments = cleaned.split("/")
+
+    # Find the 'met' (metrics) segment and check if there's a version after it
+    prefix = None
+    for i, segment in enumerate(segments):
+        if segment == "met":
+            # Check if there's a version suffix (e.g., "3.5", "4.0")
+            if i + 1 < len(segments):
+                version = segments[i + 1]
+                prefix = f"eba_met_{version}"
+            else:
+                prefix = "eba_met"
+            break
+
+    # If we didn't find 'met', fall back to the standard logic
+    if prefix is None:
+        prefix = _derive_csv_prefix(namespace_uri)
+
+    if prefix:
+        _namespace_prefix_cache[f"metric:{namespace_uri}"] = prefix
+
+    return prefix
+
+
 def _normalize_namespaced_value(
     value: Optional[str], nsmap: Dict[Optional[str], str]
 ) -> Optional[str]:
@@ -67,6 +112,32 @@ def _normalize_namespaced_value(
     return value
 
 
+def _normalize_metric_value(value: Optional[str], nsmap: Dict[Optional[str], str]) -> Optional[str]:
+    """
+    Normalize a metric namespaced value to the CSV prefix convention.
+    For metrics, we preserve version suffixes (e.g., eba_met_3.5, eba_met_4.0).
+    Returns the original value if no namespace can be resolved.
+    """
+    if value is None:
+        return None
+
+    # Clark notation: {uri}local
+    if value.startswith("{") and "}" in value:
+        uri, local = value[1:].split("}", 1)
+        derived = _derive_metric_prefix(uri)
+        return f"{derived}:{local}" if derived else value
+
+    # Prefixed notation: prefix:local
+    if ":" in value:
+        potential_prefix, local = value.split(":", 1)
+        namespace_uri = nsmap.get(potential_prefix)
+        if namespace_uri:
+            derived = _derive_metric_prefix(namespace_uri)
+            return f"{derived}:{local}" if derived else value
+
+    return value
+
+
 class Instance:
     """
     Abstract class representing an XBRL instance file.
@@ -110,6 +181,7 @@ class Instance:
         self._contexts: Optional[Dict[str, Context]] = None
         self._facts: Optional[List[Fact]] = None
         self._facts_list_dict: Optional[List[Dict[str, Any]]] = None
+        self._df: Optional[pd.DataFrame] = None
         self._table_files: Optional[set[Path]] = None
         self._root_folder: Optional[str] = None
         self._report_file: Optional[Path] = None
@@ -199,7 +271,7 @@ class Instance:
         """
         if self.facts_list_dict is None:
             return
-        df = pd.DataFrame.from_dict(self.facts_list_dict)  # type: ignore[call-overload]
+        df = pd.DataFrame(self.facts_list_dict)
         df_columns = list(df.columns)
         ##Workaround
         # Dropping period an entity columns because in current EBA architecture,
@@ -225,7 +297,9 @@ class Instance:
                 (
                     f"{self._identifier_prefix} is not a known identifier prefix. "
                     "Default 'rs' will be used."
-                )
+                ),
+                category=IdentifierPrefixWarning,
+                stacklevel=2,
             )
             return "rs"
 
@@ -283,16 +357,17 @@ class Instance:
             raise AttributeError("XML root not loaded.")
 
         contexts: Dict[str, Context] = {}
+        namespaces: Dict[str, str] = {key or "": value for key, value in self.namespaces.items()}
         for context in self.root.findall(
             "{http://www.xbrl.org/2003/instance}context",
-            self.namespaces,  # type: ignore[arg-type]
+            namespaces,
         ):
             context_object = Context(context)
             contexts[context_object.id] = context_object
 
         self._contexts = contexts
 
-        first_ctx = self.root.find("{http://www.xbrl.org/2003/instance}context", self.namespaces)  # type: ignore[arg-type]
+        first_ctx = self.root.find("{http://www.xbrl.org/2003/instance}context", namespaces)
         if first_ctx is not None:
             entity_elem = first_ctx.find("{http://www.xbrl.org/2003/instance}entity")
             if entity_elem is not None:
@@ -335,7 +410,8 @@ class Instance:
                 href_attr = "{http://www.w3.org/1999/xlink}href"
                 if href_attr not in child.attrib:
                     continue
-                value: str = child.attrib[href_attr]  # type: ignore[assignment]
+                raw_value = child.attrib[href_attr]
+                value = str(raw_value)
                 schema_refs.append(value)
 
         # Validate that only one schemaRef exists
@@ -343,11 +419,14 @@ class Instance:
             return  # No schema reference found, module_ref will remain None
 
         if len(schema_refs) > 1:
-            raise ValueError(
-                f"Multiple schemaRef elements found in the XBRL instance. "
-                f"Only one schemaRef is expected, but {len(schema_refs)} "
-                f"were found: {schema_refs}. "
-                f"This may indicate an invalid XBRL-XML file."
+            raise SchemaRefValueError(
+                (
+                    "Multiple schemaRef elements found in the XBRL instance. "
+                    f"Only one schemaRef is expected, but {len(schema_refs)} "
+                    f"were found: {schema_refs}. "
+                    "This may indicate an invalid XBRL-XML file."
+                ),
+                offending_value=schema_refs,
             )
 
         # Process the single schema reference
@@ -356,25 +435,34 @@ class Instance:
 
         # Validate href format and extract module code
         if "/mod/" not in value:
-            raise ValueError(
-                f"Invalid href format in schemaRef. Expected href to contain '/mod/' "
-                f"but got: '{value}'. Please verify the XBRL-XML file contains a "
-                f"valid schema reference."
+            raise SchemaRefValueError(
+                (
+                    "Invalid href format in schemaRef. Expected href to contain '/mod/' "
+                    f"but got: '{value}'. Please verify the XBRL-XML file contains a "
+                    "valid schema reference."
+                ),
+                offending_value=value,
             )
 
         split_parts = value.split("/mod/")
         if len(split_parts) < 2:
-            raise ValueError(
-                f"Invalid href format in schemaRef. Could not extract module name from: '{value}'. "
-                f"Expected format: 'http://.../mod/[module_name].xsd'"
+            raise SchemaRefValueError(
+                (
+                    "Invalid href format in schemaRef. Could not extract module name "
+                    f"from: '{value}'. Expected format: 'http://.../mod/[module_name].xsd'"
+                ),
+                offending_value=value,
             )
 
         module_part = split_parts[1]
         if ".xsd" not in module_part:
-            raise ValueError(
-                f"Invalid href format in schemaRef. Expected href to end with '.xsd' "
-                f"but got: '{value}'. Please verify the XBRL-XML file contains a valid "
-                f"schema reference."
+            raise SchemaRefValueError(
+                (
+                    "Invalid href format in schemaRef. Expected href to end with '.xsd' "
+                    f"but got: '{value}'. Please verify the XBRL-XML file contains a valid "
+                    "schema reference."
+                ),
+                offending_value=value,
             )
 
         xsd_split = module_part.split(".xsd")
@@ -414,7 +502,8 @@ class Instance:
 
         units: Dict[str, str] = {}
         for unit in self.root.findall("{http://www.xbrl.org/2003/instance}unit"):
-            unit_name: str = unit.attrib["id"]  # type: ignore[assignment]
+            unit_id = unit.attrib["id"]
+            unit_name = str(unit_id)
             measure = unit.find("{http://www.xbrl.org/2003/instance}measure")
             if measure is None or measure.text is None:
                 continue
@@ -668,7 +757,7 @@ class Context:
 
     def parse(self) -> None:
         """Parses the XML node with the :obj:`Context <xbridge.xml_instance.Context>`."""
-        self._id = self.context_xml.attrib["id"]  # type: ignore[assignment]
+        self._id = str(self.context_xml.attrib["id"])
 
         entity_elem = self.context_xml.find("{http://www.xbrl.org/2003/instance}entity")
         if entity_elem is not None:
@@ -728,8 +817,8 @@ class Fact:
     def __dict__(self) -> Dict[str, Any]:  # type: ignore[override]
         metric_clean = ""
         if self.metric:
-            # Normalize metric to use consistent eba_* prefix like other dimensions
-            metric_clean = _normalize_namespaced_value(self.metric, self.fact_xml.nsmap) or ""
+            # Normalize metric using metric-specific logic that preserves version suffixes
+            metric_clean = _normalize_metric_value(self.metric, self.fact_xml.nsmap) or ""
             # If still in Clark notation, extract the local name
             if metric_clean.startswith("{") and "}" in metric_clean:
                 metric_clean = metric_clean.split("}", 1)[1]