eba-xbridge 1.5.0rc2__tar.gz → 1.5.0rc3__tar.gz

{eba_xbridge-1.5.0rc2 → eba_xbridge-1.5.0rc3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: eba-xbridge
-Version: 1.5.0rc2
+Version: 1.5.0rc3
 Summary: XBRL-XML to XBRL-CSV converter for EBA Taxonomy (version 4.1)
 License: Apache 2.0
 License-File: LICENSE

{eba_xbridge-1.5.0rc2 → eba_xbridge-1.5.0rc3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eba-xbridge"
-version = "1.5.0rc2"
+version = "1.5.0rc3"
 description = "XBRL-XML to XBRL-CSV converter for EBA Taxonomy (version 4.1)"
 license = 'Apache 2.0'
 readme = "README.rst"

{eba_xbridge-1.5.0rc2 → eba_xbridge-1.5.0rc3}/src/xbridge/__init__.py

@@ -2,4 +2,4 @@
 Init file for eba-xbridge library
 """
 
-__version__ = "1.5.0rc2"
+__version__ = "1.5.0rc3"

{eba_xbridge-1.5.0rc2 → eba_xbridge-1.5.0rc3}/src/xbridge/api.py

@@ -14,6 +14,7 @@ def convert_instance(
     output_path: Optional[Union[str, Path]] = None,
     headers_as_datapoints: bool = False,
     validate_filing_indicators: bool = True,
+    strict_validation: bool = True,
 ) -> Path:
     """
     Convert one single instance of XBRL-XML file to a CSV file
@@ -27,6 +28,9 @@ def convert_instance(
     :param validate_filing_indicators: If True, validate that no facts are orphaned
         (belong only to non-reported tables). Default is True.
 
+    :param strict_validation: If True (default), raise an error on orphaned facts. If False,
+        emit a warning instead and continue.
+
     :return: Converted CSV file.
 
     """
@@ -34,7 +38,12 @@ def convert_instance(
         output_path = Path(".")
 
     converter = Converter(instance_path)
-    return converter.convert(output_path, headers_as_datapoints, validate_filing_indicators)
+    return converter.convert(
+        output_path,
+        headers_as_datapoints,
+        validate_filing_indicators,
+        strict_validation,
+    )
 
 
 def load_instance(instance_path: Union[str, Path]) -> Instance:

{eba_xbridge-1.5.0rc2 → eba_xbridge-1.5.0rc3}/src/xbridge/converter.py

@@ -6,10 +6,11 @@ from __future__ import annotations
 
 import csv
 import json
+import warnings
 from pathlib import Path
 from shutil import rmtree
 from tempfile import TemporaryDirectory
-from typing import Any, Dict, Set, Union
+from typing import Any, Dict, Union
 from zipfile import ZipFile
 
 import pandas as pd
@@ -76,6 +77,7 @@ class Converter:
         output_path: Union[str, Path],
         headers_as_datapoints: bool = False,
         validate_filing_indicators: bool = True,
+        strict_validation: bool = True,
     ) -> Path:
         """Convert the ``XML Instance`` to a CSV file or between CSV formats"""
         if not output_path:
@@ -90,7 +92,9 @@ class Converter:
             raise ValueError("Module of the instance file not found in the taxonomy")
 
         if isinstance(self.instance, XmlInstance):
-            return self.convert_xml(output_path, headers_as_datapoints, validate_filing_indicators)
+            return self.convert_xml(
+                output_path, headers_as_datapoints, validate_filing_indicators, strict_validation
+            )
         elif isinstance(self.instance, CsvInstance):
             if self.module.architecture != "headers":
                 raise ValueError("Cannot convert CSV instance with non-headers architecture")
@@ -103,6 +107,7 @@ class Converter:
         output_path: Path,
         headers_as_datapoints: bool = False,
         validate_filing_indicators: bool = True,
+        strict_validation: bool = True,
     ) -> Path:
         module_filind_codes = [table.filing_indicator_code for table in self.module.tables]
 
@@ -147,7 +152,7 @@ class Converter:
         self._convert_filing_indicator(report_dir)
 
         if validate_filing_indicators:
-            self._validate_filing_indicators()
+            self._validate_filing_indicators(strict_validation=strict_validation)
 
         with open(MAPPING_PATH / self.module.dim_dom_file_name, "r", encoding="utf-8") as fl:
             mapping_dict: Dict[str, str] = json.load(fl)
@@ -280,111 +285,55 @@ class Converter:
             instance_df = instance_df.loc[mask]
             instance_df.drop(columns=nrd_list, inplace=True)
 
-        return instance_df
-
-    def _normalize_allowed_values(
-        self, table_df: pd.DataFrame, datapoint_df: pd.DataFrame
-    ) -> pd.DataFrame:
-        """
-        Normalizes fact values against allowed_values for each variable.
-
-        For variables with allowed_values:
-        1. Extracts code part from fact values (after ":")
-        2. Maps to correct namespaced value from allowed_values
-        3. Updates dimension columns with normalized values
-        4. Validates no unmatched codes remain
-
-        :param table_df: The merged dataframe with facts and variables
-        :param datapoint_df: The dataframe with variable definitions including allowed_values
-        :return: The normalized dataframe
-        """
-        if "allowed_values" not in datapoint_df.columns:
-            return table_df
+        # Rows missing values for required open keys do not belong to the table
+        if open_keys:
+            instance_df.dropna(subset=list(open_keys), inplace=True)
 
-        # Build mapping: datapoint → {code → full_value}
-        datapoint_allowed_map: Dict[str, Dict[str, str]] = {}
+        return instance_df
 
-        for _, row in datapoint_df.iterrows():
-            datapoint = row.get("datapoint")
-            allowed_values = row.get("allowed_values")
+    def _matching_fact_indices(self, table: Table) -> set[int]:
+        """Return indices of instance facts that actually match the table definition."""
+        if self.instance.instance_df is None:
+            return set()
 
-            if not datapoint or not allowed_values or len(allowed_values) == 0:
-                continue
+        instance_df = self._get_instance_df(table)
+        if instance_df.empty or table.variable_df is None:
+            return set()
 
-            # Group allowed values by the dimension they apply to
-            # For now, we'll apply them to all dimension columns
-            # In the future, we could make this more sophisticated
-            code_map: Dict[str, str] = {}
-            for allowed_val in allowed_values:
-                if ":" in allowed_val:
-                    code = allowed_val.split(":")[-1]
-                    code_map[code] = allowed_val
-
-            if code_map:
-                datapoint_allowed_map[datapoint] = code_map
-
-        if not datapoint_allowed_map:
-            return table_df
-
-        # Identify columns to normalize
-        # We normalize both dimension columns AND the value column (for enumerated values)
-        exclude_cols = {"datapoint", "decimals", "unit", "data_type", "allowed_values"}
-        columns_to_check = [col for col in table_df.columns if col not in exclude_cols]
-
-        # For each column that might contain namespaced values
-        for dim_col in columns_to_check:
-            if dim_col not in table_df.columns or table_df[dim_col].isna().all():
-                continue
+        open_keys = set(table.open_keys)
 
-            # Check if column contains namespaced values (contains ":")
-            sample_values = table_df[dim_col].dropna()
-            if sample_values.empty:
-                continue
+        datapoint_df = table.variable_df.copy()
 
-            has_namespace = sample_values.astype(str).str.contains(":", regex=False).any()
-            if not has_namespace:
-                continue
+        # For validation we match minimally on metric (concept) and any open keys present
+        merge_cols: list[str] = []
+        if "metric" in datapoint_df.columns and "metric" in instance_df.columns:
+            merge_cols.append("metric")
+        merge_cols.extend(
+            [key for key in open_keys if key in datapoint_df.columns and key in instance_df.columns]
+        )
 
-            # Extract codes from values (vectorized operation)
-            mask = table_df[dim_col].notna()
-            temp_code_col = f"_{dim_col}_temp_code"
-            table_df.loc[mask, temp_code_col] = (
-                table_df.loc[mask, dim_col].astype(str).str.split(":").str[-1]
-            )
+        def _strip_prefix(val: Any) -> Any:
+            if isinstance(val, str) and ":" in val:
+                return val.split(":", 1)[1]
+            return val
 
-            # Normalize values for each datapoint
-            for datapoint, code_map in datapoint_allowed_map.items():
-                dp_mask = (table_df["datapoint"] == datapoint) & mask
+        for col in merge_cols:
+            if col in datapoint_df.columns:
+                datapoint_df[col] = datapoint_df[col].map(_strip_prefix)
+            if col in instance_df.columns:
+                instance_df[col] = instance_df[col].map(_strip_prefix)
 
-                if not dp_mask.any():
-                    continue
+        instance_df = instance_df.copy()
+        instance_df["_idx"] = instance_df.index
 
-                # Store original values for error reporting
-                original_values = table_df.loc[dp_mask, dim_col].copy()
-
-                # Map codes to correct full values
-                normalized_values = table_df.loc[dp_mask, temp_code_col].map(code_map)
-
-                # Update only the values that were successfully mapped
-                mapped_mask = dp_mask & normalized_values.notna()
-                table_df.loc[mapped_mask, dim_col] = normalized_values[mapped_mask]
-
-                # Check for values that couldn't be mapped (validation errors)
-                unmapped_mask = dp_mask & normalized_values.isna()
-                if unmapped_mask.any():
-                    invalid_codes = table_df.loc[unmapped_mask, temp_code_col].unique()
-                    valid_codes = list(code_map.keys())
-                    raise ValueError(
-                        f"Invalid values for datapoint '{datapoint}' in column '{dim_col}': "
-                        f"Found codes {list(invalid_codes)} but only {valid_codes} are allowed. "
-                        f"Original values: {original_values[unmapped_mask].tolist()}"
-                    )
+        merged_df = pd.merge(datapoint_df, instance_df, on=merge_cols, how="inner")
 
-            # Clean up temporary column
-            if temp_code_col in table_df.columns:
-                table_df.drop(columns=[temp_code_col], inplace=True)
+        if open_keys:
+            valid_open_keys = [key for key in open_keys if key in merged_df.columns]
+            if valid_open_keys:
+                merged_df.dropna(subset=valid_open_keys, inplace=True)
 
-        return table_df
+        return set(merged_df["_idx"].tolist())
 
     def _variable_generator(self, table: Table) -> pd.DataFrame:
         """Returns the dataframe with the CSV file for the table
@@ -406,7 +355,7 @@ class Converter:
         )
 
         # Do the intersection and drop from datapoints the columns and records
-        datapoint_df = table.variable_df
+        datapoint_df = table.variable_df.copy()
         missing_cols = list(variable_columns - instance_columns)
         if "data_type" in missing_cols:
             missing_cols.remove("data_type")
@@ -417,10 +366,20 @@ class Converter:
 
         # Join the dataframes on the datapoint_columns
         merge_cols = list(variable_columns & instance_columns)
-        table_df = pd.merge(datapoint_df, instance_df, on=merge_cols, how="inner")
 
-        # Normalize values against allowed_values
-        table_df = self._normalize_allowed_values(table_df, datapoint_df)
+        def _strip_prefix(val: Any) -> Any:
+            if isinstance(val, str) and ":" in val:
+                return val.split(":", 1)[1]
+            return val
+
+        # Align merge columns by stripping any namespace prefixes from both sides
+        for col in merge_cols:
+            if col in datapoint_df.columns:
+                datapoint_df[col] = datapoint_df[col].map(_strip_prefix)
+            if col in instance_df.columns:
+                instance_df[col] = instance_df[col].map(_strip_prefix)
+
+        table_df = pd.merge(datapoint_df, instance_df, on=merge_cols, how="inner")
 
         if "data_type" in table_df.columns and "decimals" in table_df.columns:
             decimals_table = table_df[["decimals", "data_type"]].drop_duplicates()
@@ -432,17 +391,27 @@ class Converter:
                 decimals = row["decimals"]
 
                 if data_type not in self._decimals_parameters:
-                    self._decimals_parameters[data_type] = decimals
+                    self._decimals_parameters[data_type] = (
+                        int(decimals) if decimals not in {"INF", "#none"} else decimals
+                    )
                 else:
                     # If new value is a special value, skip it (prefer numeric values)
                     if decimals in {"INF", "#none"}:
                         pass
                     # If new value is numeric
                     else:
+                        try:
+                            decimals = int(decimals)
+                        except ValueError:
+                            raise ValueError(
+                                f"Invalid decimals value: {decimals}, "
+                                "should be integer, 'INF' or '#none'"
+                            )
+
                         # If existing value is special, replace with numeric
-                        if self._decimals_parameters[data_type] in {"INF", "#none"} or (
-                            isinstance(self._decimals_parameters[data_type], int)
-                            and decimals < self._decimals_parameters[data_type]
+                        if (
+                            self._decimals_parameters[data_type] in {"INF", "#none"}
+                            or decimals < self._decimals_parameters[data_type]
                         ):
                             self._decimals_parameters[data_type] = decimals
 
@@ -497,13 +466,6 @@ class Converter:
             # Defined by the EBA in the JSON files. We take them from the taxonomy
             # Because EBA is using exactly those for the JSON files.
 
-            for open_key in table.open_keys:
-                if open_key in datapoints.columns:
-                    dim_name = mapping_dict.get(open_key)
-                    # For open keys, there are no dim_names (they are not mapped)
-                    if dim_name and not datapoints.empty:
-                        datapoints[open_key] = dim_name + ":" + datapoints[open_key].astype(str)
-
             datapoints.sort_values(by=["datapoint"], ascending=True, inplace=True)
             output_path_table = temp_dir_path / (table.url or "table.csv")
 
@@ -550,7 +512,7 @@ class Converter:
                 if fil_ind.value and fil_ind.table:
                     self._reported_tables.append(fil_ind.table)
 
-    def _validate_filing_indicators(self) -> None:
+    def _validate_filing_indicators(self, strict_validation: bool = True) -> None:
         """Validate that no facts are orphaned (belong only to non-reported tables).
 
         Raises:
@@ -559,44 +521,56 @@ class Converter:
         if self.instance.instance_df is None or self.instance.instance_df.empty:
             return
 
-        # Step 1: Collect indices of facts that belong to ANY reported table
-        reported_fact_indices: Set[int] = set()
+        # Step 1: Track which facts belong to ANY reported table without materializing a huge set
+        reported_mask = pd.Series(False, index=self.instance.instance_df.index)
         for table in self.module.tables:
             if table.filing_indicator_code in self._reported_tables:
-                instance_df = self._get_instance_df(table)
-                if not instance_df.empty:
-                    # Add all fact indices (DataFrame row indices) to the set
-                    reported_fact_indices.update(instance_df.index)
+                reported_indices = self._matching_fact_indices(table)
+                if reported_indices:
+                    reported_mask.loc[list(reported_indices)] = True
 
         # Step 2: Find facts that belong ONLY to non-reported tables
-        all_orphaned_indices = set()
+        orphaned_mask = pd.Series(False, index=self.instance.instance_df.index)
         orphaned_per_table = {}
 
         for table in self.module.tables:
             if table.filing_indicator_code not in self._reported_tables:
-                instance_df = self._get_instance_df(table)
-                if not instance_df.empty:
-                    # Find facts that are in this table but NOT in any reported table
-                    orphaned_in_this_table = set(instance_df.index) - reported_fact_indices
+                orphaned_indices = self._matching_fact_indices(table)
+                if orphaned_indices:
+                    # Facts in this table that never appear in a reported table
+                    orphaned_in_this_table = [
+                        idx for idx in orphaned_indices if not reported_mask.loc[idx]
+                    ]
                     if orphaned_in_this_table:
+                        orphaned_mask.loc[orphaned_in_this_table] = True
                         orphaned_per_table[table.filing_indicator_code] = len(
                             orphaned_in_this_table
                         )
-                        all_orphaned_indices.update(orphaned_in_this_table)
 
-        if all_orphaned_indices:
+        total_orphaned = int(orphaned_mask.sum())
+
+        if total_orphaned:
             error_msg = (
                 f"Filing indicator inconsistency detected:\n"
-                f"Found {len(all_orphaned_indices)} fact(s) that belong ONLY"
+                f"Found {total_orphaned} fact(s) that belong ONLY"
                 f" to non-reported tables:\n"
             )
             for table_code, count in orphaned_per_table.items():
                 error_msg += f"  - {table_code}: {count} fact(s)\n"
+
+            if strict_validation:
+                error_msg += (
+                    "\nThe conversion process will not continue due to strict validation mode. "
+                    "Either set filed=true for the relevant tables "
+                    "or remove these facts from the XML."
+                )
+                raise ValueError(error_msg)
             error_msg += (
                 "\nThese facts will be excluded from the output. "
-                "Either set filed=true for the relevant tables or remove these facts from the XML."
+                "Consider setting filed=true for the relevant tables "
+                "or removing these facts from the XML."
             )
-            raise ValueError(error_msg)
+            warnings.warn(error_msg)
 
     def _convert_parameters(self, temp_dir_path: Path) -> None:
         # Workaround;

{eba_xbridge-1.5.0rc2 → eba_xbridge-1.5.0rc3}/src/xbridge/instance.py

@@ -13,6 +13,59 @@ from zipfile import ZipFile
 import pandas as pd
 from lxml import etree
 
+# Cache namespace → CSV prefix derivations to avoid repeated string work during parse
+_namespace_prefix_cache: Dict[str, str] = {}
+
+
+def _derive_csv_prefix(namespace_uri: str) -> Optional[str]:
+    """Derive the fixed CSV prefix from a namespace URI using the EBA convention."""
+    if not namespace_uri:
+        return None
+
+    cached = _namespace_prefix_cache.get(namespace_uri)
+    if cached is not None:
+        return cached
+
+    cleaned = namespace_uri.rstrip("#/")
+    if "#" in namespace_uri:
+        segment = namespace_uri.rsplit("#", 1)[-1]
+    else:
+        segment = cleaned.rsplit("/", 1)[-1] if "/" in cleaned else cleaned
+
+    if not segment:
+        return None
+
+    prefix = f"eba_{segment}"
+    _namespace_prefix_cache[namespace_uri] = prefix
+    return prefix
+
+
+def _normalize_namespaced_value(
+    value: Optional[str], nsmap: Dict[Optional[str], str]
+) -> Optional[str]:
+    """
+    Normalize a namespaced value (e.g., 'dom:qAE' or '{uri}qAE') to the CSV prefix convention.
+    Returns the original value if no namespace can be resolved.
+    """
+    if value is None:
+        return None
+
+    # Clark notation: {uri}local
+    if value.startswith("{") and "}" in value:
+        uri, local = value[1:].split("}", 1)
+        derived = _derive_csv_prefix(uri)
+        return f"{derived}:{local}" if derived else value
+
+    # Prefixed notation: prefix:local
+    if ":" in value:
+        potential_prefix, local = value.split(":", 1)
+        namespace_uri = nsmap.get(potential_prefix)
+        if namespace_uri:
+            derived = _derive_csv_prefix(namespace_uri)
+            return f"{derived}:{local}" if derived else value
+
+    return value
+
 
 class Instance:
     """
@@ -548,7 +601,7 @@ class Scenario:
                     continue
                 dimension = dimension_raw.split(":")[1]
                 value = self.get_value(child)
-                value = value.split(":")[1] if ":" in value else value
+                value = _normalize_namespaced_value(value, child.nsmap) or ""
                 self.dimensions[dimension] = value
 
     @staticmethod
@@ -667,7 +720,7 @@ class Fact:
     def parse(self) -> None:
         """Parse the XML node with the `fact <https://www.xbrl.org/guidance/xbrl-glossary/#:~:text=accounting%20standards%20body.-,Fact,-A%20fact%20is>`_."""
         self.metric = self.fact_xml.tag
-        self.value = self.fact_xml.text
+        self.value = _normalize_namespaced_value(self.fact_xml.text, self.fact_xml.nsmap)
         self.decimals = self.fact_xml.attrib.get("decimals")
         self.context = self.fact_xml.attrib.get("contextRef")
         self.unit = self.fact_xml.attrib.get("unitRef")