levelapp 0.1.15__py3-none-any.whl

levelapp/comparator/comparator.py

@@ -0,0 +1,232 @@
+"""'comparator/service.py':"""
+from collections.abc import Mapping
+from typing import Any, Dict, List, Tuple, Literal
+
+from pydantic import BaseModel
+
+from levelapp.core.base import BaseProcess
+from levelapp.comparator.extractor import DataExtractor
+from levelapp.comparator.scorer import MetricsManager, ComparisonResults
+from levelapp.comparator.schemas import EntityMetric, SetMetric, MetricConfig
+from levelapp.comparator.utils import format_evaluation_results
+
+
+class MetadataComparator(BaseProcess):
+    """Metadata comparator component."""
+
+    def __init__(
+            self,
+            reference: BaseModel | None = None,
+            generated: BaseModel | None = None,
+            metrics_manager: MetricsManager | None = None,
+    ):
+        """
+        Initialize the MetadataComparator.
+
+        Args:
+            reference (BaseModel): Reference BaseModel
+            generated (BaseModel): Extracted BaseModel
+            metrics_manager (MetricsManager): MetricsManager
+        """
+        self.extractor = DataExtractor()
+
+        self._reference = reference
+        self._generated = generated
+        self._metrics_manager = metrics_manager
+
+        self._evaluation_data: List[
+            Tuple[str, list[str], list[str], Any, Any, Any, Any, float]
+        ] = []
+
+    @property
+    def reference_data(self) -> BaseModel:
+        return self._reference
+
+    @property
+    def generated_data(self) -> BaseModel:
+        return self._generated
+
+    @property
+    def metrics_manager(self) -> MetricsManager:
+        return self._metrics_manager
+
+    @reference_data.setter
+    def reference_data(self, value: BaseModel):
+        self._reference = value
+
+    @generated_data.setter
+    def generated_data(self, value: BaseModel):
+        self._generated = value
+
+    @metrics_manager.setter
+    def metrics_manager(self, value: MetricsManager):
+        self._metrics_manager = value
+
+    def _get_score(self, field: str) -> Tuple[EntityMetric, SetMetric, float]:
+        """
+        Retrieve the scoring metric and threshold for a given field.
+
+        Args:
+            field: The field for which to retrieve the metric and threshold.
+
+        Returns:
+            A tuple containing the scoring metric and its threshold.
+        """
+        if self._metrics_manager:
+            config = self._metrics_manager.get_metrics_config(field=field)
+        else:
+            config = MetricConfig()
+
+        return config.entity_metric, config.set_metric, config.threshold
+
+    def _format_results(
+            self,
+            output_type: Literal["json", "csv"] = "json"
+    ) -> Dict[int, Any]:
+        """
+        Format the internal evaluation data for reporting or storage.
+
+        Args:
+            output_type: 'json' returns a list of dictionaries; 'csv' returns a DataFrame.
+
+        Returns:
+            Formatted evaluation results or None if no data.
+        """
+        formatted_results = format_evaluation_results(self._evaluation_data, output_type=output_type)
+
+        return dict(enumerate(formatted_results))
+
+    def evaluate(
+            self,
+            reference_list: List[str],
+            extracted_list: List[str],
+            entity_metric: EntityMetric,
+            set_metric: SetMetric,
+            threshold: float,
+    ) -> ComparisonResults:
+        """
+        Evaluates pairwise similarity between elements in two lists using fuzzy matching.
+
+        Args:
+            reference_list: Ground-truth list of strings.
+            extracted_list: Extracted list of strings to compare.
+            entity_metric (EntityMetric): entity-level comparison metric.
+            set_metric (SetMetric): set-level comparison metric.
+            threshold: Similarity threshold (0–100) for considering a match.
+
+        Returns:
+            A dict with accuracy, precision, recall, and F1-score.
+        """
+        if not (reference_list or extracted_list):
+            return ComparisonResults("", "", entity_metric.value, None, set_metric.value, None)
+
+        scores = self._metrics_manager.compute_entity_scores(
+            reference_seq=reference_list,
+            extracted_seq=extracted_list,
+            scorer=entity_metric,
+            pairwise=False
+        )
+
+        return self._metrics_manager.compute_set_scores(
+            data=scores,
+            scorer=set_metric,
+            threshold=threshold,
+        )
+
+    def _recursive_compare(
+        self,
+        ref_node: Any,
+        ext_node: Any,
+        results: Dict[str, Dict[str, float]],
+        prefix: str = "",
+        threshold: float = 99.0,
+    ) -> None:
+        """
+        Recursively compare extracted vs. reference metadata nodes.
+
+        Args:
+            ref_node: dict or list (from deep_extract reference metadata)
+            ext_node: dict or list (from deep_extract extracted metadata)
+            results: Dict to accumulate comp_results keyed by hierarchical attribute paths.
+            prefix: str, current path prefix to form hierarchical keys.
+        """
+        # Case 1: Both nodes are dicts -> recurse on keys
+        if isinstance(ref_node, Mapping) and isinstance(ext_node, Mapping):
+            all_keys = set(ref_node.keys())
+            for key in all_keys:
+                new_prefix = f"{prefix}.{key}" if prefix else key
+                ref_subnode = ref_node.get(key, [])
+                ext_subnode = ext_node.get(key, [])
+                self._recursive_compare(
+                    ref_node=ref_subnode,
+                    ext_node=ext_subnode,
+                    results=results,
+                    prefix=new_prefix,
+                    threshold=threshold,
+                )
+
+        # Case 2: Leaf nodes (lists) -> evaluate directly
+        else:
+            # Defensive: convert to list if not list
+            ref_list = ref_node if isinstance(ref_node, list) else [ref_node]
+            ext_list = ext_node if isinstance(ext_node, list) else [ext_node]
+
+            # Convert all to strings for consistent fuzzy matching
+            ref_list_str = list(map(str, ref_list))
+            ext_list_str = list(map(str, ext_list))
+
+            entity_metric_, set_metric_, threshold = self._get_score(field=prefix)
+
+            # Evaluate similarity metrics
+            comp_results = self.evaluate(
+                reference_list=ref_list_str,
+                extracted_list=ext_list_str,
+                entity_metric=entity_metric_,
+                set_metric=set_metric_,
+                threshold=threshold,
+            )
+
+            if comp_results:
+                self._evaluation_data.append(
+                    (
+                        prefix,
+                        ref_list_str,
+                        ext_list_str,
+                        comp_results.e_metric,
+                        comp_results.e_score,
+                        comp_results.s_metric,
+                        comp_results.s_score,
+                        threshold,
+                    )
+                )
+
+            results[prefix] = comp_results or {"accuracy": 0}
+
+    def run(self, indexed_mode: bool = False) -> Dict[int, Any]:
+        """
+        Launch a metadata comparison process between reference and extracted data.
+
+        Args:
+            indexed_mode: Flag to use indexed mode for metadata extraction.
+
+        Returns:
+            Dictionary with comparison results, keyed by attribute paths.
+        """
+        self._evaluation_data.clear()
+
+        ref_data = self.extractor.deep_extract(model=self.reference_data, indexed=indexed_mode)
+        ext_data = self.extractor.deep_extract(model=self.generated_data, indexed=indexed_mode)
+
+        results: Dict[str, Dict[str, float]] = {}
+
+        self._recursive_compare(
+            ref_node=ref_data,
+            ext_node=ext_data,
+            results=results,
+            prefix="",
+            threshold=1,
+        )
+
+        formatted_results = self._format_results()
+
+        return formatted_results

levelapp/comparator/extractor.py

@@ -0,0 +1,108 @@
+"""levelapp/comparator/extractor.py"""
+
+from collections import defaultdict
+from collections.abc import Sequence
+from typing import List, Dict, Any
+from pydantic import BaseModel
+
+
+class DataExtractor:
+    """
+    Extracts primitive values from nested Pydantic models, dicts, and sequences.
+    """
+    def deep_extract(
+            self, model: BaseModel,
+            indexed: bool = False
+    ) -> Dict[str, List[str]]:
+        """
+        Extracts data in a recursive way from pydantic model.
+
+        Args:
+            model: An instance of a BaseModel.
+            indexed: Switch parameter to select the extraction approach.
+
+        Returns:
+            A dictionary where keys are attribute names and values are lists of string values.
+        """
+        result: Dict[str, List[str]] = defaultdict(list)
+        for field_name, field_info in type(model).model_fields.items():
+            field_value = getattr(model, field_name)
+            self._extract_field_values(
+                value=field_value, prefix=field_name, result=result, indexed=indexed
+            )
+
+        return result
+
+    def _extract_field_values(
+            self,
+            value: Any,
+            prefix: str,
+            result: Dict[str, List[str]],
+            indexed: bool = False,
+    ) -> None:
+        """
+        Recursively extract values from a field, storing them in result with field path as key.
+
+        Args:
+            value: The value to extract (BaseModel, dict, list, or primitive).
+            prefix: The current field path (e.g., 'documents.tribunal_members').
+            result: Dictionary to store field paths and their value lists.
+            indexed: Switch parameter to select the extraction approach.
+        """
+        if isinstance(value, BaseModel):
+            self._handle_model(model=value, prefix=prefix, result=result)
+
+        elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
+            self._handle_sequence(
+                sequence=value, prefix=prefix, result=result, indexed=indexed
+            )
+
+        else:
+            result[prefix].append(value)
+
+    def _handle_model(
+        self, model: BaseModel, prefix: str, result: Dict[str, List[str]]
+    ) -> None:
+        """
+        Extract values from a Pydantic model recursively.
+
+        Args:
+            model: Pydantic BaseModel instance.
+            prefix: Current field path.
+            result: Dictionary to store field paths and value lists.
+        """
+        for field_name, field_info in type(model).model_fields.items():
+            field_value = getattr(model, field_name)
+            new_prefix = f"{prefix}.{field_name}" if prefix else field_name
+            self._extract_field_values(
+                value=field_value, prefix=new_prefix, result=result
+            )
+
+    def _handle_sequence(
+        self,
+        sequence: Sequence,
+        prefix: str,
+        result: Dict[str, List[str]],
+        indexed: bool = False,
+    ) -> None:
+        """
+        Extract values from a sequence (list or tuple) recursively.
+
+        Args:
+            sequence: List or tuple of values.
+            prefix: Current field path.
+            result: Dictionary to store field paths and value lists.
+            indexed: Switch parameter to select the extraction approach.
+        """
+        if not sequence:
+            result[prefix] = []
+
+        if indexed:
+            for i, item in enumerate(sequence):
+                new_prefix = f"{prefix}[{i}]" if prefix else f"[{i}]"
+                self._extract_field_values(value=item, prefix=new_prefix, result=result)
+        else:
+            for i, item in enumerate(sequence):
+                self._extract_field_values(
+                    value=item, prefix=prefix, result=result, indexed=indexed
+                )

levelapp/comparator/schemas.py

@@ -0,0 +1,61 @@
+"""'comparator/schemas.py': Defines Pydantic models for extracted metadata."""
+
+from enum import Enum
+
+from pydantic import BaseModel, Field
+from rapidfuzz import fuzz, utils
+
+
+class AttrCompMixin:
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, type(self)):
+            return False
+
+        attr_name = next(iter(self.__dict__.keys()))
+        _cond = (
+            fuzz.ratio(
+                s1=getattr(self, attr_name),
+                s2=getattr(other, attr_name),
+                processor=utils.default_process,
+            )
+            > 99
+        )
+        return _cond
+
+
+class CompScoreMixin:
+    def comp_score(self, other) -> float:
+        attr_name = next(iter(self.__dict__.keys()))
+        _score = fuzz.ratio(
+            s1=getattr(self, attr_name),
+            s2=getattr(other, attr_name),
+            processor=utils.default_process,
+        )
+        return _score
+
+
+class EntityMetric(str, Enum):
+    WRATIO = "wratio"
+    LEV_NORM = "lev-norm"
+    JARO_WINKLER = "jaro-winkler"
+    TOKEN_SORT_RATIO = "token-sort-ratio"
+    TOKEN_SET_RATIO = "token-set-ratio"
+
+    @classmethod
+    def list(cls):
+        return [field.value for field in cls]
+
+
+class SetMetric(str, Enum):
+    ACCURACY = "accuracy"
+    F1_SCORE = "f1-score"
+
+
+class MetricConfig(BaseModel):
+    """
+    Configuration for a field's comparison metric.
+    """
+    field_name: str = Field(default="token-set-ratio", description="Name of the field")
+    entity_metric: EntityMetric = Field(default=EntityMetric.LEV_NORM, description="Entity level metric")
+    set_metric: SetMetric = Field(default=SetMetric.ACCURACY, description="Set level metric")
+    threshold: float = Field(default=50, ge=0, le=100, description="Match threshold")

levelapp/comparator/scorer.py

@@ -0,0 +1,269 @@
+"""'comparator/scorer.py':"""
+import numpy as np
+
+from collections import namedtuple
+from typing import List, Dict, Callable, cast, Protocol
+
+from rapidfuzz import distance, process, utils, fuzz
+
+from levelapp.comparator.schemas import MetricConfig, EntityMetric, SetMetric
+from levelapp.aspects import logger
+
+ComputedScores = namedtuple(
+    typename="ComputedScores",
+    field_names=["ref", "ext", "e_metric", "e_score"],
+)
+ComparisonResults = namedtuple(
+    typename="ComparisonResults",
+    field_names=["ref", "ext", "e_metric", "e_score", "s_metric", "s_score"]
+)
+
+
+class Scorer(Protocol):
+    def __call__(self, ref: str, ext: str) -> float:
+        ...
+
+
+class MetricsManager:
+    """Manages scorer registration, score computation, metric configuration."""
+
+    def __init__(self, metrics_mapping: Dict[str, MetricConfig] | None = None):
+        self._scorers: Dict[str, Callable] = {}
+        self._metrics_mapping = metrics_mapping or {}
+        self._initialize_scorers()
+
+    @property
+    def metrics_mapping(self) -> Dict[str, MetricConfig]:
+        return self._metrics_mapping
+
+    @metrics_mapping.setter
+    def metrics_mapping(self, value: Dict[str, MetricConfig]):
+        self._metrics_mapping = value
+
+    def _initialize_scorers(self) -> None:
+        """Register existing scorers to prevent residual state."""
+        self._scorers.clear()
+
+        self.register_scorer(
+            EntityMetric.LEV_NORM.value,
+            distance.Levenshtein.normalized_similarity
+        )
+        self.register_scorer(
+            EntityMetric.JARO_WINKLER.value,
+            distance.JaroWinkler.normalized_similarity,
+        )
+        self.register_scorer(
+            EntityMetric.TOKEN_SET_RATIO.value,
+            fuzz.token_set_ratio,
+        )
+        self.register_scorer(
+            EntityMetric.TOKEN_SET_RATIO.value,
+            fuzz.token_set_ratio,
+        )
+
+        self.register_scorer(
+            EntityMetric.WRATIO.value,
+            fuzz.WRatio
+        )
+
+    def register_scorer(self, name: str, scorer: Callable) -> None:
+        """
+        Register a scorer
+
+        Args:
+            name (str): name of the scorer.
+            scorer (Callable): scorer to register.
+
+        Raises:
+            ValueError: if the scorer is not a callable.
+        """
+        self._scorers[name] = scorer
+
+    def get_scorer(self, name: str) -> Callable:
+        """
+        Retrieve a scorer by name.
+
+        Args:
+            name (str): name of the scorer.
+
+        Returns:
+            Callable: scorer.
+
+        Raises:
+            ValueError: if the passed name is not registered.
+        """
+        try:
+            scorer = self._scorers.get(name)
+            return scorer
+
+        except KeyError:
+            raise ValueError(f"[MetricsManager] '{name}' is not registered")
+
+    def get_metrics_config(self, field: str) -> MetricConfig:
+        """
+        Retrieve the metrics configuration for a given field.
+
+        Args:
+            field (str): field name.
+
+        Returns:
+            MetricConfig: metrics configuration for the given field.
+        """
+        default_config = MetricConfig(
+            field_name=field,
+            entity_metric=EntityMetric.TOKEN_SET_RATIO,
+            set_metric=SetMetric.ACCURACY,
+            threshold=0.5
+        )
+        return self._metrics_mapping.get(field, default_config)
+
+    def compute_entity_scores(
+            self,
+            reference_seq: List[str],
+            extracted_seq: List[str],
+            scorer: EntityMetric = EntityMetric.LEV_NORM,
+            pairwise: bool = True
+    ) -> List[ComputedScores]:
+        """
+        Compute the distance/similarity between ref/seq sequence entities.
+
+        Args:
+            reference_seq (List[str]): The reference sequence.
+            extracted_seq (List[str]): The extracted sequence.
+            scorer (str): Name of the scorer to use (e.g., 'levenshtein', 'jaro_winkler').
+            pairwise (bool): Whether to use pairwise distances or not.
+
+        Returns:
+            List[Tuple[str, str, np.float32]]: List of (reference, extracted, score) tuples.
+        """
+        if not reference_seq or not extracted_seq:
+            return [
+                ComputedScores(
+                    ref=reference_seq,
+                    ext=extracted_seq,
+                    e_metric=scorer.value,
+                    e_score=np.nan,
+                )
+            ]
+
+        if scorer not in EntityMetric.list():
+            logger.warning(f"[MetricsManager] Scorer name <{scorer}> is not supported.]")
+            raise ValueError(f"[MetricsManager] Scorer <{scorer}> is not registered.")
+
+        max_len = max(len(reference_seq), len(extracted_seq))
+        reference_padded = reference_seq + [""] * (max_len - len(reference_seq))
+        extracted_padded = extracted_seq + [""] * (max_len - len(extracted_seq))
+
+        scorer_func = cast(Callable, self.get_scorer(name=scorer.value))
+
+        if pairwise:
+            scores_ = process.cpdist(
+                queries=reference_padded,
+                choices=extracted_padded,
+                scorer=scorer_func,
+                processor=utils.default_process,
+                workers=-1,
+            )
+            scores = scores_.flatten()
+            res = [
+                ComputedScores(
+                    ref=reference_padded[i],
+                    ext=extracted_padded[i],
+                    e_metric=scorer.value,
+                    e_score=scores[i]
+                ) for i in range(len(scores))
+            ]
+
+        else:
+            scores_ = process.cdist(
+                queries=reference_padded,
+                choices=extracted_padded,
+                scorer=scorer_func,
+                processor=utils.default_process,
+                workers=-1,
+            )
+            scores = np.max(scores_, axis=1)
+            max_idx = np.argmax(scores_, axis=1)
+            res = [
+                ComputedScores(
+                    ref=reference_padded[i],
+                    ext=extracted_padded[max_idx[i]],
+                    e_metric=scorer.value,
+                    e_score=scores[i]
+                ) for i in range(len(scores))
+            ]
+
+        return res
+
+    @staticmethod
+    def compute_set_scores(
+            data: List[ComputedScores],
+            scorer: SetMetric = SetMetric.F1_SCORE,
+            threshold: float = 1.0,
+    ) -> ComparisonResults:
+        """
+        Compute evaluation metrics from similarity scores and return results as named tuples.
+
+        Args:
+            data: List of tuples containing reference string, extracted string, and similarity score.
+            scorer: Metric to compute.
+            threshold: Similarity threshold for considering a match.
+
+        Returns:
+            List[ComparisonResults]: List of named tuples containing reference, extracted, score, and metric value.
+        """
+        if not data:
+            return ComparisonResults("", "", None, None, None, None)
+
+        ref = [_.ref for _ in data]
+        ext = [_.ext for _ in data]
+        entity_scores = np.array([_.e_score for _ in data], dtype=np.float32)
+        entity_metric = data[0].e_metric
+
+        matches = np.count_nonzero(entity_scores >= threshold)
+
+        if len(data) == 1:
+            entity_scores = entity_scores.tolist()
+            set_scores = np.array(
+                [1 if score >= threshold else 0 for score in entity_scores], dtype=np.float32
+            ).tolist()
+            return ComparisonResults(
+                ref=ref,
+                ext=ext,
+                e_metric=entity_metric,
+                e_score=entity_scores,
+                s_metric=None,
+                s_score=set_scores
+            )
+
+        tp = matches
+        fp = len(ref) - int(matches)
+        fn = len(ext) - int(matches)
+
+        if scorer == SetMetric.ACCURACY:
+            accuracy = (tp / len(entity_scores)) if len(entity_scores) > 0 else 0.0
+            return ComparisonResults(
+                ref=ref,
+                ext=ext,
+                e_metric=entity_metric,
+                e_score=entity_scores,
+                s_metric=scorer.value,
+                s_score=accuracy
+            )
+
+        if scorer == SetMetric.F1_SCORE:
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1 = (
+                2 * (precision * recall) / (precision + recall)
+                if (precision + recall) > 0
+                else 0.0
+            )
+            return ComparisonResults(
+                ref=ref,
+                ext=ext,
+                e_metric=entity_metric,
+                e_score=entity_scores,
+                s_metric=scorer.value,
+                s_score=f1
+            )