levelapp 0.1.0__py3-none-any.whl

levelapp/clients/openai.py

@@ -0,0 +1,102 @@
+"""levelapp/clients/openai.py"""
+import os
+
+from typing import Dict, Any
+from levelapp.core.base import BaseChatClient
+
+
+class OpenAIClient(BaseChatClient):
+    """
+    Client for interacting with OpenAI's Chat Completions API.
+
+    This implementation adapts requests and responses to the OpenAI API
+    format, including chat message structure, headers, and token usage reporting.
+
+    Attributes:
+        model (str): Target model ID (default: "gpt-4o-mini").
+        base_url (str): Base endpoint for OpenAI API (default: https://api.openai.com/v1).
+        api_key (str): Authentication token for the OpenAI API.
+        max_tokens (int): Maximum tokens allowed in the completion.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.model = kwargs.get('model') or "gpt-4o-mini"
+        self.base_url = kwargs.get('base_url') or "https://api.openai.com/v1"
+        self.api_key = kwargs.get('api_key') or os.environ.get('OPENAI_API_KEY')
+        self.max_tokens = kwargs.get('max_tokens') or 1024
+
+        if not self.api_key:
+            raise ValueError("OpenAI API key not set")
+
+    @property
+    def endpoint_path(self) -> str:
+        """
+        API-specific endpoint path for chat completions.
+
+        Returns:
+            str: "/chat/completions"
+        """
+        return "/chat/completions"
+
+    def _build_endpoint(self) -> str:
+        """
+        Construct the full API endpoint URL.
+
+        Returns:
+            str: Concatenation of base_url and endpoint_path.
+        """
+        return f"{self.base_url}/{self.endpoint_path.lstrip('/')}"
+
+    def _build_headers(self) -> Dict[str, str]:
+        """
+        Build HTTP headers for the OpenAI API request.
+
+        Returns:
+            Dict[str, str]: Headers with authentication and content type.
+        """
+        return {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+
+    def _build_payload(self, message: str) -> Dict[str, Any]:
+        """
+        Construct the JSON payload for the OpenAI Chat Completions API.
+
+        Args:
+            message (str): User input or prompt to evaluate.
+
+        Returns:
+            Dict[str, Any]: Payload containing model ID, messages, and token limit.
+        """
+        return {
+            "model": self.model,
+            "messages": [{"role": "user", "content": message}],
+            "max_tokens": self.max_tokens,
+        }
+
+    def parse_response(self, response: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Parse and normalize the OpenAI API response.
+
+        - Extracts text output from `choices[0].message.content`.
+        - Attempts to JSON-parse the result if it contains structured content.
+        - Collects token usage metadata from `usage`.
+
+        Args:
+            response (Dict[str, Any]): Raw JSON response from OpenAI.
+
+        Returns:
+            Dict[str, Any]: {
+                "output": Parsed model output (dict or str),
+                "metadata": {
+                    "input_tokens": int,
+                    "output_tokens": int
+                }
+            }
+        """
+        input_tokens = response.get("usage", {}).get("prompt_tokens", 0)
+        output_tokens = response.get("usage", {}).get("completion_tokens", 0)
+        output = response.get("choices", [{}])[0].get("message", {}).get("content", "")
+        parsed = self.sanitizer.safe_load_json(text=output)
+        return {"output": parsed, "metadata": {"input_tokens": input_tokens, "output_tokens": output_tokens}}

levelapp/comparator/__init__.py

@@ -0,0 +1,5 @@
+from .comparator import MetadataComparator
+from .extractor import DataExtractor
+from .scorer import MetricsManager
+
+__all__ = ['MetadataComparator', 'DataExtractor', 'MetricsManager']

levelapp/comparator/comparator.py

@@ -0,0 +1,232 @@
+"""'comparator/service.py':"""
+from collections.abc import Mapping
+from typing import Any, Dict, List, Tuple, Literal
+
+from pydantic import BaseModel
+
+from levelapp.core.base import BaseProcess
+from levelapp.comparator.extractor import DataExtractor
+from levelapp.comparator.scorer import MetricsManager, ComparisonResults
+from levelapp.comparator.schemas import EntityMetric, SetMetric, MetricConfig
+from levelapp.comparator.utils import format_evaluation_results
+
+
+class MetadataComparator(BaseProcess):
+    """Metadata comparator component."""
+
+    def __init__(
+            self,
+            reference: BaseModel | None = None,
+            generated: BaseModel | None = None,
+            metrics_manager: MetricsManager | None = None,
+    ):
+        """
+        Initialize the MetadataComparator.
+
+        Args:
+            reference (BaseModel): Reference BaseModel
+            generated (BaseModel): Extracted BaseModel
+            metrics_manager (MetricsManager): MetricsManager
+        """
+        self.extractor = DataExtractor()
+
+        self._reference = reference
+        self._generated = generated
+        self._metrics_manager = metrics_manager
+
+        self._evaluation_data: List[
+            Tuple[str, list[str], list[str], Any, Any, Any, Any, float]
+        ] = []
+
+    @property
+    def reference_data(self) -> BaseModel:
+        return self._reference
+
+    @property
+    def generated_data(self) -> BaseModel:
+        return self._generated
+
+    @property
+    def metrics_manager(self) -> MetricsManager:
+        return self._metrics_manager
+
+    @reference_data.setter
+    def reference_data(self, value: BaseModel):
+        self._reference = value
+
+    @generated_data.setter
+    def generated_data(self, value: BaseModel):
+        self._generated = value
+
+    @metrics_manager.setter
+    def metrics_manager(self, value: MetricsManager):
+        self._metrics_manager = value
+
+    def _get_score(self, field: str) -> Tuple[EntityMetric, SetMetric, float]:
+        """
+        Retrieve the scoring metric and threshold for a given field.
+
+        Args:
+            field: The field for which to retrieve the metric and threshold.
+
+        Returns:
+            A tuple containing the scoring metric and its threshold.
+        """
+        if self._metrics_manager:
+            config = self._metrics_manager.get_metrics_config(field=field)
+        else:
+            config = MetricConfig()
+
+        return config.entity_metric, config.set_metric, config.threshold
+
+    def _format_results(
+            self,
+            output_type: Literal["json", "csv"] = "json"
+    ) -> Dict[int, Any]:
+        """
+        Format the internal evaluation data for reporting or storage.
+
+        Args:
+            output_type: 'json' returns a list of dictionaries; 'csv' returns a DataFrame.
+
+        Returns:
+            Formatted evaluation results or None if no data.
+        """
+        formatted_results = format_evaluation_results(self._evaluation_data, output_type=output_type)
+
+        return dict(enumerate(formatted_results))
+
+    def evaluate(
+            self,
+            reference_list: List[str],
+            extracted_list: List[str],
+            entity_metric: EntityMetric,
+            set_metric: SetMetric,
+            threshold: float,
+    ) -> ComparisonResults:
+        """
+        Evaluates pairwise similarity between elements in two lists using fuzzy matching.
+
+        Args:
+            reference_list: Ground-truth list of strings.
+            extracted_list: Extracted list of strings to compare.
+            entity_metric (EntityMetric): entity-level comparison metric.
+            set_metric (SetMetric): set-level comparison metric.
+            threshold: Similarity threshold (0–100) for considering a match.
+
+        Returns:
+            A dict with accuracy, precision, recall, and F1-score.
+        """
+        if not (reference_list or extracted_list):
+            return ComparisonResults("", "", entity_metric.value, None, set_metric.value, None)
+
+        scores = self._metrics_manager.compute_entity_scores(
+            reference_seq=reference_list,
+            extracted_seq=extracted_list,
+            scorer=entity_metric,
+            pairwise=False
+        )
+
+        return self._metrics_manager.compute_set_scores(
+            data=scores,
+            scorer=set_metric,
+            threshold=threshold,
+        )
+
+    def _recursive_compare(
+        self,
+        ref_node: Any,
+        ext_node: Any,
+        results: Dict[str, Dict[str, float]],
+        prefix: str = "",
+        threshold: float = 99.0,
+    ) -> None:
+        """
+        Recursively compare extracted vs. reference metadata nodes.
+
+        Args:
+            ref_node: dict or list (from deep_extract reference metadata)
+            ext_node: dict or list (from deep_extract extracted metadata)
+            results: Dict to accumulate comp_results keyed by hierarchical attribute paths.
+            prefix: str, current path prefix to form hierarchical keys.
+        """
+        # Case 1: Both nodes are dicts -> recurse on keys
+        if isinstance(ref_node, Mapping) and isinstance(ext_node, Mapping):
+            all_keys = set(ref_node.keys()) | set(ext_node.keys())
+            for key in all_keys:
+                new_prefix = f"{prefix}.{key}" if prefix else key
+                ref_subnode = ref_node.get(key, [])
+                ext_subnode = ext_node.get(key, [])
+                self._recursive_compare(
+                    ref_node=ref_subnode,
+                    ext_node=ext_subnode,
+                    results=results,
+                    prefix=new_prefix,
+                    threshold=threshold,
+                )
+
+        # Case 2: Leaf nodes (lists) -> evaluate directly
+        else:
+            # Defensive: convert to list if not list
+            ref_list = ref_node if isinstance(ref_node, list) else [ref_node]
+            ext_list = ext_node if isinstance(ext_node, list) else [ext_node]
+
+            # Convert all to strings for consistent fuzzy matching
+            ref_list_str = list(map(str, ref_list))
+            ext_list_str = list(map(str, ext_list))
+
+            entity_metric_, set_metric_, threshold = self._get_score(field=prefix)
+
+            # Evaluate similarity metrics
+            comp_results = self.evaluate(
+                reference_list=ref_list_str,
+                extracted_list=ext_list_str,
+                entity_metric=entity_metric_,
+                set_metric=set_metric_,
+                threshold=threshold,
+            )
+
+            if comp_results:
+                self._evaluation_data.append(
+                    (
+                        prefix,
+                        ref_list_str,
+                        ext_list_str,
+                        comp_results.e_metric,
+                        comp_results.e_score,
+                        comp_results.s_metric,
+                        comp_results.s_score,
+                        threshold,
+                    )
+                )
+
+            results[prefix] = comp_results or {"accuracy": 0}
+
+    def run(self, indexed_mode: bool = False) -> Dict[int, Any]:
+        """
+        Launch a metadata comparison process between reference and extracted data.
+
+        Args:
+            indexed_mode: Flag to use indexed mode for metadata extraction.
+
+        Returns:
+            Dictionary with comparison results, keyed by attribute paths.
+        """
+        self._evaluation_data.clear()
+
+        ref_data = self.extractor.deep_extract(model=self.reference_data, indexed=indexed_mode)
+        ext_data = self.extractor.deep_extract(model=self.generated_data, indexed=indexed_mode)
+
+        results: Dict[str, Dict[str, float]] = {}
+
+        self._recursive_compare(
+            ref_node=ref_data,
+            ext_node=ext_data,
+            results=results,
+            prefix="",
+            threshold=1,
+        )
+
+        formatted_results = self._format_results()
+
+        return formatted_results

levelapp/comparator/extractor.py

@@ -0,0 +1,108 @@
+"""levelapp/comparator/extractor.py"""
+
+from collections import defaultdict
+from collections.abc import Sequence
+from typing import List, Dict, Any
+from pydantic import BaseModel
+
+
+class DataExtractor:
+    """
+    Extracts primitive values from nested Pydantic models, dicts, and sequences.
+    """
+    def deep_extract(
+            self, model: BaseModel,
+            indexed: bool = False
+    ) -> Dict[str, List[str]]:
+        """
+        Extracts data in a recursive way from pydantic model.
+
+        Args:
+            model: An instance of a BaseModel.
+            indexed: Switch parameter to select the extraction approach.
+
+        Returns:
+            A dictionary where keys are attribute names and values are lists of string values.
+        """
+        result: Dict[str, List[str]] = defaultdict(list)
+        for field_name, field_info in type(model).model_fields.items():
+            field_value = getattr(model, field_name)
+            self._extract_field_values(
+                value=field_value, prefix=field_name, result=result, indexed=indexed
+            )
+
+        return result
+
+    def _extract_field_values(
+            self,
+            value: Any,
+            prefix: str,
+            result: Dict[str, List[str]],
+            indexed: bool = False,
+    ) -> None:
+        """
+        Recursively extract values from a field, storing them in result with field path as key.
+
+        Args:
+            value: The value to extract (BaseModel, dict, list, or primitive).
+            prefix: The current field path (e.g., 'documents.tribunal_members').
+            result: Dictionary to store field paths and their value lists.
+            indexed: Switch parameter to select the extraction approach.
+        """
+        if isinstance(value, BaseModel):
+            self._handle_model(model=value, prefix=prefix, result=result)
+
+        elif isinstance(value, Sequence) and not isinstance(value, (str, bytes)):
+            self._handle_sequence(
+                sequence=value, prefix=prefix, result=result, indexed=indexed
+            )
+
+        else:
+            result[prefix].append(value)
+
+    def _handle_model(
+        self, model: BaseModel, prefix: str, result: Dict[str, List[str]]
+    ) -> None:
+        """
+        Extract values from a Pydantic model recursively.
+
+        Args:
+            model: Pydantic BaseModel instance.
+            prefix: Current field path.
+            result: Dictionary to store field paths and value lists.
+        """
+        for field_name, field_info in type(model).model_fields.items():
+            field_value = getattr(model, field_name)
+            new_prefix = f"{prefix}.{field_name}" if prefix else field_name
+            self._extract_field_values(
+                value=field_value, prefix=new_prefix, result=result
+            )
+
+    def _handle_sequence(
+        self,
+        sequence: Sequence,
+        prefix: str,
+        result: Dict[str, List[str]],
+        indexed: bool = False,
+    ) -> None:
+        """
+        Extract values from a sequence (list or tuple) recursively.
+
+        Args:
+            sequence: List or tuple of values.
+            prefix: Current field path.
+            result: Dictionary to store field paths and value lists.
+            indexed: Switch parameter to select the extraction approach.
+        """
+        if not sequence:
+            result[prefix] = []
+
+        if indexed:
+            for i, item in enumerate(sequence):
+                new_prefix = f"{prefix}[{i}]" if prefix else f"[{i}]"
+                self._extract_field_values(value=item, prefix=new_prefix, result=result)
+        else:
+            for i, item in enumerate(sequence):
+                self._extract_field_values(
+                    value=item, prefix=prefix, result=result, indexed=indexed
+                )

levelapp/comparator/schemas.py

@@ -0,0 +1,61 @@
+"""'comparator/schemas.py': Defines Pydantic models for extracted metadata."""
+
+from enum import Enum
+
+from pydantic import BaseModel, Field
+from rapidfuzz import fuzz, utils
+
+
+class AttrCompMixin:
+    def __eq__(self, other) -> bool:
+        if not isinstance(other, type(self)):
+            return False
+
+        attr_name = next(iter(self.__dict__.keys()))
+        _cond = (
+            fuzz.ratio(
+                s1=getattr(self, attr_name),
+                s2=getattr(other, attr_name),
+                processor=utils.default_process,
+            )
+            > 99
+        )
+        return _cond
+
+
+class CompScoreMixin:
+    def comp_score(self, other) -> float:
+        attr_name = next(iter(self.__dict__.keys()))
+        _score = fuzz.ratio(
+            s1=getattr(self, attr_name),
+            s2=getattr(other, attr_name),
+            processor=utils.default_process,
+        )
+        return _score
+
+
+class EntityMetric(str, Enum):
+    WRATIO = "wratio"
+    LEV_NORM = "lev-norm"
+    JARO_WINKLER = "jaro-winkler"
+    TOKEN_SORT_RATIO = "token-sort-ratio"
+    TOKEN_SET_RATIO = "token-set-ratio"
+
+    @classmethod
+    def list(cls):
+        return [field.value for field in cls]
+
+
+class SetMetric(str, Enum):
+    ACCURACY = "accuracy"
+    F1_SCORE = "f1-score"
+
+
+class MetricConfig(BaseModel):
+    """
+    Configuration for a field's comparison metric.
+    """
+    field_name: str = Field(default="lev_norm", description="Name of the field")
+    entity_metric: EntityMetric = Field(default=EntityMetric.LEV_NORM, description="Entity level metric")
+    set_metric: SetMetric = Field(default=SetMetric.ACCURACY, description="Set level metric")
+    threshold: float = Field(default=100, ge=0, le=100, description="Match threshold")