levelapp 0.1.0__py3-none-any.whl

levelapp/evaluator/evaluator.py

@@ -0,0 +1,265 @@
+"""levelapp/core/evaluator.py"""
+from functools import lru_cache
+from typing import List, Dict, Any
+from collections import defaultdict
+from pydantic import BaseModel, Field
+
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+    AsyncRetrying,
+    RetryError,
+)
+
+from levelapp.clients import ClientRegistry
+from levelapp.comparator import MetricsManager, MetadataComparator
+from levelapp.config.prompts import EVAL_PROMPT_TEMPLATE
+from levelapp.core.base import BaseEvaluator, BaseChatClient
+from levelapp.aspects import MonitoringAspect, MetricType, logger, DataLoader
+
+
+class Evidence(BaseModel):
+    """Evidence details for evaluation."""
+    covered_points: List[str] = Field(
+        default_factory=list,
+        description="Key points covered the agent reply covered (<= 3 items)"
+    )
+    missing_or_wrong: List[str] = Field(
+        default_factory=list,
+        description="Key points the agent reply missed or contradicted (<= 3 items)"
+    )
+
+
+class JudgeEvaluationResults(BaseModel):
+    """Structured result of an interaction evaluation."""
+    provider: str = Field(..., description="The provider name, e.g., 'openai', 'ionos'")
+    score: int = Field(..., ge=0, le=3, description="Evaluation score between 0 and 3")
+    label: str = Field(..., description="The label of the evaluation result")
+    justification: str = Field(..., description="Short explanation of the evaluation result")
+    evidence: Evidence = Field(default_factory=Evidence, description="Detailed evidence for the evaluation")
+    raw_response: Dict[str, Any] = Field(..., description="Full unprocessed API response")
+    metadata: Dict[str, Any] = Field(..., description="Metadata about the evaluation result")
+
+    @classmethod
+    def from_parsed(cls, provider: str, parsed: Dict[str, Any], raw: Dict[str, Any]) -> "JudgeEvaluationResults":
+        """
+        Build a model instance from the provided data.
+
+        Args:
+            provider (str): The provider name.
+            parsed (Dict[str, Any]): The parsed response data.
+            raw (Dict[str, Any]): The raw response data.
+
+        Returns:
+            JudgeEvaluationResults: The constructed evaluation result instance.
+        """
+        content = parsed.get("output", {})
+        metadata = parsed.get("metadata", {})
+        return cls(
+            provider=provider,
+            score=content.get("score", 0),
+            label=content.get("label", "N/A"),
+            justification=content.get("justification", "N/A"),
+            evidence=Evidence(**content.get("evidence", {})),
+            raw_response=raw,
+            metadata=metadata,
+        )
+
+
+class JudgeEvaluator(BaseEvaluator):
+    def __init__(self):
+        self.prompt_template = EVAL_PROMPT_TEMPLATE
+        self.clients = defaultdict(BaseChatClient)
+
+    def register_client(self, provider: str, client: BaseChatClient) -> None:
+        """
+        Register LLM clients used for the evaluation.
+
+        Args:
+            provider (str): The provider name.
+            client (BaseChatClient): The LLM client to register.
+        """
+        self.clients[provider] = client
+
+    @lru_cache(maxsize=1024)
+    def _build_prompt(self, user_input: str, generated_text: str, reference_text: str) -> str:
+        """
+        Build the prompt used for the evaluation.
+
+        Args:
+            user_input (str): The user input.
+            generated_text (str): The generated text.
+            reference_text (str): The reference text.
+
+        Returns:
+            A string containing the prompt.
+        """
+        return self.prompt_template.format(
+            user_input=user_input,
+            generated_text=generated_text,
+            reference_text=reference_text
+        )
+
+    @retry(
+        retry=retry_if_exception_type((TimeoutError, ValueError, RuntimeError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    def evaluate(
+            self,
+            generated_data: str,
+            reference_data: str,
+            user_input: str,
+            provider: str,
+    ) -> JudgeEvaluationResults | None:
+        """
+        Synchronous evaluation for the generated data.
+
+        Args:
+            generated_data (str): The generated data.
+            reference_data (str): The reference data.
+            user_input (str): The user input.
+            provider (str): The LLM provider user for evaluation.
+
+        Returns:
+            JudgeEvaluationResults instance containing the evaluation results.
+
+        Raises:
+            Exception: If the evaluation failed.
+        """
+        prompt = self._build_prompt(
+            user_input=user_input,
+            generated_text=generated_data,
+            reference_text=reference_data
+        )
+        client = ClientRegistry.get(provider=provider)
+
+        try:
+            response = client.call(message=prompt)
+            logger.info(f"[{provider}] Evaluation: {response}\n{'---' * 10}")
+            parsed = client.parse_response(response=response)
+            return JudgeEvaluationResults.from_parsed(provider=provider, parsed=parsed, raw=response)
+
+        except Exception as e:
+            logger.error(f"[{provider}] Evaluation failed: {e}", exc_info=True)
+            return JudgeEvaluationResults(
+                provider=provider,
+                score=0,
+                label="N/A",
+                justification="N/A",
+                evidence=Evidence(covered_points=[], missing_or_wrong=[]),
+                raw_response={},
+                metadata={}
+            )
+
+    @MonitoringAspect.monitor(name="judge_evaluation", category=MetricType.API_CALL)
+    async def async_evaluate(
+            self,
+            generated_data: str,
+            reference_data: str,
+            user_input: str,
+            provider: str,
+    ) -> JudgeEvaluationResults | None:
+        """
+        Synchronous evaluation for the generated data.
+
+        Args:
+            generated_data (str): The generated data.
+            reference_data (str): The reference data.
+            user_input (str): The user input.
+            provider (str): The LLM provider user for evaluation.
+
+        Returns:
+            JudgeEvaluationResults instance containing the evaluation results.
+
+        Raises:
+            RetryError: If the evaluation failed.
+        """
+        prompt = self._build_prompt(
+            user_input=user_input,
+            generated_text=generated_data,
+            reference_text=reference_data
+        )
+        client = ClientRegistry.get(provider=provider)
+
+        try:
+            async for attempt in AsyncRetrying(
+                retry=retry_if_exception_type((TimeoutError, ValueError, RuntimeError)),
+                stop=stop_after_attempt(3),
+                wait=wait_exponential(multiplier=1, min=2, max=10),
+                reraise=True,
+            ):
+                with attempt:
+                    response = await client.acall(message=prompt)
+                    logger.info(f"[{provider}] Async evaluation:\n{response}\n{'---' * 10}")
+                    parsed = client.parse_response(response=response)
+                    return JudgeEvaluationResults.from_parsed(provider=provider, parsed=parsed, raw=response)
+
+        except RetryError as e:
+            logger.error(f"[{provider}] Async evaluation failed after retries: {e}", exc_info=True)
+            return JudgeEvaluationResults(
+                provider=provider,
+                score=0,
+                label="N/A",
+                justification="N/A",
+                evidence=Evidence(covered_points=[], missing_or_wrong=[]),
+                raw_response={},
+                metadata={}
+            )
+
+
+class MetadataEvaluator(BaseEvaluator):
+    def __init__(self):
+        self.data_loader = DataLoader()
+        self.comparator = MetadataComparator()
+        self.metrics_manager = MetricsManager()
+
+    def evaluate(
+            self,
+            generated_data: str | Dict[str, Any],
+            reference_data: str | Dict[str, Any],
+            metrics_mapping: Any | None = None,
+    ) -> Dict[str, float]:
+        """
+        Synchronous evaluation for the generated data.
+
+        Args:
+              generated_data (str): The generated data.
+              reference_data (str): The reference data.
+              metrics_mapping (dict): A dictionary mapping metric names to metrics.
+
+        Returns:
+              A dict containing the evaluation results.
+        """
+        gen_data = self.data_loader.create_dynamic_model(data=generated_data, model_name="GeneratedMetadata")
+        ref_data = self.data_loader.create_dynamic_model(data=reference_data, model_name="ReferenceMetadata")
+
+        if metrics_mapping:
+            self.comparator.metrics_manager = metrics_mapping
+
+        self.comparator.metrics_manager = self.metrics_manager
+        self.comparator.generated_data = gen_data
+        self.comparator.reference_data = ref_data
+
+        output = self.comparator.run(indexed_mode=False)
+        logger.info(f"Comparison results:\n{output}\n---")
+        results: Dict[str, float] = {}
+
+        for k, v in output.items():
+            field = v.get("field_name", "N/A")
+            score = v.get("set_scores", -1)
+            results[field] = int(score[0]) if isinstance(score, list) else int(score)
+
+        return results
+
+    async def async_evaluate(
+            self,
+            generated_data: str | Dict[str, Any],
+            reference_data: str | Dict[str, Any],
+            **kwargs
+    ):
+        """Not implemented yet."""
+        raise NotImplementedError()

levelapp/metrics/__init__.py

@@ -0,0 +1,67 @@
+"""levelapp/metrics/__init__.py"""
+import logging
+
+from typing import List, Dict, Type, Any
+
+from levelapp.core.base import BaseMetric
+from levelapp.metrics.exact import EXACT_METRICS
+from levelapp.metrics.fuzzy import FUZZY_METRICS
+
+logger = logging.getLogger(__name__)
+
+
+class MetricRegistry:
+    """Registry for metric classes."""
+    _metrics: Dict[str, Type[BaseMetric]] = {}
+
+    @classmethod
+    def register(cls, name: str, metric_class: Type[BaseMetric]) -> None:
+        """
+        Register a metric class under a given name.
+
+        Args:
+            name (str): Unique identifier for the metric.
+            metric_class (Type[BaseMetric]): The metric class to register.
+        """
+        if name in cls._metrics:
+            raise KeyError(f"Metric '{name}' is already registered")
+
+        cls._metrics[name] = metric_class
+        logger.info(f"Metric '{name}' registered successfully.")
+
+    @classmethod
+    def get(cls, name: str, **kwargs: Any) -> BaseMetric:
+        """
+        Retrieve an instance of a registered metric by its name.
+
+        Args:
+            name (str): The name of the metric to retrieve.
+
+        Returns:
+            Type[BaseMetric]: The metric class associated with the given name.
+
+        Raises:
+            KeyError: If the metric is not found.
+        """
+        if name not in cls._metrics:
+            raise KeyError(f"Metric '{name}' is not registered")
+
+        return cls._metrics[name](**kwargs)
+
+    @classmethod
+    def list_metrics(cls) -> List[str]:
+        return list(cls._metrics.keys())
+
+    @classmethod
+    def unregister(cls, name: str) -> None:
+        cls._metrics.pop(name, None)
+
+
+METRICS = FUZZY_METRICS | EXACT_METRICS
+
+for name_, metric_class_ in METRICS.items():
+    try:
+        MetricRegistry.register(name_, metric_class_)
+
+    except Exception as e:
+        logger.info(f"Failed to register metric {name_}: {e}")

levelapp/metrics/embedding.py

@@ -0,0 +1,2 @@
+"""levelapp/metrics/token.py"""
+# TODO-0: Implement Cosine Similarity, BERTScore, and other token-based metrics.

levelapp/metrics/exact.py

@@ -0,0 +1,182 @@
+"""levelapp/metrics/exact.py"""
+from typing import Dict, Any
+
+from rapidfuzz import distance
+
+from levelapp.core.base import BaseMetric
+from levelapp.aspects.monitor import MonitoringAspect, MetricType
+
+
+class ExactMatch(BaseMetric):
+    """Binary exact match comparison (1.0 for exact match, 0.0 otherwise)"""
+
+    @MonitoringAspect.monitor(name="exact_match", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """"
+        Compute the exact match score between generated and reference strings.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the exact match score and metadata.
+        """
+        self._validate_inputs(generated=generated, reference=reference)
+
+        score = distance.Levenshtein.normalized_similarity(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=1.0
+        )
+
+        return {
+            "score": score,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class Levenshtein(BaseMetric):
+    """Levenshtein edit distance (number of insertions, deletions, substitutions)"""
+
+    @MonitoringAspect.monitor(name="levenshtein", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the Levenshtein distance score between generated and reference strings.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the Levenshtein score and metadata.
+        """
+        self._validate_inputs(generated=generated, reference=reference)
+
+        score = distance.Levenshtein.normalized_similarity(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff or 1.0
+        )
+
+        return {
+            "score": score,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class JaroWinkler(BaseMetric):
+    """Jaro-Winkler distance (similarity measure for strings)"""
+
+    @MonitoringAspect.monitor(name="jaro-winkler", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the Jaro-Winkler distance score between generated and reference strings.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the Jaro-Winkler score and metadata.
+        """
+        self._validate_inputs(generated=generated, reference=reference)
+
+        score = distance.JaroWinkler.normalized_similarity(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class Hamming(BaseMetric):
+    """Hamming distance (character substitutions only, for equal-length strings)"""
+
+    @MonitoringAspect.monitor(name="hamming", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the Hamming distance score between generated and reference strings.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the Hamming score and metadata.
+        """
+        self._validate_inputs(generated=generated, reference=reference)
+
+        score = distance.Hamming.normalized_similarity(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class PrefixMatch(BaseMetric):
+    """Prefix similarity (1.0 if generated starts with reference)"""
+
+    @MonitoringAspect.monitor(name="prefix-match", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the Prefix similarity score between generated and reference strings.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the Prefix similarity and metadata.
+        """
+        self._validate_inputs(generated=generated, reference=reference)
+
+        score = distance.Prefix.normalized_similarity(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+# Registry of all exact metrics
+EXACT_METRICS = {
+    "exact_match": ExactMatch,
+    "levenshtein": Levenshtein,
+    "jaro_winkler": JaroWinkler,
+    "hamming": Hamming,
+    "prefix_match": PrefixMatch
+}

levelapp/metrics/fuzzy.py

@@ -0,0 +1,80 @@
+"""levelapp/metrics/fuzzy.py"""
+from rapidfuzz import fuzz
+
+from typing import Dict, Any
+
+from levelapp.core.base import BaseMetric
+from levelapp.aspects.monitor import MonitoringAspect, MetricType
+
+
+class FuzzyRatio(BaseMetric):
+    """A metric that computes the fuzzy ratio between two texts."""
+
+    @MonitoringAspect.monitor(name="fuzzy-ratio", category=MetricType.API_CALL, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the fuzzy ratio between the generated text and the reference text.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the fuzzy ratio score and metadata.
+        """
+        score = fuzz.ratio(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        # TODO-0: Return results as Pydantic model.
+        return {
+            "score": score / 100,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class PartialRatio(BaseMetric):
+    """
+    A metric that computes the partial fuzzy ratio between two texts.
+    This is useful for evaluating how similar two pieces of text are,
+    allowing for partial matches.
+    """
+
+    @MonitoringAspect.monitor(name="partial-ratio", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the partial fuzzy ratio between the generated text and the reference text.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the partial fuzzy ratio.
+        """
+        score = fuzz.partial_ratio(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score / 100,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+FUZZY_METRICS = {
+    "fuzzy_ratio": FuzzyRatio,
+    "partial_ratio": PartialRatio,
+}

levelapp/metrics/token.py

@@ -0,0 +1,103 @@
+"""levelapp/metrics/token.py"""
+from rapidfuzz import fuzz
+
+from typing import Dict, Any
+
+from levelapp.core.base import BaseMetric
+from levelapp.aspects.monitor import MonitoringAspect, MetricType
+
+
+class WeightedRatio(BaseMetric):
+    """A metric that calculates a weighted ratio based on the other ratio algorithms"""
+
+    @MonitoringAspect.monitor(name="weighted-ratio", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the token-based metric between the generated text and the reference text.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the score and metadata.
+        """
+        score = fuzz.WRatio(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score / 100,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class TokenSetRatio(BaseMetric):
+    """
+    A metric that compares the words in the strings based
+    on unique and common words between them using fuzz.ratio.
+    """
+
+    @MonitoringAspect.monitor(name="token-set-ratio", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the token-based metric between the generated text and the reference text.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the score and metadata.
+        """
+        score = fuzz.token_set_ratio(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score / 100,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }
+
+
+class TokenSortRatio(BaseMetric):
+    """A metric that sorts the words in the strings and calculates the fuzz.ratio between them."""
+
+    @MonitoringAspect.monitor(name="token-sort-ratio", category=MetricType.SCORING, cached=True, enable_timing=True)
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """
+        Compute the token-based metric between the generated text and the reference text.
+
+        Args:
+            generated (str): The text generated by the agent.
+            reference (str): The expected reference text.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the score and metadata.
+        """
+        score = fuzz.token_sort_ratio(
+            s1=generated,
+            s2=reference,
+            processor=self.processor,
+            score_cutoff=self.score_cutoff
+        )
+
+        return {
+            "score": score / 100,
+            "metadata": self._build_metadata(
+                generated_length=len(generated),
+                reference_length=len(reference)
+            )
+        }