levelapp 0.1.0__py3-none-any.whl

levelapp/comparator/scorer.py

@@ -0,0 +1,271 @@
+"""'comparator/scorer.py':"""
+import numpy as np
+
+from collections import namedtuple
+from typing import List, Dict, Callable, cast, Protocol
+
+from rapidfuzz import distance, process, utils, fuzz
+
+from levelapp.comparator.schemas import MetricConfig, EntityMetric, SetMetric
+from levelapp.aspects import logger
+
+ComputedScores = namedtuple(
+    typename="ComputedScores",
+    field_names=["ref", "ext", "e_metric", "e_score"],
+)
+ComparisonResults = namedtuple(
+    typename="ComparisonResults",
+    field_names=["ref", "ext", "e_metric", "e_score", "s_metric", "s_score"]
+)
+
+
+class Scorer(Protocol):
+    def __call__(self, ref: str, ext: str) -> float:
+        ...
+
+
+class MetricsManager:
+    """Manages scorer registration, score computation, metric configuration."""
+
+    def __init__(self, metrics_mapping: Dict[str, MetricConfig] | None = None):
+        self._scorers: Dict[str, Callable] = {}
+        self._metrics_mapping = metrics_mapping or {}
+        self._initialize_scorers()
+
+    @property
+    def metrics_mapping(self) -> Dict[str, MetricConfig]:
+        return self._metrics_mapping
+
+    @metrics_mapping.setter
+    def metrics_mapping(self, value: Dict[str, MetricConfig]):
+        self._metrics_mapping = value
+
+    def _initialize_scorers(self) -> None:
+        """Register existing scorers to prevent residual state."""
+        self._scorers.clear()
+
+        self.register_scorer(
+            EntityMetric.LEV_NORM.value,
+            distance.Levenshtein.normalized_similarity
+        )
+        self.register_scorer(
+            EntityMetric.JARO_WINKLER.value,
+            distance.JaroWinkler.normalized_similarity,
+        )
+        self.register_scorer(
+            EntityMetric.TOKEN_SET_RATIO.value,
+            fuzz.token_set_ratio,
+        )
+        self.register_scorer(
+            EntityMetric.TOKEN_SET_RATIO.value,
+            fuzz.token_set_ratio,
+        )
+
+        self.register_scorer(
+            EntityMetric.WRATIO.value,
+            fuzz.WRatio
+        )
+
+    def register_scorer(self, name: str, scorer: Callable) -> None:
+        """
+        Register a scorer
+
+        Args:
+            name (str): name of the scorer.
+            scorer (Callable): scorer to register.
+
+        Raises:
+            ValueError: if the scorer is not a callable.
+        """
+        self._scorers[name] = scorer
+        logger.info(f"[MetricsManager] Registered scorer: {name}")
+
+    def get_scorer(self, name: str) -> Callable:
+        """
+        Retrieve a scorer by name.
+
+        Args:
+            name (str): name of the scorer.
+
+        Returns:
+            Callable: scorer.
+
+        Raises:
+            ValueError: if the passed name is not registered.
+        """
+        try:
+            scorer = self._scorers.get(name)
+            logger.info(f"[get_scorer] Retrieved scorer: {name}")
+            return scorer
+
+        except KeyError:
+            raise ValueError(f"[MetricsManager] '{name}' is not registered")
+
+    def get_metrics_config(self, field: str) -> MetricConfig:
+        """
+        Retrieve the metrics configuration for a given field.
+
+        Args:
+            field (str): field name.
+
+        Returns:
+            MetricConfig: metrics configuration for the given field.
+        """
+        default_config = MetricConfig(
+            field_name=field,
+            entity_metric=EntityMetric.LEV_NORM,
+            set_metric=SetMetric.ACCURACY,
+            threshold=1
+        )
+        return self._metrics_mapping.get(field, default_config)
+
+    def compute_entity_scores(
+            self,
+            reference_seq: List[str],
+            extracted_seq: List[str],
+            scorer: EntityMetric = EntityMetric.LEV_NORM,
+            pairwise: bool = True
+    ) -> List[ComputedScores]:
+        """
+        Compute the distance/similarity between ref/seq sequence entities.
+
+        Args:
+            reference_seq (List[str]): The reference sequence.
+            extracted_seq (List[str]): The extracted sequence.
+            scorer (str): Name of the scorer to use (e.g., 'levenshtein', 'jaro_winkler').
+            pairwise (bool): Whether to use pairwise distances or not.
+
+        Returns:
+            List[Tuple[str, str, np.float32]]: List of (reference, extracted, score) tuples.
+        """
+        if not reference_seq or not extracted_seq:
+            return [
+                ComputedScores(
+                    ref=reference_seq,
+                    ext=extracted_seq,
+                    e_metric=scorer.value,
+                    e_score=np.nan,
+                )
+            ]
+
+        if scorer not in EntityMetric.list():
+            logger.warning(f"[MetricsManager] Scorer name <{scorer}> is not supported.]")
+            raise ValueError(f"[MetricsManager] Scorer <{scorer}> is not registered.")
+
+        max_len = max(len(reference_seq), len(extracted_seq))
+        reference_padded = reference_seq + [""] * (max_len - len(reference_seq))
+        extracted_padded = extracted_seq + [""] * (max_len - len(extracted_seq))
+
+        scorer_func = cast(Callable, self.get_scorer(name=scorer.value))
+
+        if pairwise:
+            scores_ = process.cpdist(
+                queries=reference_padded,
+                choices=extracted_padded,
+                scorer=scorer_func,
+                processor=utils.default_process,
+                workers=-1,
+            )
+            scores = scores_.flatten()
+            res = [
+                ComputedScores(
+                    ref=reference_padded[i],
+                    ext=extracted_padded[i],
+                    e_metric=scorer.value,
+                    e_score=scores[i]
+                ) for i in range(len(scores))
+            ]
+
+        else:
+            scores_ = process.cdist(
+                queries=reference_padded,
+                choices=extracted_padded,
+                scorer=scorer_func,
+                processor=utils.default_process,
+                workers=-1,
+            )
+            scores = np.max(scores_, axis=1)
+            max_idx = np.argmax(scores_, axis=1)
+            res = [
+                ComputedScores(
+                    ref=reference_padded[i],
+                    ext=extracted_padded[max_idx[i]],
+                    e_metric=scorer.value,
+                    e_score=scores[i]
+                ) for i in range(len(scores))
+            ]
+
+        return res
+
+    @staticmethod
+    def compute_set_scores(
+            data: List[ComputedScores],
+            scorer: SetMetric = SetMetric.F1_SCORE,
+            threshold: float = 1.0,
+    ) -> ComparisonResults:
+        """
+        Compute evaluation metrics from similarity scores and return results as named tuples.
+
+        Args:
+            data: List of tuples containing reference string, extracted string, and similarity score.
+            scorer: Metric to compute.
+            threshold: Similarity threshold for considering a match.
+
+        Returns:
+            List[ComparisonResults]: List of named tuples containing reference, extracted, score, and metric value.
+        """
+        if not data:
+            return ComparisonResults("", "", None, None, None, None)
+
+        ref = [_.ref for _ in data]
+        ext = [_.ext for _ in data]
+        entity_scores = np.array([_.e_score for _ in data], dtype=np.float32)
+        entity_metric = data[0].e_metric
+
+        matches = np.count_nonzero(entity_scores >= threshold)
+
+        if len(data) == 1:
+            entity_scores = entity_scores.tolist()
+            set_scores = np.array(
+                [1 if score >= threshold else 0 for score in entity_scores], dtype=np.float32
+            ).tolist()
+            return ComparisonResults(
+                ref=ref,
+                ext=ext,
+                e_metric=entity_metric,
+                e_score=entity_scores,
+                s_metric=None,
+                s_score=set_scores
+            )
+
+        tp = matches
+        fp = len(ref) - int(matches)
+        fn = len(ext) - int(matches)
+
+        if scorer == SetMetric.ACCURACY:
+            accuracy = (tp / len(entity_scores)) if len(entity_scores) > 0 else 0.0
+            return ComparisonResults(
+                ref=ref,
+                ext=ext,
+                e_metric=entity_metric,
+                e_score=entity_scores,
+                s_metric=scorer.value,
+                s_score=accuracy
+            )
+
+        if scorer == SetMetric.F1_SCORE:
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1 = (
+                2 * (precision * recall) / (precision + recall)
+                if (precision + recall) > 0
+                else 0.0
+            )
+            return ComparisonResults(
+                ref=ref,
+                ext=ext,
+                e_metric=entity_metric,
+                e_score=entity_scores,
+                s_metric=scorer.value,
+                s_score=f1
+            )

levelapp/comparator/utils.py

@@ -0,0 +1,136 @@
+"""levelapp/comparator/aspects.py:"""
+
+import re
+import json
+import logging
+import pandas as pd
+
+from typing import List, Dict, Any, Literal, Union
+from pathlib import Path
+
+
+def format_evaluation_results(
+    evaluation_results: List[tuple],
+    output_type: Literal["json", "csv"] = "json"
+) -> Union[List[Dict[str, Any]], pd.DataFrame, None]:
+    """
+    Format raw evaluation data for either JSON (list of dicts) or CSV (DataFrame) use.
+
+    Args:
+        evaluation_results: List of evaluation result tuples.
+        output_type: 'json' returns List[dict]; 'csv' returns a DataFrame.
+
+    Returns:
+        Formatted evaluation data or None if empty input.
+    """
+    if not evaluation_results:
+        logging.warning("No evaluation data to format.")
+        return None
+
+    rows = [
+        {
+            "field_name": field_name,
+            "reference_values": ref_values,
+            "extracted_values": ext_values,
+            "entity_metric": e_metric,
+            "entity_scores": e_scores,
+            "set_metric": s_metric,
+            "set_scores": s_scores,
+            "threshold": threshold,
+        }
+        for (field_name, ref_values, ext_values, e_metric, e_scores, s_metric, s_scores, threshold)
+        in evaluation_results
+    ]
+
+    return pd.DataFrame(rows) if output_type == "csv" else rows
+
+
+def store_evaluation_output(
+    formatted_data: Union[pd.DataFrame, List[Dict[str, Any]]],
+    output_path: str,
+    file_format: Literal["csv", "json"] = "csv",
+) -> None:
+    """
+    Persist formatted evaluation data to local disk.
+
+    Args:
+        formatted_data: Output from `format_evaluation_data`.
+        output_path: File path prefix (no extension).
+        file_format: 'csv' or 'json'.
+
+    Raises:
+        ValueError for unsupported formats or invalid data type.
+    """
+    if not formatted_data:
+        logging.warning("No data provided for local storage.")
+        return
+
+    try:
+        if file_format == "csv":
+            if not isinstance(formatted_data, pd.DataFrame):
+                raise TypeError("CSV output requires a pandas DataFrame.")
+            path = f"{output_path}.csv"
+            formatted_data.to_csv(path, index=False)
+
+        elif file_format == "json":
+            if not isinstance(formatted_data, list):
+                raise TypeError("JSON output requires a list of dictionaries.")
+            path = f"{output_path}.json"
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(formatted_data, f, indent=2, ensure_ascii=False)
+
+        else:
+            raise ValueError(f"Unsupported file format: {file_format}")
+
+        logging.info(f"Evaluation data saved to {path}")
+
+    except Exception as e:
+        logging.error(f"Failed to save evaluation output: {e}")
+
+
+def safe_load_json_file(file_path: Union[str, Path]) -> Any:
+    """
+    Load a potentially malformed JSON file by pre-sanitizing its content at the byte/text level.
+
+    Args:
+        file_path: Path to the potentially malformed JSON file.
+
+    Returns:
+        Parsed JSON content (as a Python dict or list).
+
+    Raises:
+        ValueError: If JSON parsing fails even after pre-sanitization.
+    """
+    with open(file_path, "rb") as f:
+        raw_bytes = f.read()
+
+    raw_text = raw_bytes.decode("utf-8", errors="replace")
+    sanitized_text = _clean_malformed_json_text(raw_text)
+
+    try:
+        return json.loads(sanitized_text)
+
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to decode JSON after sanitization: {e}")
+
+
+def _clean_malformed_json_text(text: str) -> str:
+    """
+    Remove common forms of JSON text corruption before parsing.
+
+    Args:
+        text: Raw JSON string content.
+
+    Returns:
+        A sanitized string safe for json.loads() parsing.
+    """
+    # Strip BOM (please do not delete this comment)
+    text = text.lstrip('\ufeff')
+
+    # Remove non-printable control characters except \t, \n, \r (please do not delete this comment)
+    text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
+
+    # Remove invalid characters (like \uFFFD or strange CP1252 remnants) (please do not delete this comment)
+    text = text.replace("\ufffd", "?")
+
+    return text

levelapp/config/__init__.py

@@ -0,0 +1,5 @@
+from .endpoint import EndpointConfig
+from .prompts import EVAL_PROMPT_TEMPLATE
+
+
+__all__ = ['EndpointConfig', 'EVAL_PROMPT_TEMPLATE']

levelapp/config/endpoint.py

@@ -0,0 +1,190 @@
+"""levelapp/config/endpoint.py"""
+import os
+import json
+import yaml
+
+from string import Template
+from dotenv import load_dotenv
+
+from enum import Enum
+from typing import Literal, Dict, Any
+from pydantic import BaseModel, HttpUrl, SecretStr, Field, computed_field
+
+from levelapp.aspects import logger
+
+
+class TemplateType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+class EndpointConfig(BaseModel):
+    """
+    Configuration class for user system's endpoint.
+
+    Parameters:
+        base_url (HttpUrl): The base url of the endpoint.
+        method (Literal['POST', 'GET']): The HTTP method to use (POST or GET).
+        api_key (SecretStr): The API key to use.
+        bearer_token (SecretStr): The Bearer token to use.
+        model_id (str): The model to use (if applicable).
+        default_request_payload_template (Dict[str, Any]): The payload template to use.
+        generated_request_payload_template (Dict[str, Any]): The generated payload template from a provided file.
+        variables (Dict[str, Any]): The variables to populate the payload template.
+
+    Note:
+        Either you use the provided configuration YAML file, providing the following:\n
+        - base_url (HttpUrl): The base url of the endpoint.
+        - method (Literal['POST', 'GET']): The HTTP method to use (POST or GET).
+        - api_key (SecretStr): The API key to use.
+        - bearer_token (SecretStr): The Bearer token to use.
+        - model_id (str): The model to use (if applicable).
+        - default_payload_template (Dict[str, Any]): The payload template to use.
+        - generated_payload_template (Dict[str, Any]): The generated payload template from a provided file.
+        - variables (Dict[str, Any]): The variables to populate the payload template.
+
+        Or manually configure the model instance by assigning the proper values to the model fields.\n
+        You can also provide the path in the .env file for the payload template (ENDPOINT_PAYLOAD_PATH)
+        and the response template (ENDPOINT_RESPONSE_PATH) separately. The files can be either YAML or JSON only.
+    """
+    load_dotenv()
+
+    # Required
+    method: Literal["POST", "GET"] = Field(default="POST")
+    base_url: HttpUrl = Field(default=HttpUrl)
+    url_path: str = Field(default='')
+
+    # Auth
+    api_key: SecretStr | None = Field(default=None)
+    bearer_token: SecretStr | None = Field(default=None)
+    model_id: str | None = Field(default='')
+
+    # Data
+    default_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
+    generated_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
+    default_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
+    generated_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
+
+    # Variables
+    variables: Dict[str, Any] = Field(default_factory=dict)
+
+    @computed_field()
+    @property
+    def full_url(self) -> str:
+        return str(self.base_url) + self.url_path
+
+    @computed_field()
+    @property
+    def headers(self) -> Dict[str, Any]:
+        headers: Dict[str, Any] = {"Content-Type": "application/json"}
+        if self.model_id:
+            headers["x-model-id"] = self.model_id
+        if self.bearer_token:
+            headers["Authorization"] = f"Bearer {self.bearer_token.get_secret_value()}"
+        if self.api_key:
+            headers["x-api-key"] = self.api_key.get_secret_value()
+        return headers
+
+    @computed_field
+    @property
+    def request_payload(self) -> Dict[str, Any]:
+        """Return fully prepared payload depending on template or full payload."""
+        # First, load the request payload template (either from YAML config file or from specific template)
+        if not self.variables:
+            return self.default_request_payload_template
+
+        if not self.default_request_payload_template:
+            self.load_template(template_type=TemplateType.REQUEST)
+            base_template = self.generated_request_payload_template
+        else:
+            base_template = self.default_request_payload_template
+
+        # Second, replace the placeholders with the variables
+        payload = self._replace_placeholders(obj=base_template, variables=self.variables)
+
+        # Third, merge the "request_payload" if present in variables
+        additional_payload_data = self.variables.get("request_payload", {})
+        if additional_payload_data:
+            payload.update(additional_payload_data)
+
+        self.variables.clear()
+
+        return payload
+
+    @computed_field
+    @property
+    def response_payload(self) -> Dict[str, Any]:
+        if not self.variables:
+            return self.default_response_payload_template
+
+        if not self.default_response_payload_template:
+            self.load_template(template_type=TemplateType.RESPONSE)
+            base_template = self.generated_response_payload_template
+        else:
+            base_template = self.default_response_payload_template
+
+        response_payload = self._replace_placeholders(obj=base_template, variables=self.variables)
+        self.variables.clear()
+
+        return response_payload
+
+    @staticmethod
+    def _replace_placeholders(obj: Any, variables: Dict[str, Any]) -> Dict[str, Any]:
+        """Recursively replace placeholders in payload template with variables."""
+        def _replace(_obj):
+            if isinstance(_obj, str):
+                subst = Template(_obj).safe_substitute(variables)
+                if '$' in subst:
+                    logger.warning(f"[EndpointConfig] Unsubstituted placeholder in payload:\n{subst}\n\n")
+                return subst
+
+            elif isinstance(_obj, dict):
+                return {k: _replace(v) for k, v in _obj.items()}
+
+            elif isinstance(_obj, list):
+                return [_replace(v) for v in _obj]
+
+            return _obj
+
+        return _replace(obj)
+
+    def load_template(
+            self,
+            template_type: TemplateType = TemplateType.REQUEST,
+            path: str | None = None
+    ) -> Dict[str, Any]:
+        try:
+            if not path:
+                env_var = "ENDPOINT_PAYLOAD_PATH" if template_type == TemplateType.REQUEST else "ENDPOINT_RESPONSE_PATH"
+                path = os.getenv(env_var, '')
+
+            if not os.path.exists(path):
+                raise FileNotFoundError(f"The provide payload template file path '{path}' does not exist.")
+
+            with open(path, "r", encoding="utf-8") as f:
+                if path.endswith((".yaml", ".yml")):
+                    data = yaml.safe_load(f)
+
+                elif path.endswith(".json"):
+                    data = json.load(f)
+
+                else:
+                    raise ValueError("[EndpointConfig] Unsupported file format.")
+
+                self.generated_request_payload_template = data
+                return data
+
+        except FileNotFoundError as e:
+            raise FileNotFoundError(f"[EndpointConfig] Payload template file '{e.filename}' not found in path.")
+
+        except yaml.YAMLError as e:
+            raise ValueError(f"[EndpointConfig] Error parsing YAML file:\n{e}")
+
+        except json.JSONDecodeError as e:
+            raise ValueError(f"[EndpointConfig] Error parsing JSON file:\n{e}")
+
+        except IOError as e:
+            raise IOError(f"[EndpointConfig] Error reading file:\n{e}")
+
+        except Exception as e:
+            raise ValueError(f"[EndpointConfig] Unexpected error loading configuration:\n{e}")

levelapp/config/prompts.py

@@ -0,0 +1,35 @@
+EVAL_PROMPT_TEMPLATE = """
+You are an impartial evaluator for a conversational system.
+Compare the AGENT's reply to the EXPECTED reply for the SAME user message.
+
+Consider only:
+1) Semantic Coverage — does the AGENT cover the key points in EXPECTED?
+2) Faithfulness — no contradictions or invented details relative to EXPECTED.
+3) Appropriateness — tone/format suitable for the user message.
+Ignore minor wording/punctuation differences. Do NOT reward verbosity.
+
+Scale (integer):
+0 = Poor (misses key points or contradicts)
+1 = Moderate (captures some ideas, noticeable gaps)
+2 = Good (mostly matches, minor omissions/differences)
+3 = Excellent (semantically equivalent; no meaningful differences)
+
+USER_MESSAGE:
+\"\"\"{user_input}\"\"\"
+
+EXPECTED (reference reply):
+\"\"\"{reference_text}\"\"\"
+
+AGENT (model reply):
+\"\"\"{generated_text}\"\"\"
+
+Return ONLY a single JSON object on one line with exactly these keys:
+- "score": <0|1|2|3>,
+- "label": "<Poor|Moderate|Good|Excellent>",
+- "justification": "<1-2 concise sentences>",
+- "evidence":
+    - "covered_points": ["<short phrase>", "..."],   // <=3 items
+    - "missing_or_wrong": ["<short phrase>", "..."]  // <=3 items
+    
+Do NOT include any additional text, explanations, or formatting (e.g., "JSON object:", ```json or ```, or markdown).
+"""