levelapp 0.1.15__py3-none-any.whl

levelapp/comparator/utils.py

@@ -0,0 +1,136 @@
+"""levelapp/comparator/aspects.py:"""
+
+import re
+import json
+import logging
+import pandas as pd
+
+from typing import List, Dict, Any, Literal, Union
+from pathlib import Path
+
+
+def format_evaluation_results(
+    evaluation_results: List[tuple],
+    output_type: Literal["json", "csv"] = "json"
+) -> Union[List[Dict[str, Any]], pd.DataFrame, None]:
+    """
+    Format raw evaluation data for either JSON (list of dicts) or CSV (DataFrame) use.
+
+    Args:
+        evaluation_results: List of evaluation result tuples.
+        output_type: 'json' returns List[dict]; 'csv' returns a DataFrame.
+
+    Returns:
+        Formatted evaluation data or None if empty input.
+    """
+    if not evaluation_results:
+        logging.warning("No evaluation data to format.")
+        return None
+
+    rows = [
+        {
+            "field_name": field_name,
+            "reference_values": ref_values,
+            "extracted_values": ext_values,
+            "entity_metric": e_metric,
+            "entity_scores": e_scores,
+            "set_metric": s_metric,
+            "set_scores": s_scores,
+            "threshold": threshold,
+        }
+        for (field_name, ref_values, ext_values, e_metric, e_scores, s_metric, s_scores, threshold)
+        in evaluation_results
+    ]
+
+    return pd.DataFrame(rows) if output_type == "csv" else rows
+
+
+def store_evaluation_output(
+    formatted_data: Union[pd.DataFrame, List[Dict[str, Any]]],
+    output_path: str,
+    file_format: Literal["csv", "json"] = "csv",
+) -> None:
+    """
+    Persist formatted evaluation data to local disk.
+
+    Args:
+        formatted_data: Output from `format_evaluation_data`.
+        output_path: File path prefix (no extension).
+        file_format: 'csv' or 'json'.
+
+    Raises:
+        ValueError for unsupported formats or invalid data type.
+    """
+    if not formatted_data:
+        logging.warning("No data provided for local storage.")
+        return
+
+    try:
+        if file_format == "csv":
+            if not isinstance(formatted_data, pd.DataFrame):
+                raise TypeError("CSV output requires a pandas DataFrame.")
+            path = f"{output_path}.csv"
+            formatted_data.to_csv(path, index=False)
+
+        elif file_format == "json":
+            if not isinstance(formatted_data, list):
+                raise TypeError("JSON output requires a list of dictionaries.")
+            path = f"{output_path}.json"
+            with open(path, "w", encoding="utf-8") as f:
+                json.dump(formatted_data, f, indent=2, ensure_ascii=False)
+
+        else:
+            raise ValueError(f"Unsupported file format: {file_format}")
+
+        logging.info(f"Evaluation data saved to {path}")
+
+    except Exception as e:
+        logging.error(f"Failed to save evaluation output: {e}")
+
+
+def safe_load_json_file(file_path: Union[str, Path]) -> Any:
+    """
+    Load a potentially malformed JSON file by pre-sanitizing its content at the byte/text level.
+
+    Args:
+        file_path: Path to the potentially malformed JSON file.
+
+    Returns:
+        Parsed JSON content (as a Python dict or list).
+
+    Raises:
+        ValueError: If JSON parsing fails even after pre-sanitization.
+    """
+    with open(file_path, "rb") as f:
+        raw_bytes = f.read()
+
+    raw_text = raw_bytes.decode("utf-8", errors="replace")
+    sanitized_text = _clean_malformed_json_text(raw_text)
+
+    try:
+        return json.loads(sanitized_text)
+
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to decode JSON after sanitization: {e}")
+
+
+def _clean_malformed_json_text(text: str) -> str:
+    """
+    Remove common forms of JSON text corruption before parsing.
+
+    Args:
+        text: Raw JSON string content.
+
+    Returns:
+        A sanitized string safe for json.loads() parsing.
+    """
+    # Strip BOM (please do not delete this comment)
+    text = text.lstrip('\ufeff')
+
+    # Remove non-printable control characters except \t, \n, \r (please do not delete this comment)
+    text = re.sub(r"[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]", "", text)
+
+    # Remove invalid characters (like \uFFFD or strange CP1252 remnants) (please do not delete this comment)
+    text = text.replace("\ufffd", "?")
+
+    return text

levelapp/config/__init__.py

@@ -0,0 +1,5 @@
+from .endpoint import EndpointConfig
+from .prompts import EVAL_PROMPT_TEMPLATE
+
+
+__all__ = ['EndpointConfig', 'EVAL_PROMPT_TEMPLATE']

levelapp/config/endpoint.py

@@ -0,0 +1,199 @@
+"""levelapp/config/endpoint.py"""
+import os
+import json
+import yaml
+
+from string import Template
+from dotenv import load_dotenv
+
+from enum import Enum
+from typing import Literal, Dict, Any
+from pydantic import BaseModel, HttpUrl, SecretStr, Field, computed_field
+
+from levelapp.aspects import logger
+
+
+class TemplateType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+class EndpointConfig(BaseModel):
+    """
+    Configuration class for user system's endpoint.
+
+    Parameters:
+        base_url (HttpUrl): The base url of the endpoint.
+        method (Literal['POST', 'GET']): The HTTP method to use (POST or GET).
+        api_key (SecretStr): The API key to use.
+        bearer_token (SecretStr): The Bearer token to use.
+        model_id (str): The model to use (if applicable).
+        default_request_payload_template (Dict[str, Any]): The payload template to use.
+        variables (Dict[str, Any]): The variables to populate the payload template.
+
+    Note:
+        Either you use the provided configuration YAML file, providing the following:\n
+        - base_url (HttpUrl): The base url of the endpoint.
+        - method (Literal['POST', 'GET']): The HTTP method to use (POST or GET).
+        - api_key (SecretStr): The API key to use.
+        - bearer_token (SecretStr): The Bearer token to use.
+        - model_id (str): The model to use (if applicable).
+        - default_payload_template (Dict[str, Any]): The payload template to use.
+        - variables (Dict[str, Any]): The variables to populate the payload template.
+
+        Or manually configure the model instance by assigning the proper values to the model fields.\n
+        You can also provide the path in the .env file for the payload template (ENDPOINT_PAYLOAD_PATH/)
+        and the response template (ENDPOINT_RESPONSE_PATH) separately. The files can be either YAML or JSON only.
+    """
+    load_dotenv()
+
+    # Required
+    method: Literal["POST", "GET"] = Field(default="POST")
+    base_url: HttpUrl = Field(default=HttpUrl)
+    url_path: str = Field(default='')
+
+    # Auth
+    api_key: SecretStr | None = Field(default=None)
+    bearer_token: SecretStr | None = Field(default=None)
+    model_id: str | None = Field(default='')
+
+    # Data
+    default_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
+    default_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
+
+    # Variables
+    variables: Dict[str, Any] = Field(default_factory=dict)
+
+    @computed_field()
+    @property
+    def full_url(self) -> str:
+        return str(self.base_url) + self.url_path
+
+    @computed_field()
+    @property
+    def headers(self) -> Dict[str, Any]:
+        headers: Dict[str, Any] = {"Content-Type": "application/json"}
+        if self.model_id:
+            headers["x-model-id"] = self.model_id
+        if self.bearer_token:
+            headers["Authorization"] = f"Bearer {self.bearer_token.get_secret_value()}"
+        if self.api_key:
+            headers["x-api-key"] = self.api_key.get_secret_value()
+        return headers
+
+    @computed_field
+    @property
+    def request_payload(self) -> Dict[str, Any]:
+        """
+        Return fully prepared payload depending on template or full payload.
+
+        Returns:
+            request payload (Dict[str, Any]): Populated request payload template.
+        """
+        # First, we check if we have variables to populate the template with. If not, we return the template as is.
+        if not self.variables:
+            return self.default_request_payload_template
+
+        if not self.default_request_payload_template:
+            base_template = self.load_template(template_type=TemplateType.REQUEST)
+        else:
+            base_template = self.default_request_payload_template
+
+        # Second, replace the placeholders with the variables
+        payload = self._replace_placeholders(obj=base_template, variables=self.variables)
+
+        # Third, merge the "request_payload" if present in variables
+        additional_payload_data = self.variables.get("request_payload", {})
+        if additional_payload_data:
+            payload.update(additional_payload_data)
+
+        self.variables.clear()
+
+        return payload
+
+    @computed_field
+    @property
+    def response_payload(self) -> Dict[str, Any]:
+        if not self.variables:
+            return self.default_response_payload_template
+
+        if not self.default_response_payload_template:
+            base_template = self.load_template(template_type=TemplateType.RESPONSE)
+        else:
+            base_template = self.default_response_payload_template
+
+        response_payload = self._replace_placeholders(obj=base_template, variables=self.variables)
+        self.variables.clear()
+
+        return response_payload
+
+    @staticmethod
+    def _replace_placeholders(obj: Any, variables: Dict[str, Any]) -> Dict[str, Any]:
+        """Recursively replace placeholders in payload template with variables."""
+        def _replace(_obj):
+            if isinstance(_obj, str):
+                subst = Template(_obj).safe_substitute(variables)
+                if '$' in subst:
+                    logger.warning(f"[EndpointConfig] Unsubstituted placeholder in payload:\n{subst}\n\n")
+                return subst
+
+            elif isinstance(_obj, dict):
+                return {k: _replace(v) for k, v in _obj.items()}
+
+            elif isinstance(_obj, list):
+                return [_replace(v) for v in _obj]
+
+            return _obj
+
+        return _replace(obj)
+
+    @staticmethod
+    def load_template(
+            template_type: TemplateType = TemplateType.REQUEST,
+            path: str | None = None
+    ) -> Dict[str, Any]:
+        """
+        Load request/response payload template from JSON/YAML file.
+
+        Args:
+            template_type (TemplateType): The type of template to load (REQUEST or RESPONSE).
+            path (str): The path of the payload template file to load.
+
+        Returns:
+            Payload template (Dict[str, Any]): Payload template.
+        """
+        try:
+            # If no path was provided, we check the env. variables.
+            if not path:
+                env_var = "ENDPOINT_PAYLOAD_PATH" if template_type == TemplateType.REQUEST else "ENDPOINT_RESPONSE_PATH"
+                path = os.getenv(env_var, '')
+
+            if not os.path.exists(path):
+                raise FileNotFoundError(f"The provide payload template file path '{path}' does not exist.")
+
+            with open(path, "r", encoding="utf-8") as f:
+                if path.endswith((".yaml", ".yml")):
+                    data = yaml.safe_load(f)
+
+                elif path.endswith(".json"):
+                    data = json.load(f)
+
+                else:
+                    raise ValueError("[EndpointConfig] Unsupported file format.")
+
+                return data
+
+        except FileNotFoundError as e:
+            raise FileNotFoundError(f"[EndpointConfig] Payload template file '{e.filename}' not found in path.")
+
+        except yaml.YAMLError as e:
+            raise ValueError(f"[EndpointConfig] Error parsing YAML file:\n{e}")
+
+        except json.JSONDecodeError as e:
+            raise ValueError(f"[EndpointConfig] Error parsing JSON file:\n{e}")
+
+        except IOError as e:
+            raise IOError(f"[EndpointConfig] Error reading file:\n{e}")
+
+        except Exception as e:
+            raise ValueError(f"[EndpointConfig] Unexpected error loading configuration:\n{e}")

levelapp/config/prompts.py

@@ -0,0 +1,57 @@
+EVAL_PROMPT_TEMPLATE = """
+You are an impartial evaluator for a conversational system.
+Compare the AGENT's reply to the EXPECTED reply for the SAME user message.
+
+Consider only:
+1) Semantic Coverage — does the AGENT cover the key points in EXPECTED?
+2) Faithfulness — no contradictions or invented details relative to EXPECTED.
+3) Appropriateness — tone/format suitable for the user message.
+Ignore minor wording/punctuation differences. Do NOT reward verbosity.
+
+Scale (integer):
+0 = Poor (misses key points or contradicts)
+1 = Moderate (captures some ideas, noticeable gaps)
+2 = Good (mostly matches, minor omissions/differences)
+3 = Excellent (semantically equivalent; no meaningful differences)
+
+USER_MESSAGE:
+\"\"\"{user_input}\"\"\"
+
+EXPECTED (reference reply):
+\"\"\"{reference_text}\"\"\"
+
+AGENT (model reply):
+\"\"\"{generated_text}\"\"\"
+
+Return ONLY a single JSON object on one line with exactly these keys:
+- "score": <0|1|2|3>,
+- "label": "<Poor|Moderate|Good|Excellent>",
+- "justification": "<1-2 concise sentences>",
+- "evidence":
+    - "covered_points": ["<short phrase>", "..."],   // <=3 items
+    - "missing_or_wrong": ["<short phrase>", "..."]  // <=3 items
+    
+Do NOT include any additional text, explanations, or formatting (e.g., "JSON object:", ```json or ```, or markdown).
+"""
+
+
+SUMMARIZATION_PROMPT_TEMPLATE = """
+You are reviewing evaluation justifications from LLM judges about replies generated by a virtual assistant.
+Interpret the context from the verdicts: (e.g., real-estate leasing, medical appointment scheduling, etc.).
+
+Each justification contains the judge's assessment of how well the assistant's response matched the expected reply.
+Your task is to **identify and summarize only the negative points**, such as:
+- Errors or inaccuracies
+- Misunderstandings or misinterpretations
+- Missing or incomplete information
+- Failure to meet expectations or requirements
+
+**Instructions:**
+- Return up to {max_bullets} concise bullet points.
+- Start each point with "- " and focus on clarity and relevance.
+- Avoid redundancy and prioritize actionable feedback.
+
+---
+- Judge: {judge}
+- Verdicts: {verdicts}
+"""