levelapp 0.1.15__py3-none-any.whl

levelapp/endpoint/parsers.py

@@ -0,0 +1,119 @@
+"""levelapp/endpoint/parsers.py"""
+from typing import List, Dict, Any
+
+from levelapp.endpoint.schemas import RequestSchemaConfig, ResponseMappingConfig
+
+
+class RequestPayloadBuilder:
+    def build(self, schema: List[RequestSchemaConfig], context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Builds nested JSON payloads using dot-notation paths.
+
+        Args:
+            schema (List[RequestSchemaConfig]): List of request schema configurations.
+            context (Dict[str, Any]): Context for building the payload.
+
+        Returns:
+            payload (Dict[str, Any]): Request payload.
+        """
+        payload = {}
+
+        for field_config in schema:
+            value = self._resolve_value(config=field_config, context=context)
+            if value is None and field_config.required:
+                raise ValueError(f"Required field '{field_config.field_path}' has no value")
+
+            self._set_nested_value(obj=payload, path=field_config.field_path, value=value)
+
+        return payload
+
+    @staticmethod
+    def _resolve_value(config: RequestSchemaConfig, context: Dict[str, Any]) -> Any:
+        """
+        Resolve value based on type: static, env, or dynamic.
+
+        Args:
+            config (RequestSchemaConfig): Request schema configuration.
+            context (Dict[str, Any]): Context for building the payload.
+
+        Returns:
+            Any: Value resolved.
+        """
+        if config.value_type == "static":
+            return config.value
+        elif config.value_type == "env":
+            import os
+            return os.getenv(config.value)
+        elif config.value_type == "dynamic":
+            return context.get(config.value, None)
+
+        return config.value
+
+    @staticmethod
+    def _set_nested_value(obj: Dict, path: str, value: Any) -> None:
+        parts: List[str] = path.split(".")
+        for part in parts[:-1]:
+            obj = obj.setdefault(part, {})
+
+        obj[parts[-1]] = value
+
+
+class ResponseDataExtractor:
+    """Extracts data from API response using mapping-based config."""
+    def extract(
+            self,
+            response_data: Dict[str, Any],
+            mappings: List[ResponseMappingConfig]
+    ) -> Dict[str, Any]:
+        """
+        Extracts data from API response using mapping-based config.
+
+        Args:
+            response_data (Dict[str, Any]): API response data.
+            mappings (List[ResponseMappingConfig]): List of response mappings.
+
+        Returns:
+            Dict[str, Any]: Extracted data.
+        """
+        result: Dict[str, Any] = {}
+
+        for mapping in mappings:
+            try:
+                value = self._extract_by_path(obj=response_data, path=mapping.field_path, default=mapping.default)
+                result[mapping.extract_as] = value
+
+            except Exception as e:
+                print(f"Failed to extract '{mapping.field_path}':\n{e}")
+                result[mapping.extract_as] = mapping.default
+
+        return result
+
+    @staticmethod
+    def _extract_by_path(obj: Dict, path: str, default: Any = "N/A") -> Any:
+        """
+        Extracts value using JSON path-like notation.
+        """
+        parts = path.split(".")
+        current = obj
+
+        for part in parts:
+            if not isinstance(current, dict):
+                print("[extract_by_path][WARNING] the response data is not a dict.")
+                return default
+
+            try:
+                if '[' in part and ']' in part:
+                    key, idx = part.split('[')
+                    idx = int(idx.rstrip(']'))
+                    current = current[key][idx] if key else current[idx]
+                else:
+                    if part not in current:
+                        print(f"[extract_by_path][WARNING] Key '{part}' is missing from response.")
+                        return default
+                    current = current.get(part)
+
+            except (KeyError, IndexError, TypeError, AttributeError) as e:
+                print(f"[extract_by_path][ERROR] Error type <{e.__class__.__name__}> : {e.args[0]}")
+                return default
+
+        return current

levelapp/endpoint/schemas.py

@@ -0,0 +1,38 @@
+"""levelapp/endpoint/schemas.py"""
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class HttpMethod(str, Enum):
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    PATCH = "PATCH"
+    DELETE = "DELETE"
+
+
+class HeaderConfig(BaseModel):
+    """Secure header configuration with environment variables support."""
+    name: str
+    value: str
+    secure: bool = False
+
+    class Config:
+        frozen = True
+
+
+class RequestSchemaConfig(BaseModel):
+    """Schema Definition for request payload population."""
+    field_path: str  # JSON path-like: "data.user.id"
+    value: Any
+    value_type: str = "static"
+    required: bool = True
+
+
+class ResponseMappingConfig(BaseModel):
+    """Response data extraction mapping."""
+    field_path: str
+    extract_as: str
+    default: Any = None

levelapp/endpoint/tester.py

@@ -0,0 +1,52 @@
+"""levelapp/endpoint/tester.py"""
+import logging
+from typing import Dict, Any
+
+from levelapp.endpoint.client import EndpointConfig, APIClient
+from levelapp.endpoint.parsers import RequestPayloadBuilder, ResponseDataExtractor
+
+
+class ConnectivityTester:
+    """Tests REST endpoint connectivity with configurable behavior."""
+    def __init__(self, config: EndpointConfig):
+        self.config = config
+        self.client = APIClient(config=config)
+        self.payload_builder = RequestPayloadBuilder()
+        self.response_extractor = ResponseDataExtractor()
+        self.logger = logging.getLogger(f"ConnectivityTester.{self.config.name}")
+
+    async def test(self, context: Dict[str, Any] = None) -> Dict[str, Any]:
+        """Execute connectivity test (template method)."""
+        context = context or {}
+
+        self.logger.info(f"Starting connectivity test for '{self.config.name}'")
+
+        try:
+            payload = None
+            if self.config.request_schema:
+                payload = self.payload_builder.build(schema=self.config.request_schema, context=context)
+                self.logger.debug(f"Request payload: {payload}")
+
+            response = await self.client.execute(payload=payload)
+            self.logger.debug(f"Response status: {response.status_code}")
+
+            response_data = response.json() if response.text else {}
+            extracted = self.response_extractor.extract(
+                response_data=response_data,
+                mappings=self.config.response_mapping,
+            )
+
+            return {
+                "success": True,
+                "status_code": response.status_code,
+                "extracted_data": extracted,
+                "raw_response": response,
+            }
+
+        except Exception as e:
+            self.logger.error(f"Connectivity test failed: {e}", exc_info=e)
+            return {
+                "success": False,
+                "error": str(e),
+                "error_type": type(e).__name__,
+            }

levelapp/evaluator/__init__.py

@@ -0,0 +1,3 @@
+from .evaluator import JudgeEvaluator, MetadataEvaluator
+
+__all__ = ['JudgeEvaluator', 'MetadataEvaluator']

levelapp/evaluator/evaluator.py

@@ -0,0 +1,307 @@
+"""levelapp/core/evaluator.py"""
+from functools import lru_cache
+from typing import List, Dict, Any, TYPE_CHECKING
+from pydantic import BaseModel, Field
+
+from tenacity import (
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+    AsyncRetrying,
+    RetryError,
+)
+
+from levelapp.clients import ClientRegistry
+from levelapp.comparator import MetricsManager, MetadataComparator
+from levelapp.config.prompts import EVAL_PROMPT_TEMPLATE
+from levelapp.core.base import BaseEvaluator, BaseChatClient
+from levelapp.aspects import MonitoringAspect, MetricType, logger, DataLoader
+
+if TYPE_CHECKING:
+    from levelapp.workflow.config import WorkflowConfig
+
+
+class Evidence(BaseModel):
+    """Evidence details for evaluation."""
+    covered_points: List[str] = Field(
+        default_factory=list,
+        description="Key points covered the agent reply covered (<= 3 items)"
+    )
+    missing_or_wrong: List[str] = Field(
+        default_factory=list,
+        description="Key points the agent reply missed or contradicted (<= 3 items)"
+    )
+
+
+class JudgeEvaluationResults(BaseModel):
+    """Structured result of an interaction evaluation."""
+    provider: str = Field(..., description="The provider name, e.g., 'openai', 'ionos'")
+    score: int = Field(..., ge=0, le=3, description="Evaluation score between 0 and 3")
+    label: str = Field(..., description="The label of the evaluation result")
+    justification: str = Field(..., description="Short explanation of the evaluation result")
+    evidence: Evidence = Field(default_factory=Evidence, description="Detailed evidence for the evaluation")
+    raw_response: Dict[str, Any] = Field(..., description="Full unprocessed API response", exclude=True)
+    metadata: Dict[str, Any] = Field(..., description="Metadata about the evaluation result")
+
+    @classmethod
+    def from_parsed(cls, provider: str, parsed: Dict[str, Any], raw: Dict[str, Any]) -> "JudgeEvaluationResults":
+        """
+        Build a model instance from the provided data.
+
+        Args:
+            provider (str): The provider name.
+            parsed (Dict[str, Any]): The parsed response data.
+            raw (Dict[str, Any]): The raw response data.
+
+        Returns:
+            JudgeEvaluationResults: The constructed evaluation result instance.
+        """
+        content = parsed.get("output", {})
+        metadata = parsed.get("metadata", {})
+        return cls(
+            provider=provider,
+            score=content.get("score", 0),
+            label=content.get("label", "N/A"),
+            justification=content.get("justification", "N/A"),
+            evidence=Evidence(**content.get("evidence", {})),
+            raw_response=raw,
+            metadata=metadata,
+        )
+
+
+class JudgeEvaluator(BaseEvaluator):
+    """LLM-as-a-judge evaluator class"""
+    def __init__(self, config: "WorkflowConfig | None" = None):
+        """
+        Initialize the JudgeEvaluator.
+
+        Args:
+            config (WorkflowConfig | None): The configuration of the workflow.
+        """
+        if config:
+            self.config = config
+            self.providers = config.evaluation.providers
+
+        self.prompt_template = EVAL_PROMPT_TEMPLATE
+        self.client_registry = ClientRegistry
+
+    def select_client(self, provider: str) -> BaseChatClient:
+        """
+        Select an LLM client to use for the evaluation.
+
+        Args:
+            provider (str): The provider name.
+
+        Returns:
+            client (BaseChatClient): The LLM client to use for the evaluation.
+        """
+        if provider not in self.client_registry.list_providers():
+            logger.warning(f"[JudgeEvaluator] {provider} is not registered. Defaulting to 'OpenAI'.")
+            return self.client_registry.get(provider="openai")
+
+        return self.client_registry.get(provider=provider)
+
+    @lru_cache(maxsize=1024)
+    def _build_prompt(self, user_input: str, generated_text: str, reference_text: str) -> str:
+        """
+        Build the prompt used for the evaluation.
+
+        Args:
+            user_input (str): The user input.
+            generated_text (str): The generated text.
+            reference_text (str): The reference text.
+
+        Returns:
+            A string containing the prompt.
+        """
+        return self.prompt_template.format(
+            user_input=user_input,
+            generated_text=generated_text,
+            reference_text=reference_text
+        )
+
+    @retry(
+        retry=retry_if_exception_type((TimeoutError, ValueError, RuntimeError)),
+        stop=stop_after_attempt(3),
+        wait=wait_exponential(multiplier=1, min=2, max=10),
+        reraise=True,
+    )
+    def evaluate(
+            self,
+            generated_data: str,
+            reference_data: str,
+            user_input: str,
+            provider: str,
+    ) -> JudgeEvaluationResults | None:
+        """
+        Synchronous evaluation for the generated data.
+
+        Args:
+            generated_data (str): The generated data.
+            reference_data (str): The reference data.
+            user_input (str): The user input.
+            provider (str): The LLM provider user for evaluation.
+
+        Returns:
+            JudgeEvaluationResults instance containing the evaluation results.
+
+        Raises:
+            Exception: If the evaluation failed.
+        """
+        prompt = self._build_prompt(
+            user_input=user_input,
+            generated_text=generated_data,
+            reference_text=reference_data
+        )
+        client = self.select_client(provider=provider)
+
+        try:
+            response = client.call(message=prompt)
+            logger.info(f"[{provider}] Evaluation: {response}\n{'---' * 10}")
+            parsed = client.parse_response(response=response)
+            return JudgeEvaluationResults.from_parsed(provider=provider, parsed=parsed, raw=response)
+
+        except Exception as e:
+            logger.error(f"[{provider}] Evaluation failed: {e}", exc_info=True)
+            return JudgeEvaluationResults(
+                provider=provider,
+                score=0,
+                label="N/A",
+                justification="N/A",
+                evidence=Evidence(covered_points=[], missing_or_wrong=[]),
+                raw_response={},
+                metadata={}
+            )
+
+    @MonitoringAspect.monitor(name="judge_evaluation", category=MetricType.API_CALL)
+    async def async_evaluate(
+            self,
+            generated_data: str,
+            reference_data: str,
+            user_input: str,
+            provider: str,
+    ) -> JudgeEvaluationResults | None:
+        """
+        Synchronous evaluation for the generated data.
+
+        Args:
+            generated_data (str): The generated data.
+            reference_data (str): The reference data.
+            user_input (str): The user input.
+            provider (str): The LLM provider user for evaluation.
+
+        Returns:
+            JudgeEvaluationResults instance containing the evaluation results.
+
+        Raises:
+            RetryError: If the evaluation failed.
+        """
+        prompt = self._build_prompt(
+            user_input=user_input,
+            generated_text=generated_data,
+            reference_text=reference_data
+        )
+        client = self.select_client(provider=provider)
+
+        try:
+            async for attempt in AsyncRetrying(
+                retry=retry_if_exception_type((TimeoutError, ValueError, RuntimeError)),
+                stop=stop_after_attempt(3),
+                wait=wait_exponential(multiplier=1, min=2, max=10),
+                reraise=True,
+            ):
+                with attempt:
+                    response = await client.acall(message=prompt)
+                    parsed = client.parse_response(response=response)
+                    return JudgeEvaluationResults.from_parsed(provider=provider, parsed=parsed, raw=response)
+
+        except RetryError as e:
+            logger.error(f"[{provider}] Async evaluation failed after retries: {e}", exc_info=True)
+            return JudgeEvaluationResults(
+                provider=provider,
+                score=0,
+                label="N/A",
+                justification="N/A",
+                evidence=Evidence(covered_points=[], missing_or_wrong=[]),
+                raw_response={},
+                metadata={}
+            )
+
+
+class MetadataEvaluator(BaseEvaluator):
+    """Metadata evaluator class."""
+    def __init__(self, config: "WorkflowConfig | None" = None):
+        """
+        Initialize the MetadataEvaluator.
+
+        Args:
+            config (WorkflowConfig | None): The workflow configuration.
+        """
+        if config:
+            self.config = config
+            self.metrics_map = config.evaluation.metrics_map
+
+        self.data_loader = DataLoader()
+        self.comparator = MetadataComparator()
+        self.metrics_manager = MetricsManager()
+
+    def evaluate(
+            self,
+            generated_data: str | Dict[str, Any],
+            reference_data: str | Dict[str, Any],
+            metrics_mapping: Any | None = None,
+    ) -> Dict[str, float]:
+        """
+        Synchronous evaluation for the generated data.
+
+        Args:
+              generated_data (str): The generated data.
+              reference_data (str): The reference data.
+              metrics_mapping (dict): A dictionary mapping metric names to metrics.
+
+        Returns:
+              A dict containing the evaluation results.
+        """
+        gen_data = self.data_loader.create_dynamic_model(data=generated_data, model_name="GeneratedMetadata")
+        ref_data = self.data_loader.create_dynamic_model(data=reference_data, model_name="ReferenceMetadata")
+
+        if metrics_mapping:
+            self.comparator.metrics_manager = metrics_mapping
+        else:
+            logger.info(f"[MetadataEvaluator] Metric map: {self.metrics_map}")
+            self.comparator.metrics_manager = self.metrics_map
+
+        self.comparator.metrics_manager = self.metrics_manager
+        self.comparator.generated_data = gen_data
+        self.comparator.reference_data = ref_data
+
+        output = self.comparator.run(indexed_mode=False)
+        results: Dict[str, float] = {}
+        logger.info(f"[MetadataEvaluator] Metadata Evaluation Output:\n{output}]")
+
+        for k, v in output.items():
+            field = v.get("field_name", "N/A")
+            score = v.get("set_scores", -1)
+
+            if score is None:
+                results[field] = -1
+                continue
+
+            try:
+                val = score[0] if isinstance(score, list) else score
+                results[field] = float(val)
+
+            except (TypeError, ValueError):
+                results[field] = -1
+
+        return results
+
+    async def async_evaluate(
+            self,
+            generated_data: str | Dict[str, Any],
+            reference_data: str | Dict[str, Any],
+            **kwargs
+    ):
+        """Not implemented yet."""
+        raise NotImplementedError()

levelapp/metrics/__init__.py

@@ -0,0 +1,63 @@
+"""levelapp/metrics/__init__.py"""
+from typing import List, Dict, Type, Any
+
+from levelapp.aspects import logger
+from levelapp.core.base import BaseMetric
+from levelapp.metrics.exact import EXACT_METRICS
+from levelapp.metrics.fuzzy import FUZZY_METRICS
+
+
+class MetricRegistry:
+    """Registry for metric classes."""
+    _metrics: Dict[str, Type[BaseMetric]] = {}
+
+    @classmethod
+    def register(cls, name: str, metric_class: Type[BaseMetric]) -> None:
+        """
+        Register a metric class under a given name.
+
+        Args:
+            name (str): Unique identifier for the metric.
+            metric_class (Type[BaseMetric]): The metric class to register.
+        """
+        if name in cls._metrics:
+            raise KeyError(f"Metric '{name}' is already registered")
+
+        cls._metrics[name] = metric_class
+
+    @classmethod
+    def get(cls, name: str, **kwargs: Any) -> BaseMetric:
+        """
+        Retrieve an instance of a registered metric by its name.
+
+        Args:
+            name (str): The name of the metric to retrieve.
+
+        Returns:
+            Type[BaseMetric]: The metric class associated with the given name.
+
+        Raises:
+            KeyError: If the metric is not found.
+        """
+        if name not in cls._metrics:
+            raise KeyError(f"Metric '{name}' is not registered")
+
+        return cls._metrics[name](**kwargs)
+
+    @classmethod
+    def list_metrics(cls) -> List[str]:
+        return list(cls._metrics.keys())
+
+    @classmethod
+    def unregister(cls, name: str) -> None:
+        cls._metrics.pop(name, None)
+
+
+METRICS = FUZZY_METRICS | EXACT_METRICS
+
+for name_, metric_class_ in METRICS.items():
+    try:
+        MetricRegistry.register(name_, metric_class_)
+
+    except Exception as e:
+        logger.info(f"Failed to register metric {name_}: {e}")

levelapp/metrics/embedding.py

@@ -0,0 +1,56 @@
+"""levelapp/metrics/embeddings.py"""
+from __future__ import annotations
+
+import importlib
+
+from importlib import util
+from typing import Any, Dict
+
+from levelapp.core.base import BaseMetric
+
+
+class EmbeddingMetric(BaseMetric):
+    """
+    Abstract embeddings metric that dynamically delegates to a backend implementation (Torch or Scikit).
+    """
+    def __init__(self, backend: str | None = None, **kwargs: Any):
+        """
+        Initialize the embeddings metric.
+
+        Args:
+            backend (str, optional): Embedding metric backend 'torch' or 'scikit'. Defaults to None.
+        """
+        super().__init__(processor=kwargs.get("processor"), score_cutoff=kwargs.get("score_cutoff"))
+        self.backend_name = backend or self._detect_backend()
+        self.backend = self._load_backend(self.backend_name)(**kwargs)
+
+    @staticmethod
+    def _detect_backend() -> str:
+        """Auto-detect which embeddings backend to use."""
+        if util.find_spec("torch") and util.find_spec("transformers"):
+            return "torch"
+
+        elif util.find_spec("sklearn"):
+            return "scikit"
+
+        raise ImportError(
+            "No embeddings backend available. Install with 'pip install levelapp[embeddings]' "
+            "for Torch support, or ensure scikit-learn is installed."
+        )
+
+    @staticmethod
+    def _load_backend(backend: str):
+        if backend == "torch":
+            module = importlib.import_module("levelapp.metrics.embeddings.torch_based")
+            return getattr(module, "TorchEmbeddingMetric")
+
+        elif backend == "scikit":
+            module = importlib.import_module("levelapp.metrics.embeddings.sentence_transformer")
+            return getattr(module, "SentenceEmbeddingMetric")
+
+        else:
+            raise ValueError(f"Unknown embeddings backend: {backend}")
+
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        """Delegate to selected backend implementation."""
+        return self.backend.compute(generated, reference)

levelapp/metrics/embeddings/sentence_transformer.py

@@ -0,0 +1,30 @@
+"""levelapp/metrics/embeddings/sentence_transformer.py"""
+import numpy as np
+
+from typing import Any, Dict
+
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from levelapp.core.base import BaseMetric
+
+
+class SentenceEmbeddingMetric(BaseMetric):
+    """Lightweight embeddings similarity using TF-IDF cosine similarity."""
+    def __init__(self, **kwargs):
+        super().__init__(processor=kwargs.get("processor"), score_cutoff=kwargs.get("score_cutoff"))
+        self.vectorizer = TfidfVectorizer()
+
+    def compute(self, generated: str, reference: str) -> Dict[str, Any]:
+        self._validate_inputs(generated=generated, reference=reference)
+
+        corpus = [reference, generated]
+        tfidf_matrix = self.vectorizer.fit_transform(corpus)
+        similarity = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])[0][0]
+        # clamping for numerical stability
+        similarity = float(np.clip(similarity, 0.0, 1.0))
+
+        return {
+            "similarity": similarity,
+            "metadata": self._build_metadata(backend="scikit", vectorizer="TF-IDF"),
+        }