levelapp 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

levelapp/aspects/monitor.py

@@ -343,6 +343,7 @@ class FunctionMonitor:
             category: MetricType,
             enable_timing: bool,
             track_memory: bool,
+            verbose=False
     ) -> Callable[P, T]:
         """
         Wrap function execution with timing and error handling.
@@ -352,6 +353,7 @@ class FunctionMonitor:
             name: Unique identifier for the function
             enable_timing: Enable execution time logging
             track_memory: Enable memory tracking
+            verbose: Enable verbose logging
 
         Returns:
             Wrapped function
@@ -402,7 +404,7 @@ class FunctionMonitor:
 
                     self._aggregated_stats[name].update(metrics=metrics)
 
-                if enable_timing and metrics.duration is not None:
+                if verbose and enable_timing and metrics.duration is not None:
                     log_message = f"[FunctionMonitor] Executed '{name}' in {metrics.duration:.4f}s"
                     if metrics.cache_hit:
                         log_message += " (cache hit)"

levelapp/clients/__init__.py

@@ -44,7 +44,6 @@ class ClientRegistry:
 
         cls._wrap_client_methods(client_class)
         cls._clients[provider] = client_class
-        logger.info(f"[ClientRegistry] Registered client for provider: {provider}")
 
     @classmethod
     def _wrap_client_methods(cls, client_class: Type[BaseChatClient]) -> None:

levelapp/comparator/scorer.py

@@ -78,7 +78,6 @@ class MetricsManager:
             ValueError: if the scorer is not a callable.
         """
         self._scorers[name] = scorer
-        logger.info(f"[MetricsManager] Registered scorer: {name}")
 
     def get_scorer(self, name: str) -> Callable:
         """
@@ -95,7 +94,6 @@ class MetricsManager:
         """
         try:
             scorer = self._scorers.get(name)
-            logger.info(f"[get_scorer] Retrieved scorer: {name}")
             return scorer
 
         except KeyError:

levelapp/config/endpoint.py

@@ -29,7 +29,6 @@ class EndpointConfig(BaseModel):
         bearer_token (SecretStr): The Bearer token to use.
         model_id (str): The model to use (if applicable).
         default_request_payload_template (Dict[str, Any]): The payload template to use.
-        generated_request_payload_template (Dict[str, Any]): The generated payload template from a provided file.
         variables (Dict[str, Any]): The variables to populate the payload template.
 
     Note:
@@ -40,11 +39,10 @@ class EndpointConfig(BaseModel):
         - bearer_token (SecretStr): The Bearer token to use.
         - model_id (str): The model to use (if applicable).
         - default_payload_template (Dict[str, Any]): The payload template to use.
-        - generated_payload_template (Dict[str, Any]): The generated payload template from a provided file.
         - variables (Dict[str, Any]): The variables to populate the payload template.
 
         Or manually configure the model instance by assigning the proper values to the model fields.\n
-        You can also provide the path in the .env file for the payload template (ENDPOINT_PAYLOAD_PATH)
+        You can also provide the path in the .env file for the payload template (ENDPOINT_PAYLOAD_PATH/)
         and the response template (ENDPOINT_RESPONSE_PATH) separately. The files can be either YAML or JSON only.
     """
     load_dotenv()
@@ -61,9 +59,7 @@ class EndpointConfig(BaseModel):
 
     # Data
     default_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
-    generated_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
     default_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
-    generated_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
 
     # Variables
     variables: Dict[str, Any] = Field(default_factory=dict)
@@ -88,14 +84,18 @@ class EndpointConfig(BaseModel):
     @computed_field
     @property
     def request_payload(self) -> Dict[str, Any]:
-        """Return fully prepared payload depending on template or full payload."""
-        # First, load the request payload template (either from YAML config file or from specific template)
+        """
+        Return fully prepared payload depending on template or full payload.
+
+        Returns:
+            request payload (Dict[str, Any]): Populated request payload template.
+        """
+        # First, we check if we have variables to populate the template with. If not, we return the template as is.
         if not self.variables:
             return self.default_request_payload_template
 
         if not self.default_request_payload_template:
-            self.load_template(template_type=TemplateType.REQUEST)
-            base_template = self.generated_request_payload_template
+            base_template = self.load_template(template_type=TemplateType.REQUEST)
         else:
             base_template = self.default_request_payload_template
 
@@ -118,8 +118,7 @@ class EndpointConfig(BaseModel):
             return self.default_response_payload_template
 
         if not self.default_response_payload_template:
-            self.load_template(template_type=TemplateType.RESPONSE)
-            base_template = self.generated_response_payload_template
+            base_template = self.load_template(template_type=TemplateType.RESPONSE)
         else:
             base_template = self.default_response_payload_template
 
@@ -148,12 +147,23 @@ class EndpointConfig(BaseModel):
 
         return _replace(obj)
 
+    @staticmethod
     def load_template(
-            self,
             template_type: TemplateType = TemplateType.REQUEST,
             path: str | None = None
     ) -> Dict[str, Any]:
+        """
+        Load request/response payload template from JSON/YAML file.
+
+        Args:
+            template_type (TemplateType): The type of template to load (REQUEST or RESPONSE).
+            path (str): The path of the payload template file to load.
+
+        Returns:
+            Payload template (Dict[str, Any]): Payload template.
+        """
         try:
+            # If no path was provided, we check the env. variables.
             if not path:
                 env_var = "ENDPOINT_PAYLOAD_PATH" if template_type == TemplateType.REQUEST else "ENDPOINT_RESPONSE_PATH"
                 path = os.getenv(env_var, '')
@@ -171,7 +181,6 @@ class EndpointConfig(BaseModel):
                 else:
                     raise ValueError("[EndpointConfig] Unsupported file format.")
 
-                self.generated_request_payload_template = data
                 return data
 
         except FileNotFoundError as e:

levelapp/config/endpoint_.py

@@ -0,0 +1,62 @@
+from abc import ABC
+from enum import Enum
+from typing import Any, List
+
+from pydantic import BaseModel, Field
+
+
+class HttpMethod(str, Enum):
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    Patch = "PATCH"
+    DELETE = "DELETE"
+
+
+class HeaderConfig(BaseModel):
+    """Secure header configuration with environment variables support."""
+    name: str
+    value: str
+    secure: bool = False
+
+    class Config:
+        frozen = True
+
+
+class RequestSchemaConfig(BaseModel):
+    """Schema definition for request payload population."""
+    field_path: str  # JSON path-like: "data.user.id"
+    value: Any
+    value_type: str = "static"  # static, env, dynamic
+    required: bool = True
+
+
+class ResponseMappingConfig(BaseModel):
+    """Response data extraction mapping."""
+    field_path: str  # JSON path-like: "data.results[0].id"
+    extract_as: str  # Name to extract as
+    default: Any = None
+
+
+class EndpointConfig(BaseModel):
+    """Complete endpoint configuration."""
+    name: str
+    base_url: str
+    path: str
+    method: HttpMethod
+    headers: List[HeaderConfig] = Field(default_factory=list)
+    request_schema: List[RequestSchemaConfig] = Field(default_factory=list)
+    response_mapping: List[ResponseMappingConfig] = Field(default_factory=list)
+    timeout: int = 30
+    retry_count: int = 3
+    retry_backoff: float = 1.0
+
+    @classmethod
+    def validate_path(cls, v: str) -> str:
+        if not v.startswith('/'):
+            return f'/{v}'
+        return v
+
+
+class PayloadBuilder(ABC):
+    """Abstract base for payload construction strategies."""

levelapp/config/prompts.py

@@ -33,3 +33,25 @@ Return ONLY a single JSON object on one line with exactly these keys:
     
 Do NOT include any additional text, explanations, or formatting (e.g., "JSON object:", ```json or ```, or markdown).
 """
+
+
+SUMMARIZATION_PROMPT_TEMPLATE = """
+You are reviewing evaluation justifications from LLM judges about replies generated by a virtual assistant.
+Interpret the context from the verdicts: (e.g., real-estate leasing, medical appointment scheduling, etc.).
+
+Each justification contains the judge's assessment of how well the assistant's response matched the expected reply.
+Your task is to **identify and summarize only the negative points**, such as:
+- Errors or inaccuracies
+- Misunderstandings or misinterpretations
+- Missing or incomplete information
+- Failure to meet expectations or requirements
+
+**Instructions:**
+- Return up to {max_bullets} concise bullet points.
+- Start each point with "- " and focus on clarity and relevance.
+- Avoid redundancy and prioritize actionable feedback.
+
+---
+- Judge: {judge}
+- Verdicts: {verdicts}
+"""

levelapp/core/schemas.py

@@ -0,0 +1,24 @@
+from enum import Enum
+
+
+class ExtendedEnum(Enum):
+    @classmethod
+    def list(cls):
+        return [e.value for e in cls]
+
+
+class WorkflowType(ExtendedEnum):
+    SIMULATOR = "SIMULATOR"
+    COMPARATOR = "COMPARATOR"
+    ASSESSOR = "ASSESSOR"
+
+
+class RepositoryType(ExtendedEnum):
+    FIRESTORE = "FIRESTORE"
+    FILESYSTEM = "FILESYSTEM"
+
+
+class EvaluatorType(ExtendedEnum):
+    JUDGE = "JUDGE"
+    REFERENCE = "REFERENCE"
+    RAG = "RAG"

levelapp/core/session.py

@@ -1,27 +1,23 @@
 """levelapp/core/session.py"""
 import threading
 
+from abc import ABC
+
 from dataclasses import dataclass, field
 from typing import Dict, List, Any
 
 from datetime import datetime
 from humanize import precisedelta
 
-from levelapp.workflow import MainFactory
+from levelapp.workflow import MainFactory, WorkflowConfig
 from levelapp.workflow.base import BaseWorkflow
-from levelapp.workflow.schemas import WorkflowConfig, WorkflowContext
-from levelapp.aspects import FunctionMonitor, MetricType, ExecutionMetrics, MonitoringAspect, logger
+from levelapp.aspects import MetricType, ExecutionMetrics, MonitoringAspect, logger
+from levelapp.workflow.context import WorkflowContextBuilder
 
 
-@dataclass
-class SessionMetadata:
-    """Metadata for an evaluation session."""
-    session_name: str
-    started_at: datetime | None = None
-    ended_at: datetime | None = None
-    total_executions: int = 0
-    total_duration: float = 0.0
-    steps: Dict[str, 'StepMetadata'] = field(default_factory=dict)
+class TemporalStatusMixin(ABC):
+    started_at: datetime | None
+    ended_at: datetime | None
 
     @property
     def is_active(self) -> bool:
@@ -37,7 +33,18 @@ class SessionMetadata:
 
 
 @dataclass
-class StepMetadata:
+class SessionMetadata(TemporalStatusMixin):
+    """Metadata for an evaluation session."""
+    session_name: str
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    total_executions: int = 0
+    total_duration: float = 0.0
+    steps: Dict[str, 'StepMetadata'] = field(default_factory=dict)
+
+
+@dataclass
+class StepMetadata(TemporalStatusMixin):
     """Metadata for a specific step within an evaluation session."""
     step_name: str
     session_name: str
@@ -47,27 +54,29 @@ class StepMetadata:
     error_count: int = 0
     procedures_stats: List[ExecutionMetrics] | None = None
 
-    @property
-    def is_active(self) -> bool:
-        """Check if the step is currently active."""
-        return self.ended_at is None
-
-    @property
-    def duration(self) -> float | None:
-        """Calculate the duration of the step in seconds."""
-        if not self.is_active:
-            return (self.ended_at - self.started_at).total_seconds()
-        return None
-
 
 class StepContext:
     """Context manager for an evaluation step within an EvaluationSession."""
-    def __init__(self, session: "EvaluationSession", step_name: str, category: MetricType):
+    def __init__(
+            self,
+            session: "EvaluationSession",
+            step_name: str,
+            category: MetricType,
+    ):
+        """
+        Initialize StepContext.
+
+        Args:
+            session (EvaluationSession): Evaluation session.
+            step_name (str): Step name.
+            category (MetricType): Metric type.
+        """
         self.session = session
         self.step_name = step_name
         self.category = category
+
         self.step_meta: StepMetadata | None = None
-        self.full_step_name = f"{session.session_name}.{step_name}"
+        self.full_step_name = f"<{session.session_name}:{step_name}>"
         self._monitored_func = None
         self._func_gen = None
 
@@ -80,35 +89,50 @@ class StepContext:
             )
             self.session.session_metadata.steps[self.step_name] = self.step_meta
 
-        # Wrap with FunctionMonitor
-        self._monitored_func = self.session.monitor.monitor(
-            name=self.full_step_name,
-            category=self.category,
-            enable_timing=True,
-            track_memory=True,
-        )(self._step_wrapper)
-
-        # Start monitoring
-        self._func_gen = self._monitored_func()
-        next(self._func_gen)  # Enter monitoring
+        if self.session.enable_monitoring:
+            # Wrap with FunctionMonitor
+            self._monitored_func = self.session.monitor.monitor(
+                name=self.full_step_name,
+                category=self.category,
+                enable_timing=True,
+                track_memory=True,
+                verbose=self.session.verbose,
+            )(self._step_wrapper)
+
+            # Start monitoring
+            try:
+                self._func_gen = self._monitored_func()
+                next(self._func_gen)  # Enter monitoring
+            except Exception as e:
+                logger.error(f"[StepContext] Failed to initialize monitoring for {self.full_step_name}:\n{e}")
+                raise
+
         return self  # returning self allows nested instrumentation
 
+    # noinspection PyMethodMayBeStatic
     def _step_wrapper(self):
         yield  # Actual user step execution happens here
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            next(self._func_gen)  # Exit monitoring
-        except StopIteration:
-            pass
+        if self.session.enable_monitoring:
+            try:
+                next(self._func_gen)  # Exit monitoring
+            except StopIteration:
+                pass
 
         with self.session.lock:
             self.step_meta.ended_at = datetime.now()
+
             if exc_type:
                 self.step_meta.error_count += 1
+
             self.session.session_metadata.total_executions += 1
-            if self.step_meta.duration:
-                self.session.monitor.update_procedure_duration(name=self.full_step_name, value=self.step_meta.duration)
+
+            if self.session.enable_monitoring and self.step_meta.duration:
+                self.session.monitor.update_procedure_duration(
+                    name=self.full_step_name,
+                    value=self.step_meta.duration
+                )
                 self.session.session_metadata.total_duration += self.step_meta.duration
 
         return False
@@ -119,29 +143,34 @@ class EvaluationSession:
     def __init__(
             self,
             session_name: str = "test-session",
-            monitor: FunctionMonitor | None = None,
-            workflow_config: WorkflowConfig | None = None
+            workflow_config: WorkflowConfig | None = None,
+            enable_monitoring: bool = True,
+            verbose: bool = False
     ):
         """
         Initialize Evaluation Session.
 
         Args:
             session_name (str): Name of the session
-            monitor (FunctionMonitor): Function monitoring aspect
             workflow_config (WorkflowConfig): Workflow configuration.
+            enable_monitoring (bool): Switch monitoring on. Defaults to True.
+            verbose (bool): Verbose mode. Defaults to False.
         """
         self._NAME = self.__class__.__name__
 
         self.session_name = session_name
-        self.monitor = monitor or MonitoringAspect
         self.workflow_config = workflow_config
-        self.workflow_type = workflow_config.workflow
+        self.enable_monitoring = enable_monitoring
+        self.verbose = verbose
 
         self.workflow: BaseWorkflow | None = None
 
         self.session_metadata = SessionMetadata(session_name=session_name)
+        self.monitor = MonitoringAspect if enable_monitoring else None
         self._lock = threading.RLock()
 
+        logger.info("[EvaluationSession] Evaluation session initialized.")
+
     @property
     def lock(self):
         return self._lock
@@ -154,17 +183,13 @@ class EvaluationSession:
             if not self.workflow_config:
                 raise ValueError(f"{self._NAME}: Workflow configuration must be provided")
 
-            context = WorkflowContext(
-                config=self.workflow_config,
-                repository=MainFactory.create_repository(self.workflow_config),
-                evaluators=MainFactory.create_evaluator(self.workflow_config),
-                endpoint_config=self.workflow_config.endpoint_config,
-                inputs=self.workflow_config.inputs
-            )
-            self.workflow = MainFactory.create_workflow(self.workflow_type, context)
+            context_builder = WorkflowContextBuilder(self.workflow_config)
+            context = context_builder.build()
+
+            self.workflow = MainFactory.create_workflow(context=context)
 
         logger.info(
-            f"[{self._NAME}] Starting evaluation session: {self.session_name}, "
+            f"[{self._NAME}] Starting evaluation session: {self.session_name} - "
             f"Workflow: '{self.workflow.name}'"
         )
         return self
@@ -178,6 +203,7 @@ class EvaluationSession:
 
         if exc_type:
             logger.error(f"[{self._NAME}] Session ended with error: {exc_val}", exc_info=True)
+
         return False
 
     def step(self, step_name: str, category: MetricType = MetricType.CUSTOM) -> StepContext:
@@ -201,6 +227,19 @@ class EvaluationSession:
             self.workflow.collect_results()
 
     def get_stats(self) -> Dict[str, Any]:
+        if self.enable_monitoring:
+            return {
+                "session": {
+                    "name": self.session_name,
+                    "duration": precisedelta(self.session_metadata.duration, suppress=['minutes']),
+                    "start_time": self.session_metadata.started_at.isoformat(),
+                    "end_time": self.session_metadata.ended_at.isoformat(),
+                    "steps": len(self.session_metadata.steps),
+                    "errors": sum(s.error_count for s in self.session_metadata.steps.values())
+                },
+                "stats": self.monitor.get_all_stats()
+            }
+
         return {
             "session": {
                 "name": self.session_name,
@@ -210,5 +249,4 @@ class EvaluationSession:
                 "steps": len(self.session_metadata.steps),
                 "errors": sum(s.error_count for s in self.session_metadata.steps.values())
             },
-            "stats": self.monitor.get_all_stats()
         }

levelapp/evaluator/evaluator.py

@@ -1,7 +1,6 @@
 """levelapp/core/evaluator.py"""
 from functools import lru_cache
-from typing import List, Dict, Any
-from collections import defaultdict
+from typing import List, Dict, Any, TYPE_CHECKING
 from pydantic import BaseModel, Field
 
 from tenacity import (
@@ -19,6 +18,9 @@ from levelapp.config.prompts import EVAL_PROMPT_TEMPLATE
 from levelapp.core.base import BaseEvaluator, BaseChatClient
 from levelapp.aspects import MonitoringAspect, MetricType, logger, DataLoader
 
+if TYPE_CHECKING:
+    from levelapp.workflow.config import WorkflowConfig
+
 
 class Evidence(BaseModel):
     """Evidence details for evaluation."""
@@ -39,7 +41,7 @@ class JudgeEvaluationResults(BaseModel):
     label: str = Field(..., description="The label of the evaluation result")
     justification: str = Field(..., description="Short explanation of the evaluation result")
     evidence: Evidence = Field(default_factory=Evidence, description="Detailed evidence for the evaluation")
-    raw_response: Dict[str, Any] = Field(..., description="Full unprocessed API response")
+    raw_response: Dict[str, Any] = Field(..., description="Full unprocessed API response", exclude=True)
     metadata: Dict[str, Any] = Field(..., description="Metadata about the evaluation result")
 
     @classmethod
@@ -69,19 +71,36 @@ class JudgeEvaluationResults(BaseModel):
 
 
 class JudgeEvaluator(BaseEvaluator):
-    def __init__(self):
+    """LLM-as-a-judge evaluator class"""
+    def __init__(self, config: "WorkflowConfig | None" = None):
+        """
+        Initialize the JudgeEvaluator.
+
+        Args:
+            config (WorkflowConfig | None): The configuration of the workflow.
+        """
+        if config:
+            self.config = config
+            self.providers = config.evaluation.providers
+
         self.prompt_template = EVAL_PROMPT_TEMPLATE
-        self.clients = defaultdict(BaseChatClient)
+        self.client_registry = ClientRegistry
 
-    def register_client(self, provider: str, client: BaseChatClient) -> None:
+    def select_client(self, provider: str) -> BaseChatClient:
         """
-        Register LLM clients used for the evaluation.
+        Select an LLM client to use for the evaluation.
 
         Args:
             provider (str): The provider name.
-            client (BaseChatClient): The LLM client to register.
+
+        Returns:
+            client (BaseChatClient): The LLM client to use for the evaluation.
         """
-        self.clients[provider] = client
+        if provider not in self.client_registry.list_providers():
+            logger.warning(f"[JudgeEvaluator] {provider} is not registered. Defaulting to 'OpenAI'.")
+            return self.client_registry.get(provider="openai")
+
+        return self.client_registry.get(provider=provider)
 
     @lru_cache(maxsize=1024)
     def _build_prompt(self, user_input: str, generated_text: str, reference_text: str) -> str:
@@ -135,7 +154,7 @@ class JudgeEvaluator(BaseEvaluator):
             generated_text=generated_data,
             reference_text=reference_data
         )
-        client = ClientRegistry.get(provider=provider)
+        client = self.select_client(provider=provider)
 
         try:
             response = client.call(message=prompt)
@@ -183,7 +202,7 @@ class JudgeEvaluator(BaseEvaluator):
             generated_text=generated_data,
             reference_text=reference_data
         )
-        client = ClientRegistry.get(provider=provider)
+        client = self.select_client(provider=provider)
 
         try:
             async for attempt in AsyncRetrying(
@@ -194,7 +213,6 @@ class JudgeEvaluator(BaseEvaluator):
             ):
                 with attempt:
                     response = await client.acall(message=prompt)
-                    logger.info(f"[{provider}] Async evaluation:\n{response}\n{'---' * 10}")
                     parsed = client.parse_response(response=response)
                     return JudgeEvaluationResults.from_parsed(provider=provider, parsed=parsed, raw=response)
 
@@ -212,7 +230,18 @@ class JudgeEvaluator(BaseEvaluator):
 
 
 class MetadataEvaluator(BaseEvaluator):
-    def __init__(self):
+    """Metadata evaluator class."""
+    def __init__(self, config: "WorkflowConfig | None" = None):
+        """
+        Initialize the MetadataEvaluator.
+
+        Args:
+            config (WorkflowConfig | None): The workflow configuration.
+        """
+        if config:
+            self.config = config
+            self.metics_map = config.evaluation.metrics_map
+
         self.data_loader = DataLoader()
         self.comparator = MetadataComparator()
         self.metrics_manager = MetricsManager()
@@ -245,7 +274,6 @@ class MetadataEvaluator(BaseEvaluator):
         self.comparator.reference_data = ref_data
 
         output = self.comparator.run(indexed_mode=False)
-        logger.info(f"Comparison results:\n{output}\n---")
         results: Dict[str, float] = {}
 
         for k, v in output.items():

levelapp/metrics/__init__.py

@@ -1,14 +1,11 @@
 """levelapp/metrics/__init__.py"""
-import logging
-
 from typing import List, Dict, Type, Any
 
+from levelapp.aspects import logger
 from levelapp.core.base import BaseMetric
 from levelapp.metrics.exact import EXACT_METRICS
 from levelapp.metrics.fuzzy import FUZZY_METRICS
 
-logger = logging.getLogger(__name__)
-
 
 class MetricRegistry:
     """Registry for metric classes."""
@@ -27,7 +24,6 @@ class MetricRegistry:
             raise KeyError(f"Metric '{name}' is already registered")
 
         cls._metrics[name] = metric_class
-        logger.info(f"Metric '{name}' registered successfully.")
 
     @classmethod
     def get(cls, name: str, **kwargs: Any) -> BaseMetric: