levelapp 0.1.0__py3-none-any.whl → 0.1.1__py3-none-any.whl

levelapp/core/schemas.py

@@ -0,0 +1,26 @@
+from enum import Enum
+
+
+class ExtendedEnum(Enum):
+    @classmethod
+    def list(cls):
+        return [e.value for e in cls]
+
+
+class WorkflowType(ExtendedEnum):
+    SIMULATOR = "SIMULATOR"
+    COMPARATOR = "COMPARATOR"
+    ASSESSOR = "ASSESSOR"
+
+
+class RepositoryType(ExtendedEnum):
+    FIRESTORE = "FIRESTORE"
+    FILESYSTEM = "FILESYSTEM"
+
+
+class EvaluatorType(ExtendedEnum):
+    JUDGE = "JUDGE"
+    REFERENCE = "REFERENCE"
+    RAG = "RAG"
+
+

levelapp/core/session.py

@@ -1,27 +1,23 @@
 """levelapp/core/session.py"""
 import threading
 
+from abc import ABC
+
 from dataclasses import dataclass, field
 from typing import Dict, List, Any
 
 from datetime import datetime
 from humanize import precisedelta
 
-from levelapp.workflow import MainFactory
+from levelapp.workflow import MainFactory, WorkflowConfig
 from levelapp.workflow.base import BaseWorkflow
-from levelapp.workflow.schemas import WorkflowConfig, WorkflowContext
-from levelapp.aspects import FunctionMonitor, MetricType, ExecutionMetrics, MonitoringAspect, logger
+from levelapp.aspects import MetricType, ExecutionMetrics, MonitoringAspect, logger
+from levelapp.workflow.context import WorkflowContextBuilder
 
 
-@dataclass
-class SessionMetadata:
-    """Metadata for an evaluation session."""
-    session_name: str
-    started_at: datetime | None = None
-    ended_at: datetime | None = None
-    total_executions: int = 0
-    total_duration: float = 0.0
-    steps: Dict[str, 'StepMetadata'] = field(default_factory=dict)
+class TemporalStatusMixin(ABC):
+    started_at: datetime | None
+    ended_at: datetime | None
 
     @property
     def is_active(self) -> bool:
@@ -37,7 +33,18 @@ class SessionMetadata:
 
 
 @dataclass
-class StepMetadata:
+class SessionMetadata(TemporalStatusMixin):
+    """Metadata for an evaluation session."""
+    session_name: str
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    total_executions: int = 0
+    total_duration: float = 0.0
+    steps: Dict[str, 'StepMetadata'] = field(default_factory=dict)
+
+
+@dataclass
+class StepMetadata(TemporalStatusMixin):
     """Metadata for a specific step within an evaluation session."""
     step_name: str
     session_name: str
@@ -47,27 +54,21 @@ class StepMetadata:
     error_count: int = 0
     procedures_stats: List[ExecutionMetrics] | None = None
 
-    @property
-    def is_active(self) -> bool:
-        """Check if the step is currently active."""
-        return self.ended_at is None
-
-    @property
-    def duration(self) -> float | None:
-        """Calculate the duration of the step in seconds."""
-        if not self.is_active:
-            return (self.ended_at - self.started_at).total_seconds()
-        return None
-
 
 class StepContext:
     """Context manager for an evaluation step within an EvaluationSession."""
-    def __init__(self, session: "EvaluationSession", step_name: str, category: MetricType):
+    def __init__(
+            self,
+            session: "EvaluationSession",
+            step_name: str,
+            category: MetricType,
+    ):
         self.session = session
         self.step_name = step_name
         self.category = category
+
         self.step_meta: StepMetadata | None = None
-        self.full_step_name = f"{session.session_name}.{step_name}"
+        self.full_step_name = f"<{session.session_name}:{step_name}>"
         self._monitored_func = None
         self._func_gen = None
 
@@ -80,35 +81,49 @@ class StepContext:
             )
             self.session.session_metadata.steps[self.step_name] = self.step_meta
 
-        # Wrap with FunctionMonitor
-        self._monitored_func = self.session.monitor.monitor(
-            name=self.full_step_name,
-            category=self.category,
-            enable_timing=True,
-            track_memory=True,
-        )(self._step_wrapper)
-
-        # Start monitoring
-        self._func_gen = self._monitored_func()
-        next(self._func_gen)  # Enter monitoring
+        if self.session.enable_monitoring:
+            # Wrap with FunctionMonitor
+            self._monitored_func = self.session.monitor.monitor(
+                name=self.full_step_name,
+                category=self.category,
+                enable_timing=True,
+                track_memory=True,
+            )(self._step_wrapper)
+
+            # Start monitoring
+            try:
+                self._func_gen = self._monitored_func()
+                next(self._func_gen)  # Enter monitoring
+            except Exception as e:
+                logger.error(f"[StepContext] Failed to initialize monitoring for {self.full_step_name}:\n{e}")
+                raise
+
         return self  # returning self allows nested instrumentation
 
+    # noinspection PyMethodMayBeStatic
     def _step_wrapper(self):
         yield  # Actual user step execution happens here
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        try:
-            next(self._func_gen)  # Exit monitoring
-        except StopIteration:
-            pass
+        if self.session.enable_monitoring:
+            try:
+                next(self._func_gen)  # Exit monitoring
+            except StopIteration:
+                pass
 
         with self.session.lock:
             self.step_meta.ended_at = datetime.now()
+
             if exc_type:
                 self.step_meta.error_count += 1
+
             self.session.session_metadata.total_executions += 1
+
             if self.step_meta.duration:
-                self.session.monitor.update_procedure_duration(name=self.full_step_name, value=self.step_meta.duration)
+                self.session.monitor.update_procedure_duration(
+                    name=self.full_step_name,
+                    value=self.step_meta.duration
+                )
                 self.session.session_metadata.total_duration += self.step_meta.duration
 
         return False
@@ -119,29 +134,30 @@ class EvaluationSession:
     def __init__(
             self,
             session_name: str = "test-session",
-            monitor: FunctionMonitor | None = None,
-            workflow_config: WorkflowConfig | None = None
+            workflow_config: WorkflowConfig | None = None,
+            enable_monitoring: bool = True,
     ):
         """
         Initialize Evaluation Session.
 
         Args:
             session_name (str): Name of the session
-            monitor (FunctionMonitor): Function monitoring aspect
             workflow_config (WorkflowConfig): Workflow configuration.
         """
         self._NAME = self.__class__.__name__
 
         self.session_name = session_name
-        self.monitor = monitor or MonitoringAspect
         self.workflow_config = workflow_config
-        self.workflow_type = workflow_config.workflow
+        self.enable_monitoring = enable_monitoring
 
         self.workflow: BaseWorkflow | None = None
 
         self.session_metadata = SessionMetadata(session_name=session_name)
+        self.monitor = MonitoringAspect if enable_monitoring else None
         self._lock = threading.RLock()
 
+        logger.info("[EvaluationSession] Evaluation session initialized.")
+
     @property
     def lock(self):
         return self._lock
@@ -154,14 +170,10 @@ class EvaluationSession:
             if not self.workflow_config:
                 raise ValueError(f"{self._NAME}: Workflow configuration must be provided")
 
-            context = WorkflowContext(
-                config=self.workflow_config,
-                repository=MainFactory.create_repository(self.workflow_config),
-                evaluators=MainFactory.create_evaluator(self.workflow_config),
-                endpoint_config=self.workflow_config.endpoint_config,
-                inputs=self.workflow_config.inputs
-            )
-            self.workflow = MainFactory.create_workflow(self.workflow_type, context)
+            context_builder = WorkflowContextBuilder(self.workflow_config)
+            context = context_builder.build()
+
+            self.workflow = MainFactory.create_workflow(context=context)
 
         logger.info(
             f"[{self._NAME}] Starting evaluation session: {self.session_name}, "

levelapp/evaluator/evaluator.py

@@ -1,7 +1,6 @@
 """levelapp/core/evaluator.py"""
 from functools import lru_cache
-from typing import List, Dict, Any
-from collections import defaultdict
+from typing import List, Dict, Any, TYPE_CHECKING
 from pydantic import BaseModel, Field
 
 from tenacity import (
@@ -19,6 +18,9 @@ from levelapp.config.prompts import EVAL_PROMPT_TEMPLATE
 from levelapp.core.base import BaseEvaluator, BaseChatClient
 from levelapp.aspects import MonitoringAspect, MetricType, logger, DataLoader
 
+if TYPE_CHECKING:
+    from levelapp.workflow.config import WorkflowConfig
+
 
 class Evidence(BaseModel):
     """Evidence details for evaluation."""
@@ -69,19 +71,29 @@ class JudgeEvaluationResults(BaseModel):
 
 
 class JudgeEvaluator(BaseEvaluator):
-    def __init__(self):
+    def __init__(self, config: "WorkflowConfig | None" = None):
+        if config:
+            self.config = config
+            self.providers = config.evaluation.providers
+
         self.prompt_template = EVAL_PROMPT_TEMPLATE
-        self.clients = defaultdict(BaseChatClient)
+        self.client_registry = ClientRegistry
 
-    def register_client(self, provider: str, client: BaseChatClient) -> None:
+    def select_client(self, provider: str) -> BaseChatClient:
         """
-        Register LLM clients used for the evaluation.
+        Select an LLM client to use for the evaluation.
 
         Args:
             provider (str): The provider name.
-            client (BaseChatClient): The LLM client to register.
+
+        Returns:
+            client (BaseChatClient): The LLM client to use for the evaluation.
         """
-        self.clients[provider] = client
+        if provider not in self.client_registry.list_providers():
+            logger.warning(f"[JudgeEvaluator] {provider} is not registered. Defaulting to 'OpenAI'.")
+            return self.client_registry.get(provider="openai")
+
+        return self.client_registry.get(provider=provider)
 
     @lru_cache(maxsize=1024)
     def _build_prompt(self, user_input: str, generated_text: str, reference_text: str) -> str:
@@ -135,7 +147,7 @@ class JudgeEvaluator(BaseEvaluator):
             generated_text=generated_data,
             reference_text=reference_data
         )
-        client = ClientRegistry.get(provider=provider)
+        client = self.select_client(provider=provider)
 
         try:
             response = client.call(message=prompt)
@@ -183,7 +195,7 @@ class JudgeEvaluator(BaseEvaluator):
             generated_text=generated_data,
             reference_text=reference_data
         )
-        client = ClientRegistry.get(provider=provider)
+        client = self.select_client(provider=provider)
 
         try:
             async for attempt in AsyncRetrying(
@@ -212,7 +224,11 @@ class JudgeEvaluator(BaseEvaluator):
 
 
 class MetadataEvaluator(BaseEvaluator):
-    def __init__(self):
+    def __init__(self, config: "WorkflowConfig | None"= None):
+        if config:
+            self.config = config
+            self.metics_map = config.evaluation.metrics_map
+
         self.data_loader = DataLoader()
         self.comparator = MetadataComparator()
         self.metrics_manager = MetricsManager()

levelapp/repository/firestore.py

@@ -1,7 +1,5 @@
 """levelapp/repository/firestore.py"""
-import google.auth
-
-from typing import List, Dict, Any, Type
+from typing import List, Dict, Any, Type, TYPE_CHECKING
 from pydantic import ValidationError
 
 from google.cloud import firestore_v1
@@ -13,15 +11,25 @@ from levelapp.core.base import BaseRepository, Model
 from levelapp.aspects import logger
 
 
+if TYPE_CHECKING:
+    from levelapp.workflow.config import WorkflowConfig
+
+
 class FirestoreRepository(BaseRepository):
     """
     Firestore implementation of BaseRepository.
     (Uses hierarchical path: {user_id}/{collection_id}/{document_id}
     """
 
-    def __init__(self, project_id: str | Any = None, database_name: str | Any = '(default)'):
-        self.project_id = project_id
-        self.database_name = database_name
+    def __init__(self, config: "WorkflowConfig | None"):
+        if config:
+            self.config = config
+            self.project_id: str | Any = config.repository.project_id
+            self.database_name: str | Any = config.repository.database_name
+        else:
+            self.project_id: str | Any = None
+            self.database_name: str | Any = '(default)'
+
         self.client: firestore_v1.Client | None = None
 
     def connect(self) -> None:
@@ -29,6 +37,7 @@ class FirestoreRepository(BaseRepository):
         Connects to Firestore, prioritizing the project ID passed to the constructor.
         """
         try:
+            import google.auth
             credentials, default_project_id = google.auth.default()
 
             if not credentials:

levelapp/simulator/schemas.py

@@ -8,7 +8,7 @@ from enum import Enum
 from uuid import UUID, uuid4
 from datetime import datetime
 
-from typing import Optional, Dict, Any, List
+from typing import Dict, Any, List
 from pydantic import BaseModel, Field, computed_field
 
 from levelapp.evaluator.evaluator import JudgeEvaluationResults
@@ -25,11 +25,11 @@ class Interaction(BaseModel):
     """Represents a single interaction within a conversation."""
     id: UUID = Field(default_factory=uuid4, description="Interaction identifier")
     user_message: str = Field(..., description="The user's query message")
-    generated_reply: str = Field(..., description="The agent's reply message")
+    # generated_reply: str = Field(..., description="The agent's reply message")
     reference_reply: str = Field(..., description="The preset reference message")
-    interaction_type: InteractionLevel = Field(..., description="Type of interaction")
-    reference_metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Expected metadata")
-    generated_metadata: Optional[Dict[str, Any]] = Field(default_factory=dict, description="Extracted metadata")
+    interaction_type: InteractionLevel = Field(default=InteractionLevel.INITIAL, description="Type of interaction")
+    reference_metadata: Dict[str, Any] = Field(default_factory=dict, description="Expected metadata")
+    # generated_metadata: Dict[str, Any] = Field(default_factory=dict, description="Extracted metadata")
     guardrail_flag: bool = Field(default=False, description="Flag for guardrail signaling")
     request_payload: Dict[str, Any] = Field(default_factory=dict, description="Additional request payload")
 
@@ -38,7 +38,7 @@ class ConversationScript(BaseModel):
     """Represents a basic conversation with multiple interactions."""
     id: UUID = Field(default_factory=uuid4, description="Conversation identifier")
     interactions: List[Interaction] = Field(default_factory=list, description="List of interactions")
-    description: str = Field(..., description="A short description of the conversation")
+    description: str = Field(default="no-description", description="A short description of the conversation")
     details: Dict[str, str] = Field(default_factory=dict, description="Conversation details")
 
 
@@ -58,8 +58,8 @@ class InteractionResults(BaseModel):
 
 class InteractionEvaluationResults(BaseModel):
     """Model representing the evaluation result of an interaction."""
-    judge_evaluations: Dict[str, JudgeEvaluationResults] = Field(default_factory=dict)
-    metadata_evaluation: Dict[str, float] = Field(default_factory=dict)
+    judge_evaluations: Dict[str, JudgeEvaluationResults] | None = Field(default_factory=dict)
+    metadata_evaluation: Dict[str, float] | None = Field(default_factory=dict)
     guardrail_flag: int = Field(default=0)
 
 
@@ -86,4 +86,4 @@ class TestResults(BaseModel):
     ionos_model_name: str = Field(..., alias="ionosModelName")
     test_name: str = Field(..., alias="testName")
     test_type: str = Field(..., alias="testType")
-    batch_details: Optional[SimulationResults] = Field(..., alias="results")
+    batch_details: SimulationResults | None = Field(..., alias="results")

levelapp/simulator/simulator.py

@@ -23,7 +23,7 @@ from levelapp.simulator.utils import (
     summarize_verdicts,
 )
 from levelapp.aspects import logger
-from levelapp.workflow.schemas import EvaluatorType
+from levelapp.core.schemas import EvaluatorType
 
 
 class ConversationSimulator(BaseProcess):
@@ -33,6 +33,7 @@ class ConversationSimulator(BaseProcess):
         self,
         repository: BaseRepository | None = None,
         evaluators: Dict[EvaluatorType, BaseEvaluator] | None = None,
+        providers: List[str] | None = None,
         endpoint_config: EndpointConfig | None = None,
     ):
         """
@@ -47,6 +48,7 @@ class ConversationSimulator(BaseProcess):
 
         self.repository = repository
         self.evaluators = evaluators
+        self.providers = providers
         self.endpoint_config = endpoint_config
 
         self._url: str | None = None
@@ -60,7 +62,8 @@ class ConversationSimulator(BaseProcess):
     def setup(
             self,
             repository: BaseRepository,
-            evaluators: Dict[str, BaseEvaluator],
+            evaluators: Dict[EvaluatorType, BaseEvaluator],
+            providers: List[str],
             endpoint_config: EndpointConfig,
     ) -> None:
         """
@@ -69,6 +72,7 @@ class ConversationSimulator(BaseProcess):
         Args:
             repository (BaseRepository): Repository object for storing simulation results.
             evaluators (Dict[str, BaseEvaluator]): List of evaluator objects for evaluating interactions.
+            providers (List[str]): List of LLM provider names.
             endpoint_config (EndpointConfig): Configuration object for VLA.
         """
         _LOG: str = f"[{self._CLASS_NAME}][{self.setup.__name__}]"
@@ -76,6 +80,11 @@ class ConversationSimulator(BaseProcess):
 
         self.repository = repository
         self.evaluators = evaluators
+        self.providers = providers
+
+        if not self.providers:
+            logger.warning(f"{_LOG} No LLM providers were provided. The Judge Evaluation process will not be executed.")
+
         self.endpoint_config = endpoint_config
 
         self._url = endpoint_config.full_url
@@ -87,6 +96,7 @@ class ConversationSimulator(BaseProcess):
 
         if name not in self.evaluators:
             raise KeyError(f"{_LOG} Evaluator {name} not registered.")
+
         return self.evaluators[name]
 
     async def run(
@@ -368,50 +378,106 @@ class ConversationSimulator(BaseProcess):
         """
         _LOG: str = f"[{self._CLASS_NAME}][{self.evaluate_interaction.__name__}]"
 
-        judge_evaluator = self.evaluators.get(EvaluatorType.JUDGE)
-        metadata_evaluator = self.evaluators.get(EvaluatorType.REFERENCE)
+        judge_evaluator: BaseEvaluator | None = self.evaluators.get(EvaluatorType.JUDGE, None)
+        metadata_evaluator: BaseEvaluator | None = self.evaluators.get(EvaluatorType.REFERENCE, None)
 
-        if not judge_evaluator:
-            raise ValueError(f"{_LOG} No Judge Evaluator found.")
+        evaluation_results = InteractionEvaluationResults()
 
-        openai_eval_task = judge_evaluator.async_evaluate(
-            generated_data=generated_reply,
-            reference_data=reference_reply,
-            user_input=user_input,
-            provider="openai"
-        )
+        if judge_evaluator and self.providers:
+            await self._judge_evaluation(
+                user_input=user_input,
+                generated_reply=generated_reply,
+                reference_reply=reference_reply,
+                providers=self.providers,
+                judge_evaluator=judge_evaluator,
+                evaluation_results=evaluation_results,
+            )
+        else:
+            logger.info(f"[{_LOG}] Judge evaluation skipped (no evaluator or no providers).")
 
-        ionos_eval_task = judge_evaluator.async_evaluate(
-            provider="ionos",
-            user_input=user_input,
-            generated_data=generated_reply,
-            reference_data=reference_reply,
-        )
+        if metadata_evaluator and reference_metadata:
+            self._metadata_evaluation(
+                metadata_evaluator=metadata_evaluator,
+                generated_metadata=generated_metadata,
+                reference_metadata=reference_metadata,
+                evaluation_results=evaluation_results,
+            )
+        else:
+            logger.info(f"[{_LOG}] Metadata evaluation skipped (no evaluator or no reference metadata).")
 
-        openai_judge_evaluation, ionos_judge_evaluation = await asyncio.gather(
-            openai_eval_task, ionos_eval_task
-        )
+        evaluation_results.guardrail_flag = 1 if generated_guardrail == reference_guardrail else 0
 
-        if not metadata_evaluator:
-            raise ValueError(f"{_LOG} No Metadata Evaluator found.")
+        return evaluation_results
 
-        metadata_evaluation = {}
-        if reference_metadata:
-            metadata_evaluation = metadata_evaluator.evaluate(
-                generated_data=generated_metadata,
-                reference_data=reference_metadata,
+    async def _judge_evaluation(
+            self,
+            user_input: str,
+            generated_reply: str,
+            reference_reply: str,
+            providers: List[str],
+            judge_evaluator: BaseEvaluator,
+            evaluation_results: InteractionEvaluationResults,
+    ) -> None:
+        """
+        Run LLM-as-a-judge evaluation using multiple providers (async).
+
+        Args:
+            user_input (str): The user input message.
+            generated_reply (str): The generated agent reply.
+            reference_reply (str): The reference agent reply.
+            providers (List[str]): List of judge provider names.
+            judge_evaluator (BaseEvaluator): Evaluator instance.
+            evaluation_results (InteractionEvaluationResults): Results container (Pydantic model).
+
+        Returns:
+            None
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][judge_evaluation]"
+
+        tasks = {
+            provider: judge_evaluator.async_evaluate(
+                generated_data=generated_reply,
+                reference_data=reference_reply,
+                user_input=user_input,
+                provider=provider,
             )
+            for provider in providers
+        }
 
-        guardrail_flag = 1 if generated_guardrail == reference_guardrail else 0
+        results = await asyncio.gather(*tasks.values(), return_exceptions=True)
 
-        return InteractionEvaluationResults(
-            judge_evaluations={
-                openai_judge_evaluation.provider: openai_judge_evaluation,
-                ionos_judge_evaluation.provider: ionos_judge_evaluation
-            },
-            metadata_evaluation=metadata_evaluation,
-            guardrail_flag=guardrail_flag,
-        )
+        for provider, result in zip(tasks.keys(), results):
+            if isinstance(result, Exception):
+                logger.error(f"{_LOG} Provider '{provider}' failed to perform Judge Evaluation.")
+                continue
+
+            evaluation_results.judge_evaluations[provider] = result
+
+    def _metadata_evaluation(
+            self,
+            metadata_evaluator: BaseEvaluator,
+            generated_metadata: Dict[str, Any],
+            reference_metadata: Dict[str, Any],
+            evaluation_results: InteractionEvaluationResults,
+    ) -> None:
+        """
+        Run metadata evaluation using the provided evaluator.
+
+        Args:
+            metadata_evaluator (BaseEvaluator): Evaluator for metadata comparison.
+            generated_metadata (Dict[str, Any]): The generated metadata.
+            reference_metadata (Dict[str, Any]): The reference metadata.
+            evaluation_results (InteractionEvaluationResults): Results container.
+        """
+        _LOG: str = f"[{self._CLASS_NAME}][metadata_evaluation]"
+
+        try:
+            evaluation_results.metadata_evaluation = metadata_evaluator.evaluate(
+                generated_data=generated_metadata,
+                reference_data=reference_metadata,
+            )
+        except Exception as e:
+            logger.error(f"[{_LOG}] Metadata evaluation failed:\n{e}", exc_info=e)
 
     @staticmethod
     def store_evaluation_results(

levelapp/workflow/__init__.py

@@ -1,5 +1,6 @@
 from . import registration
-from .schemas import WorkflowType
+from .base import BaseWorkflow
+from .config import WorkflowConfig
 from .factory import MainFactory
 
-__all__ = ["WorkflowType", "MainFactory"]
+__all__ = ["BaseWorkflow", "WorkflowConfig", "MainFactory"]