levelapp 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

levelapp/aspects/monitor.py

@@ -343,6 +343,7 @@ class FunctionMonitor:
             category: MetricType,
             enable_timing: bool,
             track_memory: bool,
+            verbose=False
     ) -> Callable[P, T]:
         """
         Wrap function execution with timing and error handling.
@@ -352,6 +353,7 @@ class FunctionMonitor:
             name: Unique identifier for the function
             enable_timing: Enable execution time logging
             track_memory: Enable memory tracking
+            verbose: Enable verbose logging
 
         Returns:
             Wrapped function
@@ -402,7 +404,7 @@ class FunctionMonitor:
 
                     self._aggregated_stats[name].update(metrics=metrics)
 
-                if enable_timing and metrics.duration is not None:
+                if verbose and enable_timing and metrics.duration is not None:
                     log_message = f"[FunctionMonitor] Executed '{name}' in {metrics.duration:.4f}s"
                     if metrics.cache_hit:
                         log_message += " (cache hit)"

levelapp/clients/__init__.py

@@ -44,7 +44,6 @@ class ClientRegistry:
 
         cls._wrap_client_methods(client_class)
         cls._clients[provider] = client_class
-        logger.info(f"[ClientRegistry] Registered client for provider: {provider}")
 
     @classmethod
     def _wrap_client_methods(cls, client_class: Type[BaseChatClient]) -> None:

levelapp/comparator/scorer.py

@@ -78,7 +78,6 @@ class MetricsManager:
             ValueError: if the scorer is not a callable.
         """
         self._scorers[name] = scorer
-        logger.info(f"[MetricsManager] Registered scorer: {name}")
 
     def get_scorer(self, name: str) -> Callable:
         """
@@ -95,7 +94,6 @@ class MetricsManager:
         """
         try:
             scorer = self._scorers.get(name)
-            logger.info(f"[get_scorer] Retrieved scorer: {name}")
             return scorer
 
         except KeyError:

levelapp/config/endpoint.py

@@ -29,7 +29,6 @@ class EndpointConfig(BaseModel):
         bearer_token (SecretStr): The Bearer token to use.
         model_id (str): The model to use (if applicable).
         default_request_payload_template (Dict[str, Any]): The payload template to use.
-        generated_request_payload_template (Dict[str, Any]): The generated payload template from a provided file.
         variables (Dict[str, Any]): The variables to populate the payload template.
 
     Note:
@@ -40,11 +39,10 @@ class EndpointConfig(BaseModel):
         - bearer_token (SecretStr): The Bearer token to use.
         - model_id (str): The model to use (if applicable).
         - default_payload_template (Dict[str, Any]): The payload template to use.
-        - generated_payload_template (Dict[str, Any]): The generated payload template from a provided file.
         - variables (Dict[str, Any]): The variables to populate the payload template.
 
         Or manually configure the model instance by assigning the proper values to the model fields.\n
-        You can also provide the path in the .env file for the payload template (ENDPOINT_PAYLOAD_PATH)
+        You can also provide the path in the .env file for the payload template (ENDPOINT_PAYLOAD_PATH/)
         and the response template (ENDPOINT_RESPONSE_PATH) separately. The files can be either YAML or JSON only.
     """
     load_dotenv()
@@ -61,9 +59,7 @@ class EndpointConfig(BaseModel):
 
     # Data
     default_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
-    generated_request_payload_template: Dict[str, Any] = Field(default_factory=dict)
     default_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
-    generated_response_payload_template: Dict[str, Any] = Field(default_factory=dict)
 
     # Variables
     variables: Dict[str, Any] = Field(default_factory=dict)
@@ -88,14 +84,18 @@ class EndpointConfig(BaseModel):
     @computed_field
     @property
     def request_payload(self) -> Dict[str, Any]:
-        """Return fully prepared payload depending on template or full payload."""
-        # First, load the request payload template (either from YAML config file or from specific template)
+        """
+        Return fully prepared payload depending on template or full payload.
+
+        Returns:
+            request payload (Dict[str, Any]): Populated request payload template.
+        """
+        # First, we check if we have variables to populate the template with. If not, we return the template as is.
         if not self.variables:
             return self.default_request_payload_template
 
         if not self.default_request_payload_template:
-            self.load_template(template_type=TemplateType.REQUEST)
-            base_template = self.generated_request_payload_template
+            base_template = self.load_template(template_type=TemplateType.REQUEST)
         else:
             base_template = self.default_request_payload_template
 
@@ -118,8 +118,7 @@ class EndpointConfig(BaseModel):
             return self.default_response_payload_template
 
         if not self.default_response_payload_template:
-            self.load_template(template_type=TemplateType.RESPONSE)
-            base_template = self.generated_response_payload_template
+            base_template = self.load_template(template_type=TemplateType.RESPONSE)
         else:
             base_template = self.default_response_payload_template
 
@@ -148,12 +147,23 @@ class EndpointConfig(BaseModel):
 
         return _replace(obj)
 
+    @staticmethod
     def load_template(
-            self,
             template_type: TemplateType = TemplateType.REQUEST,
             path: str | None = None
     ) -> Dict[str, Any]:
+        """
+        Load request/response payload template from JSON/YAML file.
+
+        Args:
+            template_type (TemplateType): The type of template to load (REQUEST or RESPONSE).
+            path (str): The path of the payload template file to load.
+
+        Returns:
+            Payload template (Dict[str, Any]): Payload template.
+        """
         try:
+            # If no path was provided, we check the env. variables.
             if not path:
                 env_var = "ENDPOINT_PAYLOAD_PATH" if template_type == TemplateType.REQUEST else "ENDPOINT_RESPONSE_PATH"
                 path = os.getenv(env_var, '')
@@ -171,7 +181,6 @@ class EndpointConfig(BaseModel):
                 else:
                     raise ValueError("[EndpointConfig] Unsupported file format.")
 
-                self.generated_request_payload_template = data
                 return data
 
         except FileNotFoundError as e:

levelapp/config/endpoint_.py

@@ -0,0 +1,62 @@
+from abc import ABC
+from enum import Enum
+from typing import Any, List
+
+from pydantic import BaseModel, Field
+
+
+class HttpMethod(str, Enum):
+    GET = "GET"
+    POST = "POST"
+    PUT = "PUT"
+    Patch = "PATCH"
+    DELETE = "DELETE"
+
+
+class HeaderConfig(BaseModel):
+    """Secure header configuration with environment variables support."""
+    name: str
+    value: str
+    secure: bool = False
+
+    class Config:
+        frozen = True
+
+
+class RequestSchemaConfig(BaseModel):
+    """Schema definition for request payload population."""
+    field_path: str  # JSON path-like: "data.user.id"
+    value: Any
+    value_type: str = "static"  # static, env, dynamic
+    required: bool = True
+
+
+class ResponseMappingConfig(BaseModel):
+    """Response data extraction mapping."""
+    field_path: str  # JSON path-like: "data.results[0].id"
+    extract_as: str  # Name to extract as
+    default: Any = None
+
+
+class EndpointConfig(BaseModel):
+    """Complete endpoint configuration."""
+    name: str
+    base_url: str
+    path: str
+    method: HttpMethod
+    headers: List[HeaderConfig] = Field(default_factory=list)
+    request_schema: List[RequestSchemaConfig] = Field(default_factory=list)
+    response_mapping: List[ResponseMappingConfig] = Field(default_factory=list)
+    timeout: int = 30
+    retry_count: int = 3
+    retry_backoff: float = 1.0
+
+    @classmethod
+    def validate_path(cls, v: str) -> str:
+        if not v.startswith('/'):
+            return f'/{v}'
+        return v
+
+
+class PayloadBuilder(ABC):
+    """Abstract base for payload construction strategies."""

levelapp/config/prompts.py

@@ -33,3 +33,25 @@ Return ONLY a single JSON object on one line with exactly these keys:
     
 Do NOT include any additional text, explanations, or formatting (e.g., "JSON object:", ```json or ```, or markdown).
 """
+
+
+SUMMARIZATION_PROMPT_TEMPLATE = """
+You are reviewing evaluation justifications from LLM judges about replies generated by a virtual assistant.
+Interpret the context from the verdicts: (e.g., real-estate leasing, medical appointment scheduling, etc.).
+
+Each justification contains the judge's assessment of how well the assistant's response matched the expected reply.
+Your task is to **identify and summarize only the negative points**, such as:
+- Errors or inaccuracies
+- Misunderstandings or misinterpretations
+- Missing or incomplete information
+- Failure to meet expectations or requirements
+
+**Instructions:**
+- Return up to {max_bullets} concise bullet points.
+- Start each point with "- " and focus on clarity and relevance.
+- Avoid redundancy and prioritize actionable feedback.
+
+---
+- Judge: {judge}
+- Verdicts: {verdicts}
+"""

levelapp/core/schemas.py

@@ -22,5 +22,3 @@ class EvaluatorType(ExtendedEnum):
     JUDGE = "JUDGE"
     REFERENCE = "REFERENCE"
     RAG = "RAG"
-
-

levelapp/core/session.py

@@ -63,6 +63,14 @@ class StepContext:
             step_name: str,
             category: MetricType,
     ):
+        """
+        Initialize StepContext.
+
+        Args:
+            session (EvaluationSession): Evaluation session.
+            step_name (str): Step name.
+            category (MetricType): Metric type.
+        """
         self.session = session
         self.step_name = step_name
         self.category = category
@@ -88,6 +96,7 @@ class StepContext:
                 category=self.category,
                 enable_timing=True,
                 track_memory=True,
+                verbose=self.session.verbose,
             )(self._step_wrapper)
 
             # Start monitoring
@@ -119,7 +128,7 @@ class StepContext:
 
             self.session.session_metadata.total_executions += 1
 
-            if self.step_meta.duration:
+            if self.session.enable_monitoring and self.step_meta.duration:
                 self.session.monitor.update_procedure_duration(
                     name=self.full_step_name,
                     value=self.step_meta.duration
@@ -136,6 +145,7 @@ class EvaluationSession:
             session_name: str = "test-session",
             workflow_config: WorkflowConfig | None = None,
             enable_monitoring: bool = True,
+            verbose: bool = False
     ):
         """
         Initialize Evaluation Session.
@@ -143,12 +153,15 @@ class EvaluationSession:
         Args:
             session_name (str): Name of the session
             workflow_config (WorkflowConfig): Workflow configuration.
+            enable_monitoring (bool): Switch monitoring on. Defaults to True.
+            verbose (bool): Verbose mode. Defaults to False.
         """
         self._NAME = self.__class__.__name__
 
         self.session_name = session_name
         self.workflow_config = workflow_config
         self.enable_monitoring = enable_monitoring
+        self.verbose = verbose
 
         self.workflow: BaseWorkflow | None = None
 
@@ -176,7 +189,7 @@ class EvaluationSession:
             self.workflow = MainFactory.create_workflow(context=context)
 
         logger.info(
-            f"[{self._NAME}] Starting evaluation session: {self.session_name}, "
+            f"[{self._NAME}] Starting evaluation session: {self.session_name} - "
             f"Workflow: '{self.workflow.name}'"
         )
         return self
@@ -190,6 +203,7 @@ class EvaluationSession:
 
         if exc_type:
             logger.error(f"[{self._NAME}] Session ended with error: {exc_val}", exc_info=True)
+
         return False
 
     def step(self, step_name: str, category: MetricType = MetricType.CUSTOM) -> StepContext:
@@ -213,6 +227,19 @@ class EvaluationSession:
             self.workflow.collect_results()
 
     def get_stats(self) -> Dict[str, Any]:
+        if self.enable_monitoring:
+            return {
+                "session": {
+                    "name": self.session_name,
+                    "duration": precisedelta(self.session_metadata.duration, suppress=['minutes']),
+                    "start_time": self.session_metadata.started_at.isoformat(),
+                    "end_time": self.session_metadata.ended_at.isoformat(),
+                    "steps": len(self.session_metadata.steps),
+                    "errors": sum(s.error_count for s in self.session_metadata.steps.values())
+                },
+                "stats": self.monitor.get_all_stats()
+            }
+
         return {
             "session": {
                 "name": self.session_name,
@@ -222,5 +249,4 @@ class EvaluationSession:
                 "steps": len(self.session_metadata.steps),
                 "errors": sum(s.error_count for s in self.session_metadata.steps.values())
             },
-            "stats": self.monitor.get_all_stats()
         }

levelapp/evaluator/evaluator.py

@@ -41,7 +41,7 @@ class JudgeEvaluationResults(BaseModel):
     label: str = Field(..., description="The label of the evaluation result")
     justification: str = Field(..., description="Short explanation of the evaluation result")
     evidence: Evidence = Field(default_factory=Evidence, description="Detailed evidence for the evaluation")
-    raw_response: Dict[str, Any] = Field(..., description="Full unprocessed API response")
+    raw_response: Dict[str, Any] = Field(..., description="Full unprocessed API response", exclude=True)
     metadata: Dict[str, Any] = Field(..., description="Metadata about the evaluation result")
 
     @classmethod
@@ -71,7 +71,14 @@ class JudgeEvaluationResults(BaseModel):
 
 
 class JudgeEvaluator(BaseEvaluator):
+    """LLM-as-a-judge evaluator class"""
     def __init__(self, config: "WorkflowConfig | None" = None):
+        """
+        Initialize the JudgeEvaluator.
+
+        Args:
+            config (WorkflowConfig | None): The configuration of the workflow.
+        """
         if config:
             self.config = config
             self.providers = config.evaluation.providers
@@ -206,7 +213,6 @@ class JudgeEvaluator(BaseEvaluator):
             ):
                 with attempt:
                     response = await client.acall(message=prompt)
-                    logger.info(f"[{provider}] Async evaluation:\n{response}\n{'---' * 10}")
                     parsed = client.parse_response(response=response)
                     return JudgeEvaluationResults.from_parsed(provider=provider, parsed=parsed, raw=response)
 
@@ -224,7 +230,14 @@ class JudgeEvaluator(BaseEvaluator):
 
 
 class MetadataEvaluator(BaseEvaluator):
-    def __init__(self, config: "WorkflowConfig | None"= None):
+    """Metadata evaluator class."""
+    def __init__(self, config: "WorkflowConfig | None" = None):
+        """
+        Initialize the MetadataEvaluator.
+
+        Args:
+            config (WorkflowConfig | None): The workflow configuration.
+        """
         if config:
             self.config = config
             self.metics_map = config.evaluation.metrics_map
@@ -261,7 +274,6 @@ class MetadataEvaluator(BaseEvaluator):
         self.comparator.reference_data = ref_data
 
         output = self.comparator.run(indexed_mode=False)
-        logger.info(f"Comparison results:\n{output}\n---")
         results: Dict[str, float] = {}
 
         for k, v in output.items():

levelapp/metrics/__init__.py

@@ -1,14 +1,11 @@
 """levelapp/metrics/__init__.py"""
-import logging
-
 from typing import List, Dict, Type, Any
 
+from levelapp.aspects import logger
 from levelapp.core.base import BaseMetric
 from levelapp.metrics.exact import EXACT_METRICS
 from levelapp.metrics.fuzzy import FUZZY_METRICS
 
-logger = logging.getLogger(__name__)
-
 
 class MetricRegistry:
     """Registry for metric classes."""
@@ -27,7 +24,6 @@ class MetricRegistry:
             raise KeyError(f"Metric '{name}' is already registered")
 
         cls._metrics[name] = metric_class
-        logger.info(f"Metric '{name}' registered successfully.")
 
     @classmethod
     def get(cls, name: str, **kwargs: Any) -> BaseMetric:

levelapp/simulator/schemas.py

@@ -9,7 +9,7 @@ from uuid import UUID, uuid4
 from datetime import datetime
 
 from typing import Dict, Any, List
-from pydantic import BaseModel, Field, computed_field
+from pydantic import BaseModel, Field, computed_field, field_validator
 
 from levelapp.evaluator.evaluator import JudgeEvaluationResults
 
@@ -64,26 +64,20 @@ class InteractionEvaluationResults(BaseModel):
 
 
 class SimulationResults(BaseModel):
-    # Initial data
-    project_id: str = Field(default_factory=uuid4, description="Project identifier")
-    user_id: str = Field(default_factory=uuid4, description="User identifier")
-    batch_id: str = Field(default_factory=uuid4, description="Batch identifier")
     # Collected data
     started_at: datetime = datetime.now()
     finished_at: datetime
     # Collected Results
     evaluation_summary: Dict[str, Any] | None = Field(default_factory=dict, description="Evaluation result")
     average_scores: Dict[str, Any] | None = Field(default_factory=dict, description="Average scores")
+    interaction_results: List[Dict[str, Any]] | None = Field(default_factory=list, description="detailed results")
+
+    @computed_field
+    @property
+    def batch_id(self) -> str:
+        return str(uuid4())
 
     @computed_field
     @property
     def elapsed_time(self) -> float:
         return (self.finished_at - self.started_at).total_seconds()
-
-
-class TestResults(BaseModel):
-    api_host: str = Field(..., alias="apiHost")
-    ionos_model_name: str = Field(..., alias="ionosModelName")
-    test_name: str = Field(..., alias="testName")
-    test_type: str = Field(..., alias="testType")
-    batch_details: SimulationResults | None = Field(..., alias="results")

levelapp/simulator/simulator.py

@@ -1,6 +1,7 @@
 """
 'simulators/service.py': Service layer to manage conversation simulation and evaluation.
 """
+import json
 import time
 import asyncio
 
@@ -92,6 +93,15 @@ class ConversationSimulator(BaseProcess):
         self._headers = endpoint_config.headers
 
     def get_evaluator(self, name: EvaluatorType) -> BaseEvaluator:
+        """
+        Retrieve an evaluator by name.
+
+        Args:
+            name (EvaluatorType): Name of evaluator.
+
+        Returns:
+            An evaluator object.
+        """
         _LOG: str = f"[{self._CLASS_NAME}][{self.get_evaluator.__name__}]"
 
         if name not in self.evaluators:
@@ -103,7 +113,7 @@ class ConversationSimulator(BaseProcess):
         self,
         test_batch: ScriptsBatch,
         attempts: int = 1,
-    ) -> Dict[str, Any]:
+    ) -> Any:
         """
         Run a batch test for the given batch name and details.
 
@@ -129,9 +139,10 @@ class ConversationSimulator(BaseProcess):
             finished_at=finished_at,
             evaluation_summary=self.verdict_summaries,
             average_scores=results.get("average_scores", {}),
+            interaction_results=results.get("results")
         )
 
-        return {"results": results, "status": "COMPLETE"}
+        return results.model_dump_json(indent=2)
 
     async def simulate_conversation(self, attempts: int = 1) -> Dict[str, Any]:
         """
@@ -171,10 +182,11 @@ class ConversationSimulator(BaseProcess):
                 verdicts=verdicts, judge=judge
             )
 
-        return {"scripts": results, "average_scores": overall_average_scores}
+        return {"results": results, "average_scores": overall_average_scores}
 
     async def simulate_single_scenario(
-        self, script: ConversationScript, attempts: int = 1
+        self, script: ConversationScript,
+            attempts: int = 1
     ) -> Dict[str, Any]:
         """
         Simulate a single scenario with the given number of attempts, concurrently.
@@ -193,19 +205,18 @@ class ConversationSimulator(BaseProcess):
         all_attempts_verdicts: Dict[str, List[str]] = defaultdict(list)
 
         async def simulate_attempt(attempt_number: int) -> Dict[str, Any]:
-            logger.info(f"{_LOG} Running attempt: {attempt_number + 1}/{attempts}")
+            logger.info(f"{_LOG} Running attempt: {attempt_number + 1}/{attempts}\n---")
             start_time = time.time()
 
             collected_scores: Dict[str, List[Any]] = defaultdict(list)
             collected_verdicts: Dict[str, List[str]] = defaultdict(list)
 
-            initial_interaction_results = await self.simulate_interactions(
+            interaction_results = await self.simulate_interactions(
                 script=script,
                 evaluation_verdicts=collected_verdicts,
                 collected_scores=collected_scores,
             )
 
-            logger.info(f"{_LOG} collected_scores: {collected_scores}\n---")
             single_attempt_scores = calculate_average_scores(collected_scores)
 
             for target, scores in single_attempt_scores.items():
@@ -225,7 +236,7 @@ class ConversationSimulator(BaseProcess):
                 "attempt": attempt_number + 1,
                 "script_id": script.id,
                 "total_duration": elapsed_time,
-                "interaction_results": initial_interaction_results,
+                "interaction_results": interaction_results,
                 "evaluation_verdicts": collected_verdicts,
                 "average_scores": single_attempt_scores,
             }
@@ -238,10 +249,6 @@ class ConversationSimulator(BaseProcess):
         for judge_, verdicts_ in all_attempts_verdicts.items():
             self.evaluation_verdicts[judge_].extend(verdicts_)
 
-        logger.info(
-            f"{_LOG} average scores:\n{average_scores}\n---"
-        )
-
         return {
             "script_id": script.id,
             "attempts": attempt_results,
@@ -324,8 +331,6 @@ class ConversationSimulator(BaseProcess):
                 reference_guardrail=reference_guardrail_flag,
             )
 
-            logger.info(f"{_LOG} Evaluation results:\n{evaluation_results.model_dump()}\n")
-
             self.store_evaluation_results(
                 results=evaluation_results,
                 evaluation_verdicts=evaluation_verdicts,
@@ -333,9 +338,7 @@ class ConversationSimulator(BaseProcess):
             )
 
             elapsed_time = time.time() - start_time
-            logger.info(
-                f"{_LOG} Interaction simulation complete in {elapsed_time:.2f} seconds.\n---"
-            )
+            logger.info(f"{_LOG} Interaction simulation complete in {elapsed_time:.2f} seconds.\n---")
 
             result = {
                 "user_message": user_message,
@@ -494,7 +497,7 @@ class ConversationSimulator(BaseProcess):
             collected_scores (Dict[str, List[Any]]): The collected scores.
         """
         for provider in results.judge_evaluations.keys():
-            evaluation_verdicts[f"{provider}_verdicts_summary"].append(
+            evaluation_verdicts[f"{provider}"].append(
                 results.judge_evaluations.get(provider, "").justification
             )
 

levelapp/simulator/utils.py

@@ -1,17 +1,18 @@
 """
 'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
 """
+import ast
 import json
-
 import httpx
-import arrow
 
 from uuid import UUID
-from typing import Dict, Any, Optional, List, Union
+from string import Template
+from typing import Any, Dict, List, Union
 
-from openai import OpenAI
 from pydantic import ValidationError
 
+from levelapp.clients import ClientRegistry
+from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
 from levelapp.simulator.schemas import InteractionResults
 from levelapp.aspects import MonitoringAspect, MetricType, logger
 
@@ -48,7 +49,14 @@ def extract_interaction_details(
             missing_keys = required_keys - response_dict.keys()
             logger.warning(f"[extract_interaction_details] Missing data: {missing_keys}]")
 
-        return InteractionResults.model_validate(response_dict)
+        output = {}
+        for k, v in template.items():
+            output[k] = Template(v).safe_substitute(response_dict)
+
+        raw_value = output.get("generated_metadata", {})
+        output["generated_metadata"] = ast.literal_eval(raw_value) if isinstance(raw_value, str) else raw_value
+
+        return InteractionResults.model_validate(output)
 
     except json.JSONDecodeError as e:
         logger.error(f"[extract_interaction_details] Failed to extract details:\n{e}")
@@ -64,7 +72,7 @@ async def async_interaction_request(
         url: str,
         headers: Dict[str, str],
         payload: Dict[str, Any],
-) -> Optional[httpx.Response]:
+) -> httpx.Response | None:
     """
     Perform an asynchronous interaction request.
 
@@ -74,7 +82,7 @@ async def async_interaction_request(
         payload (Dict[str, Any]): The payload to send in the request.
 
     Returns:
-        Optional[httpx.Response]: The response from the interaction request, or None if an error occurred.
+        httpx.Response: The response from the interaction request, or None if an error occurred.
     """
     try:
         async with httpx.AsyncClient(timeout=180) as client:
@@ -92,42 +100,6 @@ async def async_interaction_request(
     return None
 
 
-def parse_date_value(raw_date_value: Optional[str], default_date_value: Optional[str] = "") -> str:
-    """
-    Cleans and parses a dehumanized relative date string to ISO format.
-
-    Args:
-        raw_date_value (Optional[str]): The raw date value to parse.
-        default_date_value (Optional[str]): The default value to return if parsing fails. Defaults to an empty string.
-
-    Returns:
-        str: The parsed date in ISO format, or the default value if parsing fails.
-    """
-    if not raw_date_value:
-        logger.info(f"[parse_date_value] No raw value provided. returning default: '{default_date_value}'")
-        return default_date_value
-
-    clean = raw_date_value.replace("{{", "").replace("}}", "").replace("_", " ").strip()
-    clean += 's' if not clean.endswith('s') else clean
-
-    try:
-        arw = arrow.utcnow()
-        parsed_date = arw.dehumanize(clean).utcnow().format('YYYY-MM-DD')
-        return parsed_date
-
-    except arrow.parser.ParserError as e:
-        logger.error(f"[parse_date_value] Failed to parse date: '{clean}'\nParserError: {str(e)}", exc_info=True)
-        return default_date_value
-
-    except ValueError as e:
-        logger.error(f"[parse_date_value] Invalid date value: '{clean}'\nValueError: {str(e)}", exc_info=True)
-        return default_date_value
-
-    except Exception as e:
-        logger.error(f"[parse_date_value] Unexpected error.\nException: {str(e)}", exc_info=True)
-        return default_date_value
-
-
 @MonitoringAspect.monitor(
     name="average_calc",
     category=MetricType.SCORING,
@@ -157,45 +129,35 @@ def calculate_average_scores(scores: Dict[str, Union[List[float], float]]) -> Di
 
 
 @MonitoringAspect.monitor(name="summarization", category=MetricType.API_CALL)
-def summarize_verdicts(verdicts: List[str], judge: str, max_bullets: int = 5) -> List[str]:
-    """
-    Summarize the justifications for each judge.
-
-    Args:
-        verdicts (List[str]): A list of justifications.
-        judge (str): The judge or evaluator (provider) name for context.
-        max_bullets (int): The maximum number of bullets allowed per judge.
-
-    Returns:
-        List[str]: The summarized justifications.
-    """
-    if not verdicts:
-        return []
-
-    prompt = f"""
-    You are reviewing evaluation justifications from LL judges about replies generated by a virtual leasing agent.\n
-    Each justification contains the judge's assessment of how well the agent's response matched the expected reply.\n
-    Your task is to identify and summarize only the **negative points**, such as errors, misunderstandings,
-    missing information, or failure to meet expectations.\n
-    Return up to {max_bullets} bullet points. Be concise and start each point with '- '\n\n
-    ---
-    - Judge: {judge}
-    - Justifications:\n{chr(10).join(verdicts)}\n
-    """
-
-    client = OpenAI()
+def summarize_verdicts(
+        verdicts: List[str],
+        judge: str,
+        max_bullets: int = 5
+) -> List[str]:
+    client_registry = ClientRegistry()
+    client = client_registry.get(provider=judge)
 
     try:
-        result = client.chat.completions.create(
-            model="gpt-4o-mini",
-            temperature=0,
-            messages=[{"role": "user", "content": prompt}]
-        ).choices[0].message.content
+        verdicts = chr(10).join(verdicts)
+        prompt = SUMMARIZATION_PROMPT_TEMPLATE.format(max_bullets=max_bullets, judge=judge, verdicts=verdicts)
+        response = client.call(message=prompt)
+        parsed = client.parse_response(response=response)
+        striped = parsed.get("output", "").strip("")
+        bullet_points = [point.strip() for point in striped.split("- ") if point.strip()]
 
-        bullet_points = [point.strip() for point in result.split('- ') if point.strip()]
-
-        return bullet_points
+        return bullet_points[:max_bullets]
 
     except Exception as e:
         logger.error(f"[summarize_justifications] Error during summarization: {str(e)}", exc_info=True)
         return []
+
+
+# if __name__ == '__main__':
+#     template = {'generated_reply': '${agent_reply}', 'generated_metadata': '${generated_metadata}'}
+#     response_dict = {
+#         'agent_reply': "I'd be happy to help you book something for 10 AM.",
+#         'generated_metadata': {'appointment_type': 'Cardiology', 'date': 'next Monday', 'time': '10 AM'}
+#     }
+#
+#     result = extract_interaction_details(response_dict, template)
+#     print(f"result: {result.model_dump()}")

levelapp/workflow/base.py

@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
 from pydantic import ValidationError
 from functools import partial
 from pathlib import Path
-from typing import Any
+from typing import Any, Dict
 
 from levelapp.core.base import BaseProcess
 from levelapp.simulator.schemas import ScriptsBatch
@@ -60,15 +60,32 @@ class BaseWorkflow(ABC):
             self._results = await loop.run_in_executor(None, func, None)
 
     def collect_results(self) -> Any:
-        """Return unified results structure."""
+        """
+        Return unified results structure.
+
+        Returns:
+            The simulation results.
+        """
         return self._results
 
     @abstractmethod
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        """
+        Abstract method for setting up the configured process.
+
+        Args:
+            context (WorkflowContext): The workflow context.
+        """
         raise NotImplementedError
 
     @abstractmethod
     def _load_input_data(self, context: WorkflowContext) -> Any:
+        """
+        Abstract method for loading reference data.
+
+        Args:
+            context (WorkflowContext): The workflow context.
+        """
         raise NotImplementedError
 
 
@@ -77,6 +94,15 @@ class SimulatorWorkflow(BaseWorkflow):
         super().__init__(name="ConversationSimulator", context=context)
 
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        """
+        Concrete implementation for setting up the simulation workflow.
+
+        Args:
+            context (WorkflowContext): The workflow context for the simulation workflow.
+
+        Returns:
+            ConversationSimulator instance.
+        """
         simulator = ConversationSimulator()
         simulator.setup(
             repository=context.repository,
@@ -86,7 +112,16 @@ class SimulatorWorkflow(BaseWorkflow):
         )
         return simulator
 
-    def _load_input_data(self, context: WorkflowContext) -> Any:
+    def _load_input_data(self, context: WorkflowContext) -> Dict[str, Any]:
+        """
+        Concrete implementation for loading the reference data.
+
+        Args:
+            context (WorkflowContext): The workflow context for the simulation workflow.
+
+        Returns:
+            Dict[str, Any]: The reference data.
+        """
         loader = DataLoader()
         if "reference_data" in context.inputs:
             data_config = context.inputs["reference_data"]

levelapp/workflow/config.py

@@ -2,6 +2,7 @@
 from typing import List, Dict, Any, Optional
 from pydantic import BaseModel, Field
 
+from levelapp.aspects import logger
 from levelapp.config.endpoint import EndpointConfig
 from levelapp.core.schemas import WorkflowType, RepositoryType, EvaluatorType
 
@@ -47,19 +48,45 @@ class WorkflowConfig(BaseModel):
         extra = "allow"
 
     @classmethod
-    def load(cls, path: Optional[str] = None) -> "WorkflowConfig":
-        """Load workflow configuration from a YAML/JSON file."""
+    def load(cls, path: str | None = None) -> "WorkflowConfig":
+        """
+        Load workflow configuration from a YAML/JSON file.
+
+        Args:
+            path (str): YAML/JSON configuration file path.
+
+        Returns:
+            WorkflowConfig: An instance of WorkflowConfig.
+        """
         from levelapp.aspects.loader import DataLoader
 
         loader = DataLoader()
         config_dict = loader.load_raw_data(path=path)
+        logger.info(f"[{cls.__name__}] Workflow configuration loaded from '{path}' file content")
         return cls.model_validate(config_dict)
 
     @classmethod
     def from_dict(cls, content: Dict[str, Any]) -> "WorkflowConfig":
-        """Load workflow configuration from an in-memory dict."""
+        """
+        Load workflow configuration from an in-memory dict.
+
+        Args:
+            content (dict): Workflow configuration content.
+
+        Returns:
+            WorkflowConfig: An instance of WorkflowConfig.
+        """
+        logger.info(f"[{cls.__name__}] Workflow configuration loaded from provided content")
         return cls.model_validate(content)
 
     def set_reference_data(self, content: Dict[str, Any]) -> None:
-        """Load referer data from an in-memory dict."""
+        """
+        Load referer data from an in-memory dict.
+
+        Args:
+            content (dict): Workflow configuration content.
+
+        """
         self.reference_data.data = content
+        logger.info(f"[{self.__class__.__name__}] Reference data loaded from provided content")
+

levelapp/workflow/context.py

@@ -51,7 +51,6 @@ class WorkflowContextBuilder:
             inputs["reference_data"] = self.config.reference_data.data
         else:
             inputs["reference_data_path"] = self.config.reference_data.path
-            print(f"[WorkflowContextBuilder] reference data path: {inputs['reference_data_path']}")
 
         return WorkflowContext(
             config=self.config,

levelapp/workflow/factory.py

@@ -8,7 +8,6 @@ from levelapp.workflow.runtime import WorkflowContext
 
 class MainFactory:
     """Central factory for workflows."""
-
     _workflow_map: Dict[WorkflowType, Callable[[WorkflowContext], BaseWorkflow]] = {
         WorkflowType.SIMULATOR: lambda ctx: SimulatorWorkflow(ctx),
         WorkflowType.COMPARATOR: lambda ctx: ComparatorWorkflow(ctx),
@@ -16,7 +15,15 @@ class MainFactory:
 
     @classmethod
     def create_workflow(cls, context: WorkflowContext) -> BaseWorkflow:
-        """Create workflow using the given runtime context."""
+        """
+        Create workflow using the given runtime context.
+
+        Args:
+            context (WorkflowContext): the provided workflow context.
+
+        Returns:
+            BaseWorkflow: the built workflow instance from the provided context.
+        """
         wf_type = context.config.process.workflow_type
         builder = cls._workflow_map.get(wf_type)
         if not builder:
@@ -25,5 +32,11 @@ class MainFactory:
 
     @classmethod
     def register_workflow(cls, wf_type: WorkflowType, builder: Callable[[WorkflowContext], BaseWorkflow]) -> None:
-        """Register a new workflow implementation."""
+        """
+        Register a new workflow implementation.
+
+        Args:
+            wf_type (WorkflowType): the workflow type.
+            builder (Callable[[WorkflowContext], BaseWorkflow]): the workflow builder.
+        """
         cls._workflow_map[wf_type] = builder

{levelapp-0.1.1.dist-info → levelapp-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: levelapp
-Version: 0.1.1
+Version: 0.1.2
 Summary: LevelApp is an evaluation framework for AI/LLM-based software application. [Powered by Norma]
 Project-URL: Homepage, https://github.com/levelapp-org
 Project-URL: Repository, https://github.com/levelapp-org/levelapp-framework
@@ -17,14 +17,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Testing
 Requires-Python: >=3.12
-Requires-Dist: arrow>=1.3.0
 Requires-Dist: google-api-core>=2.25.1
 Requires-Dist: google-auth>=2.40.3
 Requires-Dist: google-cloud-firestore>=2.21.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: humanize>=4.13.0
 Requires-Dist: numpy>=2.3.2
-Requires-Dist: openai>=1.99.9
 Requires-Dist: pandas-stubs==2.3.0.250703
 Requires-Dist: pandas>=2.3.1
 Requires-Dist: pydantic>=2.11.7
@@ -133,7 +131,7 @@ endpoint:
     generated_metadata: "${generated_metadata}"
 
 repository:
-  type: FIRESTORE # Pick one of the following: FIRESTORE, FILESYSTEM, MONGODB.
+  type: FIRESTORE # Pick one of the following: FIRESTORE, FILESYSTEM
   project_id: "(default)"
   database_name: ""
 ```
@@ -220,14 +218,14 @@ To run an evaluation:
 
 ```python
 if __name__ == "__main__":
-    from levelapp.workflow.schemas import WorkflowConfig
+    from levelapp.workflow import WorkflowConfig
     from levelapp.core.session import EvaluationSession
 
     # Load configuration from YAML
     config = WorkflowConfig.load(path="../data/workflow_config.yaml")
 
-    # Run evaluation session
-    with EvaluationSession(session_name="test-session-1", workflow_config=config) as session:
+    # Run evaluation session (You can enable/disable the monitoring aspect)
+    with EvaluationSession(session_name="test-session-1", workflow_config=config, enable_monitoring=False) as session:
         session.run()
         results = session.workflow.collect_results()
         print("Results:", results)
@@ -243,14 +241,13 @@ if __name__ == "__main__":
     from levelapp.workflow import WorkflowConfig
     from levelapp.core.session import EvaluationSession
 
-    # Firestore -> retrieve endpoint config -> data => config_dict
-
+    
     config_dict = {
         "process": {"project_name": "test-project", "workflow_type": "SIMULATOR", "evaluation_params": {"attempts": 2}},
-        "evaluation": {"evaluators": ["JUDGE"], "providers": ["openai", "ionos"]},
+        "evaluation": {"evaluators": ["JUDGE", "REFERENCE"], "providers": ["openai", "ionos"], "metrics_map": {"field_1": "EXACT"}},
         "reference_data": {"path": "", "data": {}},
         "endpoint": {"base_url": "http://127.0.0.1:8000", "api_key": "key", "model_id": "model"},
-        "repository": {"type": "FIRESTORE", "source": "IN_MEMORY", "metrics_map": {"field_1": "EXACT"}},
+        "repository": {"type": "FIRESTORE", "source": "IN_MEMORY"},
     }
 
     content = {

{levelapp-0.1.1.dist-info → levelapp-0.1.2.dist-info}/RECORD

@@ -2,9 +2,9 @@ levelapp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 levelapp/aspects/__init__.py,sha256=_OaPcjTWBizqcUdDVj5aYue7lG9ytjQGLhPvReriKnU,326
 levelapp/aspects/loader.py,sha256=xWpcWtS25zbVhZ0UnIJEcQA9klajKk10TLK4j1IStH0,9543
 levelapp/aspects/logger.py,sha256=MJ9HphyHYkTE5-ajA_WuMUTM0qQzd0WIP243vF-pj3M,1698
-levelapp/aspects/monitor.py,sha256=lXGgxxeKZNkL5XbC_0ybES75gaRA9SuspHLHAEDbMR8,21981
+levelapp/aspects/monitor.py,sha256=KREhG_KH4f7okyheexaeKykwhzzQsFl1USC-k9YPXfQ,22065
 levelapp/aspects/sanitizer.py,sha256=zUqgb76tXJ8UUYtHp0Rz7q9PZjAHpSpHPPFfGTjjQNg,5229
-levelapp/clients/__init__.py,sha256=bbSFkYIODxEFatET4CFsKGC4m7Ng_tUkTGNcVBMa9h0,4169
+levelapp/clients/__init__.py,sha256=of3Zdkag634COXH_ca4hxXkERT8X44QS0IgfWu2yWqY,4084
 levelapp/clients/anthropic.py,sha256=Bxp-HffcIPLwM9BLcTR7n-D8ZXYVWCmbr2oH33fKV04,4030
 levelapp/clients/ionos.py,sha256=GFkLSeu8epFZV44GbNO3h1fRCKcfxscHMTFY0kPfh3o,4267
 levelapp/clients/mistral.py,sha256=e1NRvP9qN7O2zWAzBbgdQmmUDHQfCRLtVKDJCrh0DNA,3777
@@ -13,18 +13,19 @@ levelapp/comparator/__init__.py,sha256=ynmc0mrx-JbcCqLH-z4hOVezqGocDbDQGqgbhWy2x
 levelapp/comparator/comparator.py,sha256=yk0FWREnWKhIbXlsYpieqPJPqrlWXzyBMjVSznGqKY8,8183
 levelapp/comparator/extractor.py,sha256=vJ9iEoWAtXo2r9r7X72uUQPKW3UZE9Kx3uIjCufEp9k,3910
 levelapp/comparator/schemas.py,sha256=lUAQzEyStidt2ePQgV2zq-An5MLBrVSw6t8fB0FQKJs,1803
-levelapp/comparator/scorer.py,sha256=x5tU28SQekCNXmtK6wxr9ht7MZkqI78eYmZA6srkp4o,9167
+levelapp/comparator/scorer.py,sha256=LBRy8H11rXulSa-k40BcycPcMAHgdUm13qS7ibWHq6I,9032
 levelapp/comparator/utils.py,sha256=Eu48nDrNzFr0lwAJJS0aNhKsAWQ72syTEWYMNYfg764,4331
 levelapp/config/__init__.py,sha256=9oaajE5zW-OVWOszUzMAG6nHDSbLQWa3KT6bVoSvzRA,137
-levelapp/config/endpoint.py,sha256=ll34rZ0KRmUwI81EWJ3HX9i6pziq2YrQb84kv4ErymI,7649
-levelapp/config/prompts.py,sha256=crjOk01weLz5_IdF6dDZWPfSmiKNL8SgnbThyf4Jz2o,1345
+levelapp/config/endpoint.py,sha256=B-uIEKF-0_Y6Vo8MZ8eoCZocRkghijrdpwT3zq0FDLk,7647
+levelapp/config/endpoint_.py,sha256=-abrIYKbFPLxTqNst-zbCI4MpMCmCMe9VZ6O8OwNRiE,1629
+levelapp/config/prompts.py,sha256=NXOKRp5l1VQ9LO0pUojVH6TDJhWyZImsAvZEz2QiD9k,2206
 levelapp/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 levelapp/core/base.py,sha256=oh4OkKgwGxmw_jgjX6wrBoK0KPc1JvCMZfbZP_mGmIg,12453
-levelapp/core/schemas.py,sha256=UrUnU6h90uqS1LpcFqmMEpgWJ3772ZD5QBIytczmAbE,469
-levelapp/core/session.py,sha256=6bP6s3iWxofWL9LT4qv90VGOntAIa6PBJ_EaWZT0Ur4,7918
+levelapp/core/schemas.py,sha256=E47d93MMOj4eRYZIqUyLBiE5Ye7WgwkOJPOWQ6swRmo,465
+levelapp/core/session.py,sha256=6utDbLdg6DjwHL5dP-4wGe4_f7gFgEukuNNeOnbCbtA,9035
 levelapp/evaluator/__init__.py,sha256=K-P75Q1FXXLCNqH1wyhT9sf4y2R9a1qR5449AXEsY1k,109
-levelapp/evaluator/evaluator.py,sha256=SSveWDIXVg9CTLqexAZJSRpR_wtd5f1bD_s5dG5HJyc,10544
-levelapp/metrics/__init__.py,sha256=1y4gDLOu2Jz4QVIgPH-v9YMgaWOFr263tYLUTiFJ-fc,1965
+levelapp/evaluator/evaluator.py,sha256=JCRgQps9GKlJBDYw9xzVrC2_aGy0GhGAJ0ZkSC_IWWA,10806
+levelapp/metrics/__init__.py,sha256=x8iTaeDezJyQ9-NFe8GGvzwIBhyAJHWSRfBE3JRX-PE,1878
 levelapp/metrics/embedding.py,sha256=wvlT8Q5DjDT6GrAIFtc5aFbA_80hDLUXMP4RbSpSwHE,115
 levelapp/metrics/exact.py,sha256=Kb13nD2OVLrl3iYHaXrxDfrxDuhW0SMVvLAEXPaJtlY,6235
 levelapp/metrics/fuzzy.py,sha256=Rg8ashzMxtQwKO-z_LLzdj2PDIRqL4CBw6PGRf9IBrI,2598
@@ -33,17 +34,17 @@ levelapp/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 levelapp/repository/__init__.py,sha256=hNmFRZ7kKJN1mMlOHeW9xf0j9Q7gqTXYJ3hMCzk9to4,79
 levelapp/repository/firestore.py,sha256=K9JgxsNCelAKtzTDv19c1dHRlitMeRzo7H3caTlKuF8,10369
 levelapp/simulator/__init__.py,sha256=8Dz8g7rbpBZX3WoknVmMVoWm_VT72ZL9BABOF1xFpqs,83
-levelapp/simulator/schemas.py,sha256=eEFNNWiAJ8FAIObWcFWF1pL9LmjWwz_2Y-yfY3pHESc,4061
-levelapp/simulator/simulator.py,sha256=RKzpV-yDmtugJ3ScJTTA9MwxTdD-oEkwRZLm8N_scjg,19972
-levelapp/simulator/utils.py,sha256=qx0JdV1ZDQdTRVKa9xfq278ASrE44GBXSnJZJuhICqo,7365
+levelapp/simulator/schemas.py,sha256=YGprtuRZ4m33WBD35xj1Ib5EbMTdDCOp-wCykf-Iz-4,3700
+levelapp/simulator/simulator.py,sha256=ytgjUE9G8z2oW6rEzSdVkyncPiJcZCLWyvLR0cI5rk8,19895
+levelapp/simulator/utils.py,sha256=d1O4Q4Yl1lAAJWLJDiwNjwt0hD9bGlCan4a2G21E7yw,5930
 levelapp/workflow/__init__.py,sha256=27b2obG7ObhR43yd2uH-R0koRB7-DG8Emnvrq8EjsTA,193
-levelapp/workflow/base.py,sha256=t-vJzwv_OJ9W_pORySJwZq9IENGbWAF3-9-7ozaKDPk,4637
-levelapp/workflow/config.py,sha256=ClQaKSWxj7rFcOEQ4budmgOqMBskg6wAibf_gzqUf1o,2142
-levelapp/workflow/context.py,sha256=DzyZEb8WHug6vWfzf7BIjZAwtmv43HqgbaB20Pw3eWo,2660
-levelapp/workflow/factory.py,sha256=PZHp3AVt61Eop3HwGQDfbO0ju5k7rvNDHKy09eywMTQ,1245
+levelapp/workflow/base.py,sha256=1A_xKSBOmVjfMbRBcNhDK6G17SEjqRIm-XjMw45IPC4,5596
+levelapp/workflow/config.py,sha256=MlHt1PsXD09aukB93fvKTew0D8WD4_jdnO93Nn6b2U0,2923
+levelapp/workflow/context.py,sha256=gjAZXHEdlsXqWY6DbXOfKXNbxQbahRPSnNzyWDqryPU,2559
+levelapp/workflow/factory.py,sha256=z1ttJmI59sU9HgOvPo3ixUJ_oPv838XgehfuOorlTt8,1634
 levelapp/workflow/registration.py,sha256=VHUHjLHXad5kjcKukaEOIf7hBZ09bT3HAzVmIT08aLo,359
 levelapp/workflow/runtime.py,sha256=cFyXNWXSuURKbrMDHdkTcjeItM9wHP-5DPljntwYL5g,686
-levelapp-0.1.1.dist-info/METADATA,sha256=ozbAgnWY4gl149zqzPgYS7-qkKGutJFb9qL0CoYHbh0,12500
-levelapp-0.1.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-levelapp-0.1.1.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-levelapp-0.1.1.dist-info/RECORD,,
+levelapp-0.1.2.dist-info/METADATA,sha256=d0ZoS0BTpHpq65Q1KLeqlN12XQiCAyJ-zPP17T22D6o,12446
+levelapp-0.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+levelapp-0.1.2.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp-0.1.2.dist-info/RECORD,,