levelapp 0.1.3__py3-none-any.whl → 0.1.5__py3-none-any.whl

levelapp/simulator/schemas.py

@@ -9,7 +9,7 @@ from uuid import UUID, uuid4
 from datetime import datetime
 
 from typing import Dict, Any, List
-from pydantic import BaseModel, Field, computed_field, field_validator
+from pydantic import BaseModel, Field, computed_field
 
 from levelapp.evaluator.evaluator import JudgeEvaluationResults
 
@@ -24,13 +24,12 @@ class InteractionLevel(str, Enum):
 class Interaction(BaseModel):
     """Represents a single interaction within a conversation."""
     id: UUID = Field(default_factory=uuid4, description="Interaction identifier")
+    user_message_path: str = Field(..., description="Path of the user message in the request payload")
     user_message: str = Field(..., description="The user's query message")
-    # generated_reply: str = Field(..., description="The agent's reply message")
     reference_reply: str = Field(..., description="The preset reference message")
     interaction_type: InteractionLevel = Field(default=InteractionLevel.INITIAL, description="Type of interaction")
     reference_metadata: Dict[str, Any] = Field(default_factory=dict, description="Expected metadata")
-    # generated_metadata: Dict[str, Any] = Field(default_factory=dict, description="Extracted metadata")
-    guardrail_flag: bool = Field(default=False, description="Flag for guardrail signaling")
+    guardrail_flag: Any = Field(default=False, description="Flag for guardrail signaling")
     request_payload: Dict[str, Any] = Field(default_factory=dict, description="Additional request payload")
 
 
@@ -40,6 +39,7 @@ class ConversationScript(BaseModel):
     interactions: List[Interaction] = Field(default_factory=list, description="List of interactions")
     description: str = Field(default="no-description", description="A short description of the conversation")
     details: Dict[str, str] = Field(default_factory=dict, description="Conversation details")
+    variable_request_schema: bool = Field(default=False, description="The payload schema changes for each request")
 
 
 class ScriptsBatch(BaseModel):

levelapp/simulator/simulator.py

@@ -1,7 +1,6 @@
 """
 'simulators/service.py': Service layer to manage conversation simulation and evaluation.
 """
-import json
 import time
 import asyncio
 
@@ -9,8 +8,12 @@ from datetime import datetime
 from collections import defaultdict
 from typing import Dict, Any, List
 
-from levelapp.core.base import BaseRepository, BaseProcess, BaseEvaluator
-from levelapp.config.endpoint import EndpointConfig
+
+from levelapp.core.base import BaseProcess, BaseEvaluator
+from levelapp.endpoint.client import EndpointConfig
+from levelapp.endpoint.manager import EndpointConfigManager
+
+from levelapp.core.schemas import EvaluatorType
 from levelapp.simulator.schemas import (
     InteractionEvaluationResults,
     ScriptsBatch,
@@ -18,13 +21,10 @@ from levelapp.simulator.schemas import (
     SimulationResults
 )
 from levelapp.simulator.utils import (
-    extract_interaction_details,
-    async_interaction_request,
     calculate_average_scores,
     summarize_verdicts,
 )
 from levelapp.aspects import logger
-from levelapp.core.schemas import EvaluatorType
 
 
 class ConversationSimulator(BaseProcess):
@@ -32,29 +32,26 @@ class ConversationSimulator(BaseProcess):
 
     def __init__(
         self,
-        repository: BaseRepository | None = None,
+        endpoint_config: EndpointConfig | None = None,
         evaluators: Dict[EvaluatorType, BaseEvaluator] | None = None,
         providers: List[str] | None = None,
-        endpoint_config: EndpointConfig | None = None,
+
     ):
         """
         Initialize the ConversationSimulator.
 
         Args:
-            repository (BaseRepository): Service for saving simulation results.
+            endpoint_config (EndpointConfig): Endpoint configuration.
             evaluators (EvaluationService): Service for evaluating interactions.
             endpoint_config (EndpointConfig): Configuration object for VLA.
         """
         self._CLASS_NAME = self.__class__.__name__
 
-        self.repository = repository
+        self.endpoint_config = endpoint_config
         self.evaluators = evaluators
         self.providers = providers
-        self.endpoint_config = endpoint_config
 
-        self._url: str | None = None
-        self._credentials: str | None = None
-        self._headers: Dict[str, Any] | None = None
+        self.endpoint_cm = EndpointConfigManager()
 
         self.test_batch: ScriptsBatch | None = None
         self.evaluation_verdicts: Dict[str, List[str]] = defaultdict(list)
@@ -62,36 +59,34 @@ class ConversationSimulator(BaseProcess):
 
     def setup(
             self,
-            repository: BaseRepository,
+            endpoint_config: EndpointConfig,
             evaluators: Dict[EvaluatorType, BaseEvaluator],
             providers: List[str],
-            endpoint_config: EndpointConfig,
     ) -> None:
         """
         Initialize the ConversationSimulator.
 
         Args:
-            repository (BaseRepository): Repository object for storing simulation results.
+            endpoint_config (EndpointConfig): Configuration object for user endpoint API.
             evaluators (Dict[str, BaseEvaluator]): List of evaluator objects for evaluating interactions.
             providers (List[str]): List of LLM provider names.
-            endpoint_config (EndpointConfig): Configuration object for VLA.
+
         """
         _LOG: str = f"[{self._CLASS_NAME}][{self.setup.__name__}]"
         logger.info(f"{_LOG} Setting up the Conversation Simulator..")
 
-        self.repository = repository
+        if not self.endpoint_cm:
+            self.endpoint_cm = EndpointConfigManager()
+
+        self.endpoint_config = endpoint_config
+        self.endpoint_cm.set_endpoints(endpoints_config=[endpoint_config])
+
         self.evaluators = evaluators
         self.providers = providers
 
         if not self.providers:
             logger.warning(f"{_LOG} No LLM providers were provided. The Judge Evaluation process will not be executed.")
 
-        self.endpoint_config = endpoint_config
-
-        self._url = endpoint_config.full_url
-        self._credentials = endpoint_config.api_key.get_secret_value()
-        self._headers = endpoint_config.headers
-
     def get_evaluator(self, name: EvaluatorType) -> BaseEvaluator:
         """
         Retrieve an evaluator by name.
@@ -278,22 +273,37 @@ class ConversationSimulator(BaseProcess):
         start_time = time.time()
 
         results = []
+        contextual_mode: bool = script.variable_request_schema
+        logger.info(f"{_LOG} Contextual Mode ON: {contextual_mode}")
         interactions = script.interactions
 
         for interaction in interactions:
-            user_message = interaction.user_message
-            request_payload = interaction.request_payload
-            self.endpoint_config.variables = {
-                "user_message": user_message,
-                "request_payload": request_payload
-            }
-
-            response = await async_interaction_request(
-                url=self.endpoint_config.full_url,
-                headers=self.endpoint_config.headers,
-                payload=self.endpoint_config.request_payload,
+            if contextual_mode:
+                from levelapp.simulator.utils import set_by_path
+                request_payload = interaction.request_payload
+                user_message = interaction.user_message
+                set_by_path(
+                    obj=request_payload,
+                    path=interaction.user_message_path,
+                    value=user_message,
+                )
+                logger.info(f"{_LOG} Request payload (Variable Request Schema):\n{request_payload}\n---")
+            else:
+                user_message = interaction.user_message
+                request_payload = interaction.request_payload
+                request_payload.update({"user_message": user_message})
+                logger.info(f"{_LOG} Request payload (Configured Request Schema):\n{request_payload}\n---")
+
+            mappings = self.endpoint_config.response_mapping
+
+            response = await self.endpoint_cm.send_request(
+                endpoint_config=self.endpoint_config,
+                context=request_payload,
+                contextual_mode=contextual_mode
             )
 
+            logger.info(f"{_LOG} Response:\n{response}\n---")
+
             reference_reply = interaction.reference_reply
             reference_metadata = interaction.reference_metadata
             reference_guardrail_flag: bool = interaction.guardrail_flag
@@ -312,14 +322,18 @@ class ConversationSimulator(BaseProcess):
                 results.append(result)
                 continue
 
-            interaction_details = extract_interaction_details(
-                response=response.text,
-                template=self.endpoint_config.response_payload,
+            interaction_details = self.endpoint_cm.extract_response_data(
+                response=response,
+                mappings=mappings,
             )
 
-            generated_reply = interaction_details.generated_reply
-            generated_metadata = interaction_details.generated_metadata
-            extracted_guardrail_flag: bool = interaction_details.guardrail_flag
+            logger.info(f"{_LOG} Interaction details:\n{interaction_details}\n---")
+
+            generated_reply = interaction_details.get("agent_reply", "")
+            generated_metadata = interaction_details.get("metadata", {})
+            extracted_guardrail_flag = interaction_details.get("guardrail_flag", False)
+
+            logger.info(f"{_LOG} Generated reply:\n{generated_reply}\n---")
 
             evaluation_results = await self.evaluate_interaction(
                 user_input=user_message,
@@ -346,7 +360,7 @@ class ConversationSimulator(BaseProcess):
                 "reference_reply": reference_reply,
                 "generated_metadata": generated_metadata,
                 "reference_metadata": reference_metadata,
-                "guardrail_details": interaction_details.guardrail_flag,
+                "guardrail_details": extracted_guardrail_flag,
                 "evaluation_results": evaluation_results.model_dump(),
             }
 

levelapp/simulator/utils.py

@@ -1,103 +1,74 @@
 """
 'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
 """
-import ast
-import json
 import httpx
 
-from uuid import UUID
-from string import Template
 from typing import Any, Dict, List, Union
 
-from pydantic import ValidationError
 
 from levelapp.clients import ClientRegistry
 from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
-from levelapp.simulator.schemas import InteractionResults
 from levelapp.aspects import MonitoringAspect, MetricType, logger
 
 
-class UUIDEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, UUID):
-            return str(obj)
-        return json.JSONEncoder.default(self, obj)
-
-
-def extract_interaction_details(
-        response: str | Dict[str, Any],
-        template: Dict[str, Any],
-) -> InteractionResults:
+def set_by_path(obj: Dict, path: str, value: Any) -> None:
     """
-    Extract interaction details from a VLA response.
+    Sets a value in a nested dictionary using JSON path-like notation.
 
     Args:
-        response (str): The response text from the VLA.
-        template (Dict[str, Any]): The response schema/template.
+        obj (dict): Dictionary to modify.
+        path (str): Path (e.g., "a.b[0].c") indicating where to set the value.
+        value (Any): Value to assign at the specified path.
 
     Returns:
-        InteractionResults: The extracted interaction details.
+        None
     """
-    try:
-        response_dict = response if isinstance(response, dict) else json.loads(response)
-
-        if not isinstance(response_dict, dict):
-            raise ValueError("Response is not a valid dictionary")
-
-        required_keys = {value.strip("${}") for value in template.values()}
-        if not required_keys.issubset(response_dict.keys()):
-            missing_keys = required_keys - response_dict.keys()
-            logger.warning(f"[extract_interaction_details] Missing data: {missing_keys}]")
-
-        output = {}
-        for k, v in template.items():
-            output[k] = Template(v).safe_substitute(response_dict)
-
-        raw_value = output.get("generated_metadata", {})
-        output["generated_metadata"] = ast.literal_eval(raw_value) if isinstance(raw_value, str) else raw_value
-
-        return InteractionResults.model_validate(output)
-
-    except json.JSONDecodeError as e:
-        logger.error(f"[extract_interaction_details] Failed to extract details:\n{e}")
-        return InteractionResults()
-
-    except ValidationError as e:
-        logger.exception(f"[extract_interaction_details] Failed to create an InteractionResults instance:\n{e}")
-        return InteractionResults()
-
-
-@MonitoringAspect.monitor(name="interaction_request", category=MetricType.API_CALL)
-async def async_interaction_request(
-        url: str,
-        headers: Dict[str, str],
-        payload: Dict[str, Any],
-) -> httpx.Response | None:
-    """
-    Perform an asynchronous interaction request.
-
-    Args:
-        url (str): The URL to send the request to.
-        headers (Dict[str, str]): The headers to include in the request.
-        payload (Dict[str, Any]): The payload to send in the request.
-
-    Returns:
-        httpx.Response: The response from the interaction request, or None if an error occurred.
-    """
-    try:
-        async with httpx.AsyncClient(timeout=180) as client:
-            response = await client.post(url=url, headers=headers, json=payload)
-            response.raise_for_status()
-
-            return response
-
-    except httpx.HTTPStatusError as http_err:
-        logger.error(f"[async_interaction_request] HTTP error: {http_err.response.text}", exc_info=True)
-
-    except httpx.RequestError as req_err:
-        logger.error(f"[async_interaction_request] Request error: {str(req_err)}", exc_info=True)
-
-    return None
+    parts = path.split(".")
+    current = obj
+
+    for i, part in enumerate(parts):
+        is_last = i == len(parts) - 1
+
+        try:
+            # Handle list index access, e.g., key[0] or [1]
+            if '[' in part and ']' in part:
+                key, idx = part.split('[')
+                idx = int(idx.rstrip(']'))
+
+                # If we have a key before the list
+                if key:
+                    if key not in current or not isinstance(current[key], list):
+                        current[key] = []
+                    while len(current[key]) <= idx:
+                        current[key].append({})
+                    target = current[key]
+                else:
+                    if not isinstance(current, list):
+                        print("[set_by_path][WARNING] Expected a list at this level.")
+                        return
+                    while len(current) <= idx:
+                        current.append({})
+                    target = current
+
+                if is_last:
+                    target[idx] = value
+                else:
+                    if not isinstance(target[idx], dict):
+                        target[idx] = {}
+                    current = target[idx]
+
+            else:
+                # Regular dictionary key
+                if is_last:
+                    current[part] = value
+                else:
+                    if part not in current or not isinstance(current[part], dict):
+                        current[part] = {}
+                    current = current[part]
+
+        except (KeyError, IndexError, TypeError, AttributeError) as e:
+            print(f"[set_by_path][ERROR] Error type <{e.__class__.__name__}> : {e.args[0]}")
+            return
 
 
 @MonitoringAspect.monitor(

levelapp/workflow/base.py

@@ -7,6 +7,8 @@ from pathlib import Path
 from typing import Any, Dict
 
 from levelapp.core.base import BaseProcess
+from levelapp.endpoint.client import EndpointConfig
+from levelapp.endpoint.manager import EndpointConfigManager
 from levelapp.simulator.schemas import ScriptsBatch
 from levelapp.simulator.simulator import ConversationSimulator
 from levelapp.workflow.runtime import WorkflowContext
@@ -68,6 +70,18 @@ class BaseWorkflow(ABC):
         """
         return self._results
 
+    @abstractmethod
+    async def test_connection(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Abstract method for testing endpoint connection.
+
+        Args:
+            context (Dict[str, Any]): The context (request payload) to test connectivity with.
+
+        Returns:
+            The test connectivity result.
+        """
+
     @abstractmethod
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
         """
@@ -105,13 +119,30 @@ class SimulatorWorkflow(BaseWorkflow):
         """
         simulator = ConversationSimulator()
         simulator.setup(
-            repository=context.repository,
+            endpoint_config=context.endpoint,
             evaluators=context.evaluators,
             providers=context.providers,
-            endpoint_config=context.endpoint_config,
         )
+
         return simulator
 
+    async def test_connection(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Runs a connectivity test of the configured endpoint.
+
+        Args:
+            context (Dict[str, Any]): The request payload to send for testing.
+
+        Returns:
+            The test connectivity result.
+        """
+        endpoint_cm = EndpointConfigManager()
+        endpoint_cm.set_endpoints(endpoints_config=[self.context.endpoint])
+        tester = endpoint_cm.get_tester(endpoint_name=self.context.endpoint.name)
+        results = await tester.test(context=context)
+
+        return results
+
     def _load_input_data(self, context: WorkflowContext) -> Dict[str, Any]:
         """
         Concrete implementation for loading the reference data.

levelapp/workflow/config.py

@@ -3,7 +3,7 @@ from typing import List, Dict, Any, Optional
 from pydantic import BaseModel, Field
 
 from levelapp.aspects import logger
-from levelapp.config.endpoint import EndpointConfig
+from levelapp.endpoint.client import EndpointConfig
 from levelapp.core.schemas import WorkflowType, RepositoryType, EvaluatorType
 
 
@@ -39,9 +39,9 @@ class WorkflowConfig(BaseModel):
     Supports both file-based loading and in-memory dictionary creation.
     """
     process: ProcessConfig
+    endpoint: EndpointConfig
     evaluation: EvaluationConfig
     reference_data: ReferenceDataConfig
-    endpoint: EndpointConfig
     repository: RepositoryConfig
 
     class Config:
@@ -90,3 +90,7 @@ class WorkflowConfig(BaseModel):
         self.reference_data.data = content
         logger.info(f"[{self.__class__.__name__}] Reference data loaded from provided content")
 
+
+if __name__ == '__main__':
+    workflow_config = WorkflowConfig.load(path="../../src/data/workflow_config.yaml")
+    print(f"Workflow Configuration:\n{workflow_config.model_dump_json(indent=2)}")

levelapp/workflow/context.py

@@ -1,6 +1,7 @@
 """levelapp/workflow/context.py: Builds runtime WorkflowContext from WorkflowConfig."""
 from typing import Dict, Callable
 
+from levelapp.repository.filesystem import FileSystemRepository
 from levelapp.workflow.config import WorkflowConfig
 from levelapp.core.base import BaseRepository, BaseEvaluator
 from levelapp.workflow.runtime import WorkflowContext
@@ -19,6 +20,7 @@ class WorkflowContextBuilder:
         # Map repository type to constructor that accepts the WorkflowConfig
         self.repository_map: Dict[RepositoryType, Callable[[WorkflowConfig], BaseRepository]] = {
             RepositoryType.FIRESTORE: lambda cfg: FirestoreRepository(cfg),
+            RepositoryType.FILESYSTEM: lambda cfg: FileSystemRepository(cfg),
         }
 
         # Map evaluator type to constructor that accepts the WorkflowConfig
@@ -57,6 +59,6 @@ class WorkflowContextBuilder:
             repository=repository,
             evaluators=evaluators,
             providers=providers,
-            endpoint_config=endpoint_config,
+            endpoint=endpoint_config,
             inputs=inputs,
         )

levelapp/workflow/runtime.py

@@ -2,7 +2,7 @@
 from dataclasses import dataclass
 from typing import Dict, List, Any
 
-from levelapp.config import EndpointConfig
+from levelapp.endpoint.client import EndpointConfig
 from levelapp.core.base import BaseRepository, BaseEvaluator
 from levelapp.workflow.config import WorkflowConfig
 from levelapp.core.schemas import EvaluatorType
@@ -12,8 +12,8 @@ from levelapp.core.schemas import EvaluatorType
 class WorkflowContext:
     """Immutable data holder for workflow execution context."""
     config: WorkflowConfig
+    endpoint: EndpointConfig
     repository: BaseRepository
     evaluators: Dict[EvaluatorType, BaseEvaluator]
     providers: List[str]
-    endpoint_config: EndpointConfig
-    inputs: Dict[str, Any]
+    inputs: Dict[str, Any]