levelapp 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

levelapp/simulator/schemas.py

@@ -9,7 +9,7 @@ from uuid import UUID, uuid4
 from datetime import datetime
 
 from typing import Dict, Any, List
-from pydantic import BaseModel, Field, computed_field, field_validator
+from pydantic import BaseModel, Field, computed_field
 
 from levelapp.evaluator.evaluator import JudgeEvaluationResults
 
@@ -24,13 +24,12 @@ class InteractionLevel(str, Enum):
 class Interaction(BaseModel):
     """Represents a single interaction within a conversation."""
     id: UUID = Field(default_factory=uuid4, description="Interaction identifier")
+    user_message_path: str = Field(..., description="Path of the user message in the request payload")
     user_message: str = Field(..., description="The user's query message")
-    # generated_reply: str = Field(..., description="The agent's reply message")
     reference_reply: str = Field(..., description="The preset reference message")
     interaction_type: InteractionLevel = Field(default=InteractionLevel.INITIAL, description="Type of interaction")
     reference_metadata: Dict[str, Any] = Field(default_factory=dict, description="Expected metadata")
-    # generated_metadata: Dict[str, Any] = Field(default_factory=dict, description="Extracted metadata")
-    guardrail_flag: bool = Field(default=False, description="Flag for guardrail signaling")
+    guardrail_flag: Any = Field(default=False, description="Flag for guardrail signaling")
     request_payload: Dict[str, Any] = Field(default_factory=dict, description="Additional request payload")
 
 
@@ -40,6 +39,7 @@ class ConversationScript(BaseModel):
     interactions: List[Interaction] = Field(default_factory=list, description="List of interactions")
     description: str = Field(default="no-description", description="A short description of the conversation")
     details: Dict[str, str] = Field(default_factory=dict, description="Conversation details")
+    variable_request_schema: bool = Field(default=False, description="The payload schema changes for each request")
 
 
 class ScriptsBatch(BaseModel):

levelapp/simulator/simulator.py

@@ -1,7 +1,6 @@
 """
 'simulators/service.py': Service layer to manage conversation simulation and evaluation.
 """
-import json
 import time
 import asyncio
 
@@ -9,8 +8,12 @@ from datetime import datetime
 from collections import defaultdict
 from typing import Dict, Any, List
 
-from levelapp.core.base import BaseRepository, BaseProcess, BaseEvaluator
-from levelapp.config.endpoint import EndpointConfig
+
+from levelapp.core.base import BaseProcess, BaseEvaluator
+from levelapp.endpoint.client import EndpointConfig
+from levelapp.endpoint.manager import EndpointConfigManager
+
+from levelapp.core.schemas import EvaluatorType
 from levelapp.simulator.schemas import (
     InteractionEvaluationResults,
     ScriptsBatch,
@@ -18,13 +21,10 @@ from levelapp.simulator.schemas import (
     SimulationResults
 )
 from levelapp.simulator.utils import (
-    extract_interaction_details,
-    async_interaction_request,
     calculate_average_scores,
     summarize_verdicts,
 )
 from levelapp.aspects import logger
-from levelapp.core.schemas import EvaluatorType
 
 
 class ConversationSimulator(BaseProcess):
@@ -32,29 +32,26 @@ class ConversationSimulator(BaseProcess):
 
     def __init__(
         self,
-        repository: BaseRepository | None = None,
+        endpoint_config: EndpointConfig | None = None,
         evaluators: Dict[EvaluatorType, BaseEvaluator] | None = None,
         providers: List[str] | None = None,
-        endpoint_config: EndpointConfig | None = None,
+
     ):
         """
         Initialize the ConversationSimulator.
 
         Args:
-            repository (BaseRepository): Service for saving simulation results.
+            endpoint_config (EndpointConfig): Endpoint configuration.
             evaluators (EvaluationService): Service for evaluating interactions.
             endpoint_config (EndpointConfig): Configuration object for VLA.
         """
         self._CLASS_NAME = self.__class__.__name__
 
-        self.repository = repository
+        self.endpoint_config = endpoint_config
         self.evaluators = evaluators
         self.providers = providers
-        self.endpoint_config = endpoint_config
 
-        self._url: str | None = None
-        self._credentials: str | None = None
-        self._headers: Dict[str, Any] | None = None
+        self.endpoint_cm = EndpointConfigManager()
 
         self.test_batch: ScriptsBatch | None = None
         self.evaluation_verdicts: Dict[str, List[str]] = defaultdict(list)
@@ -62,36 +59,34 @@ class ConversationSimulator(BaseProcess):
 
     def setup(
             self,
-            repository: BaseRepository,
+            endpoint_config: EndpointConfig,
             evaluators: Dict[EvaluatorType, BaseEvaluator],
             providers: List[str],
-            endpoint_config: EndpointConfig,
     ) -> None:
         """
         Initialize the ConversationSimulator.
 
         Args:
-            repository (BaseRepository): Repository object for storing simulation results.
+            endpoint_config (EndpointConfig): Configuration object for user endpoint API.
             evaluators (Dict[str, BaseEvaluator]): List of evaluator objects for evaluating interactions.
             providers (List[str]): List of LLM provider names.
-            endpoint_config (EndpointConfig): Configuration object for VLA.
+
         """
         _LOG: str = f"[{self._CLASS_NAME}][{self.setup.__name__}]"
         logger.info(f"{_LOG} Setting up the Conversation Simulator..")
 
-        self.repository = repository
+        if not self.endpoint_cm:
+            self.endpoint_cm = EndpointConfigManager()
+
+        self.endpoint_config = endpoint_config
+        self.endpoint_cm.set_endpoints(endpoints_config=[endpoint_config])
+
         self.evaluators = evaluators
         self.providers = providers
 
         if not self.providers:
             logger.warning(f"{_LOG} No LLM providers were provided. The Judge Evaluation process will not be executed.")
 
-        self.endpoint_config = endpoint_config
-
-        self._url = endpoint_config.full_url
-        self._credentials = endpoint_config.api_key.get_secret_value()
-        self._headers = endpoint_config.headers
-
     def get_evaluator(self, name: EvaluatorType) -> BaseEvaluator:
         """
         Retrieve an evaluator by name.
@@ -278,22 +273,37 @@ class ConversationSimulator(BaseProcess):
         start_time = time.time()
 
         results = []
+        contextual_mode: bool = script.variable_request_schema
+        logger.info(f"{_LOG} Contextual Mode ON: {contextual_mode}")
         interactions = script.interactions
 
         for interaction in interactions:
-            user_message = interaction.user_message
-            request_payload = interaction.request_payload
-            self.endpoint_config.variables = {
-                "user_message": user_message,
-                "request_payload": request_payload
-            }
-
-            response = await async_interaction_request(
-                url=self.endpoint_config.full_url,
-                headers=self.endpoint_config.headers,
-                payload=self.endpoint_config.request_payload,
+            if contextual_mode:
+                from levelapp.simulator.utils import set_by_path
+                request_payload = interaction.request_payload
+                user_message = interaction.user_message
+                set_by_path(
+                    obj=request_payload,
+                    path=interaction.user_message_path,
+                    value=user_message,
+                )
+                logger.info(f"{_LOG} Request payload (Variable Request Schema):\n{request_payload}\n---")
+            else:
+                user_message = interaction.user_message
+                request_payload = interaction.request_payload
+                request_payload.update({"user_message": user_message})
+                logger.info(f"{_LOG} Request payload (Configured Request Schema):\n{request_payload}\n---")
+
+            mappings = self.endpoint_config.response_mapping
+
+            response = await self.endpoint_cm.send_request(
+                endpoint_config=self.endpoint_config,
+                context=request_payload,
+                contextual_mode=contextual_mode
             )
 
+            logger.info(f"{_LOG} Response:\n{response}\n---")
+
             reference_reply = interaction.reference_reply
             reference_metadata = interaction.reference_metadata
             reference_guardrail_flag: bool = interaction.guardrail_flag
@@ -312,14 +322,18 @@ class ConversationSimulator(BaseProcess):
                 results.append(result)
                 continue
 
-            interaction_details = extract_interaction_details(
-                response=response.text,
-                template=self.endpoint_config.response_payload,
+            interaction_details = self.endpoint_cm.extract_response_data(
+                response=response,
+                mappings=mappings,
             )
 
-            generated_reply = interaction_details.generated_reply
-            generated_metadata = interaction_details.generated_metadata
-            extracted_guardrail_flag: bool = interaction_details.guardrail_flag
+            logger.info(f"{_LOG} Interaction details:\n{interaction_details}\n---")
+
+            generated_reply = interaction_details.get("agent_reply", "")
+            generated_metadata = interaction_details.get("metadata", {})
+            extracted_guardrail_flag = interaction_details.get("guardrail_flag", False)
+
+            logger.info(f"{_LOG} Generated reply:\n{generated_reply}\n---")
 
             evaluation_results = await self.evaluate_interaction(
                 user_input=user_message,
@@ -346,7 +360,7 @@ class ConversationSimulator(BaseProcess):
                 "reference_reply": reference_reply,
                 "generated_metadata": generated_metadata,
                 "reference_metadata": reference_metadata,
-                "guardrail_details": interaction_details.guardrail_flag,
+                "guardrail_details": extracted_guardrail_flag,
                 "evaluation_results": evaluation_results.model_dump(),
             }
 

levelapp/simulator/utils.py

@@ -1,197 +1,74 @@
 """
 'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
 """
-import re
-import ast
-import json
 import httpx
 
-from uuid import UUID
-from string import Template
-from typing import Any, Dict, List, Union, Iterable
+from typing import Any, Dict, List, Union
 
-from pydantic import ValidationError
 
 from levelapp.clients import ClientRegistry
 from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
-from levelapp.simulator.schemas import InteractionResults
 from levelapp.aspects import MonitoringAspect, MetricType, logger
 
 
-class UUIDEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, UUID):
-            return str(obj)
-        return json.JSONEncoder.default(self, obj)
-
-
-_PLACEHOLDER_RE = re.compile(r"\$\{([^}]+)\}")  # captures inner name(s) of ${...}
-
-
-def _traverse_path(d: Dict[str, Any], path: str):
-    """Traverse a dot-separated path (payload.metadata.budget) and return value or None."""
-    parts = path.split(".")
-    cur = d
-    try:
-        for p in parts:
-            if isinstance(cur, dict) and p in cur:
-                cur = cur[p]
-            else:
-                return None
-        return cur
-    except Exception:
-        return None
-
-
-def _recursive_find(container: Any, target_key: str):
+def set_by_path(obj: Dict, path: str, value: Any) -> None:
     """
-    Recursively search container (dicts/lists) for the first occurrence of target_key.
-    Returns the value if found, else None.
-    """
-    if isinstance(container, dict):
-        # direct hit
-        if target_key in container:
-            return container[target_key]
-        # recurse into values
-        for v in container.values():
-            found = _recursive_find(v, target_key)
-            if found is not None:
-                return found
-        return None
-
-    if isinstance(container, list):
-        for item in container:
-            found = _recursive_find(item, target_key)
-            if found is not None:
-                return found
-        return None
-
-    # not a container
-    return None
-
-
-def _extract_placeholders(template_str: str) -> Iterable[str]:
-    """Return list of placeholder names in a template string (inner contents of ${...})."""
-    return [m.group(1) for m in _PLACEHOLDER_RE.finditer(template_str)]
-
-
-def extract_interaction_details(
-    response: str | Dict[str, Any],
-    template: Dict[str, Any],
-) -> InteractionResults:
-    """
-    Parse response (str or dict), look up placeholders recursively in the response and
-    use Template.safe_substitute with a mapping built from those lookups.
-    """
-    try:
-        response_dict = response if isinstance(response, dict) else json.loads(response)
-        print(f"response:\n{response_dict}\n--")
-        if not isinstance(response_dict, dict):
-            raise ValueError("Response is not a valid dictionary")
-
-        output: Dict[str, Any] = {}
-
-        for out_key, tpl_str in template.items():
-            # Build mapping for placeholders found in tpl_str
-            placeholders = _extract_placeholders(tpl_str)
-            mapping: Dict[str, str] = {}
-
-            for ph in placeholders:
-                value = None
-
-                # 1) If ph looks like a dotted path, try explicit path traversal first
-                if "." in ph:
-                    value = _traverse_path(response_dict, ph)
-
-                # 2) If not found yet, try recursive search for the bare key (last path segment)
-                if value is None:
-                    bare = ph.split(".")[-1]
-                    value = _recursive_find(response_dict, bare)
-
-                # Prepare mapping value for Template substitution:
-                # - dict/list -> JSON string (so substitution yields valid JSON text)
-                # - None -> empty string
-                # - otherwise -> str(value)
-                if isinstance(value, (dict, list)):
-                    try:
-                        mapping[ph] = json.dumps(value, ensure_ascii=False)
-                    except Exception:
-                        mapping[ph] = str(value)
-                elif value is None:
-                    mapping[ph] = ""
-                else:
-                    mapping[ph] = str(value)
-
-            # Perform substitution using Template (safe_substitute: missing keys left intact)
-            substituted = Template(tpl_str).safe_substitute(mapping)
-            output[out_key] = substituted
-
-        # Post-process generated_metadata if present: convert JSON text back to dict/list when possible
-        raw_meta = output.get("generated_metadata", {})
-        if isinstance(raw_meta, str) and raw_meta:
-            # Try json first (since we used json.dumps above for mapping)
-            try:
-                output["generated_metadata"] = json.loads(raw_meta)
-            except Exception:
-                # fallback to ast.literal_eval (handles Python dict strings)
-                try:
-                    output["generated_metadata"] = ast.literal_eval(raw_meta)
-                except Exception:
-                    # if parsing fails, keep the original raw string or use an empty dict
-                    output["generated_metadata"] = raw_meta
-
-        # If generated_metadata is empty string, normalize to {}
-        if output.get("generated_metadata") == "":
-            output["generated_metadata"] = {}
-
-        print(f"output:\n{output}\n---")
-        # Return validated model
-        return InteractionResults.model_validate(output)
-
-    except json.JSONDecodeError as e:
-        logger.error(f"[extract_interaction_details] Failed to parse JSON response: {e}")
-        return InteractionResults()
-
-    except ValidationError as e:
-        logger.exception(f"[extract_interaction_details] InteractionResults validation failed: {e}")
-        return InteractionResults()
-
-    except Exception as e:
-        logger.exception(f"[extract_interaction_details] Unexpected error: {e}")
-        return InteractionResults()
-
-
-@MonitoringAspect.monitor(name="interaction_request", category=MetricType.API_CALL)
-async def async_interaction_request(
-        url: str,
-        headers: Dict[str, str],
-        payload: Dict[str, Any],
-) -> httpx.Response | None:
-    """
-    Perform an asynchronous interaction request.
+    Sets a value in a nested dictionary using JSON path-like notation.
 
     Args:
-        url (str): The URL to send the request to.
-        headers (Dict[str, str]): The headers to include in the request.
-        payload (Dict[str, Any]): The payload to send in the request.
+        obj (dict): Dictionary to modify.
+        path (str): Path (e.g., "a.b[0].c") indicating where to set the value.
+        value (Any): Value to assign at the specified path.
 
     Returns:
-        httpx.Response: The response from the interaction request, or None if an error occurred.
+        None
     """
-    try:
-        async with httpx.AsyncClient(timeout=180) as client:
-            response = await client.post(url=url, headers=headers, json=payload)
-            response.raise_for_status()
-
-            return response
-
-    except httpx.HTTPStatusError as http_err:
-        logger.error(f"[async_interaction_request] HTTP error: {http_err.response.text}", exc_info=True)
+    parts = path.split(".")
+    current = obj
+
+    for i, part in enumerate(parts):
+        is_last = i == len(parts) - 1
+
+        try:
+            # Handle list index access, e.g., key[0] or [1]
+            if '[' in part and ']' in part:
+                key, idx = part.split('[')
+                idx = int(idx.rstrip(']'))
+
+                # If we have a key before the list
+                if key:
+                    if key not in current or not isinstance(current[key], list):
+                        current[key] = []
+                    while len(current[key]) <= idx:
+                        current[key].append({})
+                    target = current[key]
+                else:
+                    if not isinstance(current, list):
+                        print("[set_by_path][WARNING] Expected a list at this level.")
+                        return
+                    while len(current) <= idx:
+                        current.append({})
+                    target = current
+
+                if is_last:
+                    target[idx] = value
+                else:
+                    if not isinstance(target[idx], dict):
+                        target[idx] = {}
+                    current = target[idx]
 
-    except httpx.RequestError as req_err:
-        logger.error(f"[async_interaction_request] Request error: {str(req_err)}", exc_info=True)
+            else:
+                # Regular dictionary key
+                if is_last:
+                    current[part] = value
+                else:
+                    if part not in current or not isinstance(current[part], dict):
+                        current[part] = {}
+                    current = current[part]
 
-    return None
+        except (KeyError, IndexError, TypeError, AttributeError) as e:
+            print(f"[set_by_path][ERROR] Error type <{e.__class__.__name__}> : {e.args[0]}")
+            return
 
 
 @MonitoringAspect.monitor(

levelapp/workflow/base.py

@@ -7,6 +7,8 @@ from pathlib import Path
 from typing import Any, Dict
 
 from levelapp.core.base import BaseProcess
+from levelapp.endpoint.client import EndpointConfig
+from levelapp.endpoint.manager import EndpointConfigManager
 from levelapp.simulator.schemas import ScriptsBatch
 from levelapp.simulator.simulator import ConversationSimulator
 from levelapp.workflow.runtime import WorkflowContext
@@ -68,6 +70,18 @@ class BaseWorkflow(ABC):
         """
         return self._results
 
+    @abstractmethod
+    async def test_connection(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Abstract method for testing endpoint connection.
+
+        Args:
+            context (Dict[str, Any]): The context (request payload) to test connectivity with.
+
+        Returns:
+            The test connectivity result.
+        """
+
     @abstractmethod
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
         """
@@ -105,13 +119,30 @@ class SimulatorWorkflow(BaseWorkflow):
         """
         simulator = ConversationSimulator()
         simulator.setup(
-            repository=context.repository,
+            endpoint_config=context.endpoint,
             evaluators=context.evaluators,
             providers=context.providers,
-            endpoint_config=context.endpoint_config,
         )
+
         return simulator
 
+    async def test_connection(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Runs a connectivity test of the configured endpoint.
+
+        Args:
+            context (Dict[str, Any]): The request payload to send for testing.
+
+        Returns:
+            The test connectivity result.
+        """
+        endpoint_cm = EndpointConfigManager()
+        endpoint_cm.set_endpoints(endpoints_config=[self.context.endpoint])
+        tester = endpoint_cm.get_tester(endpoint_name=self.context.endpoint.name)
+        results = await tester.test(context=context)
+
+        return results
+
     def _load_input_data(self, context: WorkflowContext) -> Dict[str, Any]:
         """
         Concrete implementation for loading the reference data.

levelapp/workflow/config.py

@@ -3,7 +3,7 @@ from typing import List, Dict, Any, Optional
 from pydantic import BaseModel, Field
 
 from levelapp.aspects import logger
-from levelapp.config.endpoint import EndpointConfig
+from levelapp.endpoint.client import EndpointConfig
 from levelapp.core.schemas import WorkflowType, RepositoryType, EvaluatorType
 
 
@@ -39,9 +39,9 @@ class WorkflowConfig(BaseModel):
     Supports both file-based loading and in-memory dictionary creation.
     """
     process: ProcessConfig
+    endpoint: EndpointConfig
     evaluation: EvaluationConfig
     reference_data: ReferenceDataConfig
-    endpoint: EndpointConfig
     repository: RepositoryConfig
 
     class Config:
@@ -90,3 +90,7 @@ class WorkflowConfig(BaseModel):
         self.reference_data.data = content
         logger.info(f"[{self.__class__.__name__}] Reference data loaded from provided content")
 
+
+if __name__ == '__main__':
+    workflow_config = WorkflowConfig.load(path="../../src/data/workflow_config.yaml")
+    print(f"Workflow Configuration:\n{workflow_config.model_dump_json(indent=2)}")

levelapp/workflow/context.py

@@ -1,6 +1,7 @@
 """levelapp/workflow/context.py: Builds runtime WorkflowContext from WorkflowConfig."""
 from typing import Dict, Callable
 
+from levelapp.repository.filesystem import FileSystemRepository
 from levelapp.workflow.config import WorkflowConfig
 from levelapp.core.base import BaseRepository, BaseEvaluator
 from levelapp.workflow.runtime import WorkflowContext
@@ -19,6 +20,7 @@ class WorkflowContextBuilder:
         # Map repository type to constructor that accepts the WorkflowConfig
         self.repository_map: Dict[RepositoryType, Callable[[WorkflowConfig], BaseRepository]] = {
             RepositoryType.FIRESTORE: lambda cfg: FirestoreRepository(cfg),
+            RepositoryType.FILESYSTEM: lambda cfg: FileSystemRepository(cfg),
         }
 
         # Map evaluator type to constructor that accepts the WorkflowConfig
@@ -57,6 +59,6 @@ class WorkflowContextBuilder:
             repository=repository,
             evaluators=evaluators,
             providers=providers,
-            endpoint_config=endpoint_config,
+            endpoint=endpoint_config,
             inputs=inputs,
         )

levelapp/workflow/runtime.py

@@ -2,7 +2,7 @@
 from dataclasses import dataclass
 from typing import Dict, List, Any
 
-from levelapp.config import EndpointConfig
+from levelapp.endpoint.client import EndpointConfig
 from levelapp.core.base import BaseRepository, BaseEvaluator
 from levelapp.workflow.config import WorkflowConfig
 from levelapp.core.schemas import EvaluatorType
@@ -12,8 +12,8 @@ from levelapp.core.schemas import EvaluatorType
 class WorkflowContext:
     """Immutable data holder for workflow execution context."""
     config: WorkflowConfig
+    endpoint: EndpointConfig
     repository: BaseRepository
     evaluators: Dict[EvaluatorType, BaseEvaluator]
     providers: List[str]
-    endpoint_config: EndpointConfig
-    inputs: Dict[str, Any]
+    inputs: Dict[str, Any]