levelapp 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

levelapp/workflow/base.py

@@ -4,13 +4,13 @@ from abc import ABC, abstractmethod
 from pydantic import ValidationError
 from functools import partial
 from pathlib import Path
-from typing import Any
+from typing import Any, Dict
 
 from levelapp.core.base import BaseProcess
 from levelapp.simulator.schemas import ScriptsBatch
 from levelapp.simulator.simulator import ConversationSimulator
+from levelapp.workflow.runtime import WorkflowContext
 from levelapp.aspects.loader import DataLoader
-from levelapp.workflow.schemas import WorkflowContext
 
 
 class BaseWorkflow(ABC):
@@ -28,6 +28,7 @@ class BaseWorkflow(ABC):
         """Validate and initialize workflow-specific settings."""
         if self._initialized:
             return
+
         self.process = self._setup_process(context=self.context)
         self._initialized = True
 
@@ -56,18 +57,35 @@ class BaseWorkflow(ABC):
         else:
             loop = asyncio.get_running_loop()
             func = partial(self.process.run, **self._input_data)
-            self._results = await loop.run_in_executor(None, func)
+            self._results = await loop.run_in_executor(None, func, None)
 
     def collect_results(self) -> Any:
-        """Return unified results structure."""
+        """
+        Return unified results structure.
+
+        Returns:
+            The simulation results.
+        """
         return self._results
 
     @abstractmethod
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        """
+        Abstract method for setting up the configured process.
+
+        Args:
+            context (WorkflowContext): The workflow context.
+        """
         raise NotImplementedError
 
     @abstractmethod
     def _load_input_data(self, context: WorkflowContext) -> Any:
+        """
+        Abstract method for loading reference data.
+
+        Args:
+            context (WorkflowContext): The workflow context.
+        """
         raise NotImplementedError
 
 
@@ -76,38 +94,67 @@ class SimulatorWorkflow(BaseWorkflow):
         super().__init__(name="ConversationSimulator", context=context)
 
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        """
+        Concrete implementation for setting up the simulation workflow.
+
+        Args:
+            context (WorkflowContext): The workflow context for the simulation workflow.
+
+        Returns:
+            ConversationSimulator instance.
+        """
         simulator = ConversationSimulator()
         simulator.setup(
             repository=context.repository,
             evaluators=context.evaluators,
+            providers=context.providers,
             endpoint_config=context.endpoint_config,
         )
         return simulator
 
-    def _load_input_data(self, context: WorkflowContext) -> Any:
+    def _load_input_data(self, context: WorkflowContext) -> Dict[str, Any]:
+        """
+        Concrete implementation for loading the reference data.
+
+        Args:
+            context (WorkflowContext): The workflow context for the simulation workflow.
+
+        Returns:
+            Dict[str, Any]: The reference data.
+        """
         loader = DataLoader()
-        reference_data_path = context.inputs.get("reference_data_path", "no-path-provided")
-        file_path = Path(reference_data_path)
+        if "reference_data" in context.inputs:
+            data_config = context.inputs["reference_data"]
+        else:
+            reference_data_path = context.inputs.get("reference_data_path", "no-path-provided")
+
+            if not reference_data_path:
+                raise RuntimeError(f"[{self.name}] No reference data available.")
+
+            file_path = Path(reference_data_path)
 
-        if not file_path.exists():
-            raise FileNotFoundError(f"[{self.name}] Reference data file not found.")
+            if not file_path.exists():
+                raise FileNotFoundError(f"[{self.name}] Reference data file not found.")
+
+            data_config = loader.load_raw_data(path=reference_data_path)
 
-        evaluation_params = context.inputs.get("evaluation_params", {})
-        data_config = loader.load_raw_data(path=reference_data_path)
         try:
             scripts_batch = ScriptsBatch.model_validate(data_config)
+
         except ValidationError as e:
             raise RuntimeError(f"[{self.name}] Validation error: {e}")
 
-        return {"test_batch": scripts_batch, "attempts": evaluation_params.get("attempts", 1)}
+        attempts = context.config.process.evaluation_params.get("attempts", 1)
+
+        return {"test_batch": scripts_batch, "attempts": attempts}
 
 
 class ComparatorWorkflow(BaseWorkflow):
+    def __init__(self, context: WorkflowContext) -> None:
+        super().__init__(name="MetadataComparator", context=context)
+
     def _setup_process(self, context: WorkflowContext) -> BaseProcess:
-        pass
+        raise NotImplementedError
 
     def _load_input_data(self, context: WorkflowContext) -> Any:
-        pass
-
-    def __init__(self, context: WorkflowContext) -> None:
-        super().__init__(name="MetadataComparator", context=context)
+        raise NotImplementedError

levelapp/workflow/config.py

@@ -0,0 +1,92 @@
+"""levelapp/workflow/config.py: Contains modular workflow configuration components."""
+from typing import List, Dict, Any, Optional
+from pydantic import BaseModel, Field
+
+from levelapp.aspects import logger
+from levelapp.config.endpoint import EndpointConfig
+from levelapp.core.schemas import WorkflowType, RepositoryType, EvaluatorType
+
+
+class ProcessConfig(BaseModel):
+    project_name: str
+    workflow_type: WorkflowType
+    evaluation_params: Dict[str, Any] = Field(default_factory=dict)
+
+
+class EvaluationConfig(BaseModel):
+    evaluators: List[EvaluatorType]
+    providers: List[str] = Field(default_factory=list)
+    metrics_map: Dict[str, str] | None = Field(default_factory=dict)
+
+
+class ReferenceDataConfig(BaseModel):
+    path: str | None
+    data: Dict[str, Any] | None = Field(default_factory=dict)
+
+
+class RepositoryConfig(BaseModel):
+    type: RepositoryType | None = None
+    project_id: str | None = None
+    database_name: str = Field(default="(default)")
+
+    class Config:
+        extra = "allow"
+
+
+class WorkflowConfig(BaseModel):
+    """
+    Static workflow configuration. Maps directly to YAML sections.
+    Supports both file-based loading and in-memory dictionary creation.
+    """
+    process: ProcessConfig
+    evaluation: EvaluationConfig
+    reference_data: ReferenceDataConfig
+    endpoint: EndpointConfig
+    repository: RepositoryConfig
+
+    class Config:
+        extra = "allow"
+
+    @classmethod
+    def load(cls, path: str | None = None) -> "WorkflowConfig":
+        """
+        Load workflow configuration from a YAML/JSON file.
+
+        Args:
+            path (str): YAML/JSON configuration file path.
+
+        Returns:
+            WorkflowConfig: An instance of WorkflowConfig.
+        """
+        from levelapp.aspects.loader import DataLoader
+
+        loader = DataLoader()
+        config_dict = loader.load_raw_data(path=path)
+        logger.info(f"[{cls.__name__}] Workflow configuration loaded from '{path}' file content")
+        return cls.model_validate(config_dict)
+
+    @classmethod
+    def from_dict(cls, content: Dict[str, Any]) -> "WorkflowConfig":
+        """
+        Load workflow configuration from an in-memory dict.
+
+        Args:
+            content (dict): Workflow configuration content.
+
+        Returns:
+            WorkflowConfig: An instance of WorkflowConfig.
+        """
+        logger.info(f"[{cls.__name__}] Workflow configuration loaded from provided content")
+        return cls.model_validate(content)
+
+    def set_reference_data(self, content: Dict[str, Any]) -> None:
+        """
+        Load referer data from an in-memory dict.
+
+        Args:
+            content (dict): Workflow configuration content.
+
+        """
+        self.reference_data.data = content
+        logger.info(f"[{self.__class__.__name__}] Reference data loaded from provided content")
+

levelapp/workflow/context.py

@@ -0,0 +1,62 @@
+"""levelapp/workflow/context.py: Builds runtime WorkflowContext from WorkflowConfig."""
+from typing import Dict, Callable
+
+from levelapp.workflow.config import WorkflowConfig
+from levelapp.core.base import BaseRepository, BaseEvaluator
+from levelapp.workflow.runtime import WorkflowContext
+from levelapp.core.schemas import EvaluatorType, RepositoryType
+
+from levelapp.repository.firestore import FirestoreRepository
+from levelapp.evaluator.evaluator import JudgeEvaluator, MetadataEvaluator
+
+
+class WorkflowContextBuilder:
+    """Builds a runtime WorkflowContext from a WorkflowConfig."""
+
+    def __init__(self, config: WorkflowConfig) -> None:
+        self.config = config
+
+        # Map repository type to constructor that accepts the WorkflowConfig
+        self.repository_map: Dict[RepositoryType, Callable[[WorkflowConfig], BaseRepository]] = {
+            RepositoryType.FIRESTORE: lambda cfg: FirestoreRepository(cfg),
+        }
+
+        # Map evaluator type to constructor that accepts the WorkflowConfig
+        self.evaluator_map: Dict[EvaluatorType, Callable[[WorkflowConfig], BaseEvaluator]] = {
+            EvaluatorType.JUDGE: lambda cfg: JudgeEvaluator(config=cfg),
+            EvaluatorType.REFERENCE: lambda cfg: MetadataEvaluator(config=cfg),
+        }
+
+    def build(self) -> WorkflowContext:
+        """
+        Build a runtime WorkflowContext from the static WorkflowConfig.
+        Supports in-memory reference data if provided.
+        """
+        # Repository instance
+        repository_type = self.config.repository.type
+        repository = self.repository_map.get(repository_type)(self.config)
+
+        # Evaluator instances
+        evaluators: Dict[EvaluatorType, BaseEvaluator] = {
+            ev: self.evaluator_map.get(ev)(self.config) for ev in self.config.evaluation.evaluators
+        }
+
+        # Providers and endpoint
+        providers = self.config.evaluation.providers
+        endpoint_config = self.config.endpoint
+
+        # Inputs include reference data path or in-memory dict
+        inputs = {}
+        if self.config.reference_data.data:
+            inputs["reference_data"] = self.config.reference_data.data
+        else:
+            inputs["reference_data_path"] = self.config.reference_data.path
+
+        return WorkflowContext(
+            config=self.config,
+            repository=repository,
+            evaluators=evaluators,
+            providers=providers,
+            endpoint_config=endpoint_config,
+            inputs=inputs,
+        )

levelapp/workflow/factory.py

@@ -1,51 +1,42 @@
-from typing import Callable, Dict
-from levelapp.workflow.schemas import WorkflowType, RepositoryType, EvaluatorType, WorkflowConfig, WorkflowContext
-from levelapp.core.base import BaseRepository, BaseEvaluator
-from levelapp.workflow.base import BaseWorkflow
+"""levelapp/workflow/factory.py: Creates workflows using WorkflowContext."""
+from typing import Dict, Callable
 
-from levelapp.repository.firestore import FirestoreRepository
-from levelapp.evaluator.evaluator import JudgeEvaluator, MetadataEvaluator
+from levelapp.core.schemas import WorkflowType
+from levelapp.workflow.base import SimulatorWorkflow, ComparatorWorkflow, BaseWorkflow
+from levelapp.workflow.runtime import WorkflowContext
 
 
 class MainFactory:
-    """Central factory for repositories, evaluators, and workflows."""
-
-    _repository_map: dict[RepositoryType, Callable[[WorkflowConfig], BaseRepository]] = {
-        RepositoryType.FIRESTORE: lambda cfg: FirestoreRepository(),
-    }
-
-    _evaluator_map: dict[EvaluatorType, Callable[[WorkflowConfig], BaseEvaluator]] = {
-        EvaluatorType.JUDGE: lambda cfg: JudgeEvaluator(),
-        EvaluatorType.REFERENCE: lambda cfg: MetadataEvaluator(),
-        # Next is the RAG evaluator..
+    """Central factory for workflows."""
+    _workflow_map: Dict[WorkflowType, Callable[[WorkflowContext], BaseWorkflow]] = {
+        WorkflowType.SIMULATOR: lambda ctx: SimulatorWorkflow(ctx),
+        WorkflowType.COMPARATOR: lambda ctx: ComparatorWorkflow(ctx),
     }
 
-    _workflow_map: dict[WorkflowType, Callable[["WorkflowContext"], BaseWorkflow]] = {}
-
-    @classmethod
-    def create_repository(cls, config: WorkflowConfig) -> BaseRepository:
-        fn = cls._repository_map.get(config.repository)
-        if not fn:
-            raise NotImplementedError(f"Repository {config.repository} not implemented")
-        return fn(config)
-
-    @classmethod
-    def create_evaluator(cls, config: WorkflowConfig) -> Dict[EvaluatorType, BaseEvaluator]:
-        evaluators: dict[EvaluatorType, BaseEvaluator] = {}
-        for ev in config.evaluators:
-            fn = cls._evaluator_map.get(ev)
-            if not fn:
-                raise NotImplementedError(f"Evaluator {config.evaluators} not implemented")
-            evaluators[ev] = fn(config)
-        return evaluators
-
     @classmethod
-    def create_workflow(cls, wf_type: WorkflowType, context: "WorkflowContext") -> BaseWorkflow:
-        fn = cls._workflow_map.get(wf_type)
-        if not fn:
-            raise NotImplementedError(f"Workflow {wf_type} not implemented")
-        return fn(context)
+    def create_workflow(cls, context: WorkflowContext) -> BaseWorkflow:
+        """
+        Create workflow using the given runtime context.
+
+        Args:
+            context (WorkflowContext): the provided workflow context.
+
+        Returns:
+            BaseWorkflow: the built workflow instance from the provided context.
+        """
+        wf_type = context.config.process.workflow_type
+        builder = cls._workflow_map.get(wf_type)
+        if not builder:
+            raise NotImplementedError(f"Workflow '{wf_type}' not implemented")
+        return builder(context)
 
     @classmethod
-    def register_workflow(cls, wf_type: WorkflowType, builder: Callable[["WorkflowContext"], BaseWorkflow]) -> None:
+    def register_workflow(cls, wf_type: WorkflowType, builder: Callable[[WorkflowContext], BaseWorkflow]) -> None:
+        """
+        Register a new workflow implementation.
+
+        Args:
+            wf_type (WorkflowType): the workflow type.
+            builder (Callable[[WorkflowContext], BaseWorkflow]): the workflow builder.
+        """
         cls._workflow_map[wf_type] = builder

levelapp/workflow/registration.py

@@ -1,5 +1,5 @@
+from levelapp.core.schemas import WorkflowType
 from levelapp.workflow.factory import MainFactory
-from levelapp.workflow.schemas import WorkflowType
 from levelapp.workflow.base import SimulatorWorkflow, ComparatorWorkflow
 
 MainFactory.register_workflow(WorkflowType.SIMULATOR, lambda ctx: SimulatorWorkflow(ctx))

levelapp/workflow/runtime.py

@@ -0,0 +1,19 @@
+"""levelapp/workflow/runtime.py: contains the workflow runtime context component."""
+from dataclasses import dataclass
+from typing import Dict, List, Any
+
+from levelapp.config import EndpointConfig
+from levelapp.core.base import BaseRepository, BaseEvaluator
+from levelapp.workflow.config import WorkflowConfig
+from levelapp.core.schemas import EvaluatorType
+
+
+@dataclass(frozen=True)
+class WorkflowContext:
+    """Immutable data holder for workflow execution context."""
+    config: WorkflowConfig
+    repository: BaseRepository
+    evaluators: Dict[EvaluatorType, BaseEvaluator]
+    providers: List[str]
+    endpoint_config: EndpointConfig
+    inputs: Dict[str, Any]

{levelapp-0.1.0.dist-info → levelapp-0.1.2.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.4
 Name: levelapp
-Version: 0.1.0
+Version: 0.1.2
 Summary: LevelApp is an evaluation framework for AI/LLM-based software application. [Powered by Norma]
 Project-URL: Homepage, https://github.com/levelapp-org
 Project-URL: Repository, https://github.com/levelapp-org/levelapp-framework
 Project-URL: Documentation, https://levelapp.readthedocs.io
 Project-URL: Issues, https://github.com/levelapp-org/levelapp-framework/issues
-Author-email: KadriSof <kadrisofyen@gmail.com>
+Author-email: Mohamed Sofiene KADRI <ms.kadri.dev@gmail.com>
 License-File: LICENSE
 Keywords: ai,evaluation,framework,llm,testing
 Classifier: Development Status :: 3 - Alpha
@@ -17,10 +17,12 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development :: Testing
 Requires-Python: >=3.12
-Requires-Dist: arrow>=1.3.0
+Requires-Dist: google-api-core>=2.25.1
+Requires-Dist: google-auth>=2.40.3
+Requires-Dist: google-cloud-firestore>=2.21.0
 Requires-Dist: httpx>=0.28.1
+Requires-Dist: humanize>=4.13.0
 Requires-Dist: numpy>=2.3.2
-Requires-Dist: openai>=1.99.9
 Requires-Dist: pandas-stubs==2.3.0.250703
 Requires-Dist: pandas>=2.3.1
 Requires-Dist: pydantic>=2.11.7
@@ -91,38 +93,47 @@ pip install levelapp
 LevelApp uses a YAML configuration file to define the evaluation setup. Create a `workflow_config.yaml` with the following structure:
 
 ```yaml
-project_name: "test-project"
-evaluation_params:
-  attempts: 1  # Number of simulation attempts.
+process:
+  project_name: "test-project"
+  workflow_type: SIMULATOR # Pick one of the following workflows: SIMULATOR, COMPARATOR, ASSESSOR.
+  evaluation_params:
+    attempts: 1  # Add the number of simulation attempts.
+    batch_size: 5
+
+evaluation:
+  evaluators: # Select from the following: JUDGE, REFERENCE, RAG.
+    - JUDGE
+    - REFERENCE
+  providers:
+    - openai
+    - ionos
+  metrics_map:
+    field_1: EXACT
+    field_2 : LEVENSHTEIN
 
-workflow: SIMULATOR  # SIMULATOR, COMPARATOR, ASSESSOR.
-repository: FIRESTORE  # FIRESTORE, FILESYSTEM, MONGODB.
-evaluators: # JUDGE, REFERENCE, RAG.
-  - JUDGE
-  - REFERENCE
+reference_data:
+  path: 
+  data:
 
-endpoint_configuration:
+endpoint:
   base_url: "http://127.0.0.1:8000"
   url_path: ''
   api_key: "<API-KEY>"
   bearer_token: "<BEARER-TOKEN>"
   model_id: "meta-llama/Meta-Llama-3.1-8B-Instruct"
-  payload_path: "../../src/data/payload_example_1.yaml"
   default_request_payload_template:
+    # Change the user message field name only according to the request payload schema (example: 'prompt' to 'message').
     prompt: "${user_message}"
     details: "${request_payload}"  # Rest of the request payload data.
   default_response_payload_template:
+    # Change the placeholder value only according to the response payload schema (example: ${agent_reply} to ${reply}).
     agent_reply: "${agent_reply}"
-    guardrail_flag: "${guardrail_flag}"
     generated_metadata: "${generated_metadata}"
 
-reference_data:
-  source: LOCAL  # LOCAL or REMOTE.
-  path: "../../src/data/conversation_example_1.json"
-
-metrics_map:
-  field_1: EXACT
-  field_2: LEVENSHTEIN
+repository:
+  type: FIRESTORE # Pick one of the following: FIRESTORE, FILESYSTEM
+  project_id: "(default)"
+  database_name: ""
 ```
 
 - **Endpoint Configuration**: Define how to interact with your LLM-based system (base URL, auth, payload templates).
@@ -133,33 +144,26 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
 
 ```json
 {
-  "id": "1fa6f6ed-3cfe-4c0b-b389-7292f58879d4",
   "scripts": [
     {
-      "id": "65f58cec-d55d-4a24-bf16-fa8327a3aa6b",
       "interactions": [
         {
-          "id": "e99a2898-6a79-4a20-ac85-dfe977ea9935",
           "user_message": "Hello, I would like to book an appointment with a doctor.",
           "reference_reply": "Sure, I can help with that. Could you please specify the type of doctor you need to see?",
           "interaction_type": "initial",
           "reference_metadata": {},
-          "generated_metadata": {},
           "guardrail_flag": false,
           "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
         },
         {
-          "id": "fe5c539a-d0a1-40ee-97bd-dbe456703ccc",
           "user_message": "I need to see a cardiologist.",
           "reference_reply": "When would you like to schedule your appointment?",
           "interaction_type": "intermediate",
           "reference_metadata": {},
-          "generated_metadata": {},
           "guardrail_flag": false,
           "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
         },
         {
-          "id": "2cfdbd1c-a065-48bb-9aa9-b958342154b1",
           "user_message": "I would like to book it for next Monday morning.",
           "reference_reply": "We have an available slot at 10 AM next Monday. Does that work for you?",
           "interaction_type": "intermediate",
@@ -168,11 +172,6 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
             "date": "next Monday",
             "time": "10 AM"
           },
-          "generated_metadata": {
-            "appointment_type": "Cardiology",
-            "date": "next Monday",
-            "time": "morning"
-          },
           "guardrail_flag": false,
           "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
         },
@@ -182,7 +181,6 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
           "reference_reply": "Your appointment with the cardiologist is booked for 10 AM next Monday. Is there anything else I can help you with?",
           "interaction_type": "final",
           "reference_metadata": {},
-          "generated_metadata": {},
           "guardrail_flag": false,
           "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
         }
@@ -195,9 +193,22 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
   ]
 }
 ```
-
 - **Fields**: Include user messages, reference/references replies, metadata for comparison, guardrail flags, and request payloads.
 
+In the `.env` you need to add the LLM providers credentials that will be used for the evaluation process. 
+```
+OPENAI_API_KEY=
+IONOS_API_KEY=
+ANTHROPIC_API_KEY=
+MISTRAL_API_KEY=
+
+# For IONOS, you must include the base URL and the model ID.
+IONOS_BASE_URL="https://inference.de-txl.ionos.com"
+IONOS_MODEL_ID="0b6c4a15-bb8d-4092-82b0-f357b77c59fd"
+
+WORKFLOW_CONFIG_PATH="../../src/data/workflow_config_1.yaml"
+```
+
 ## Usage Example
 
 To run an evaluation:
@@ -207,14 +218,14 @@ To run an evaluation:
 
 ```python
 if __name__ == "__main__":
-    from levelapp.workflow.schemas import WorkflowConfig
+    from levelapp.workflow import WorkflowConfig
     from levelapp.core.session import EvaluationSession
 
     # Load configuration from YAML
     config = WorkflowConfig.load(path="../data/workflow_config.yaml")
 
-    # Run evaluation session
-    with EvaluationSession(session_name="sim-test", workflow_config=config) as session:
+    # Run evaluation session (You can enable/disable the monitoring aspect)
+    with EvaluationSession(session_name="test-session-1", workflow_config=config, enable_monitoring=False) as session:
         session.run()
         results = session.workflow.collect_results()
         print("Results:", results)
@@ -223,6 +234,58 @@ if __name__ == "__main__":
     print(f"session stats:\n{stats}")
 ```
 
+Alternatively, if you want to pass the configuration and reference data from in-memory variables, 
+you can manually load the data like the following:
+```python
+if __name__ == "__main__":
+    from levelapp.workflow import WorkflowConfig
+    from levelapp.core.session import EvaluationSession
+
+    
+    config_dict = {
+        "process": {"project_name": "test-project", "workflow_type": "SIMULATOR", "evaluation_params": {"attempts": 2}},
+        "evaluation": {"evaluators": ["JUDGE", "REFERENCE"], "providers": ["openai", "ionos"], "metrics_map": {"field_1": "EXACT"}},
+        "reference_data": {"path": "", "data": {}},
+        "endpoint": {"base_url": "http://127.0.0.1:8000", "api_key": "key", "model_id": "model"},
+        "repository": {"type": "FIRESTORE", "source": "IN_MEMORY"},
+    }
+
+    content = {
+        "scripts": [
+            {
+                "interactions": [
+                    {
+                        "user_message": "Hello!",
+                        "reference_reply": "Hello, how can I help you!"
+                    },
+                    {
+                        "user_message": "I need an apartment",
+                        "reference_reply": "sorry, but I can only assist you with booking medical appointments."
+                    },
+                ]
+            },
+        ]
+    }
+
+    # Load configuration from a dict variable
+    config = WorkflowConfig.from_dict(content=config_dict)
+
+    # Load reference data from dict variable
+    config.set_reference_data(content=content)
+
+    evaluation_session = EvaluationSession(session_name="test-session-2", workflow_config=config)
+
+    with evaluation_session as session:
+        session.run()
+        results = session.workflow.collect_results()
+        print("Results:", results)
+
+    stats = session.get_stats()
+    print(f"session stats:\n{stats}")
+
+```
+
+
 - This loads the config, runs the specified workflow (e.g., Simulator), collects results, and prints stats.
 
 For more examples, see the `examples/` directory.