levelapp 0.1.3__tar.gz → 0.1.4__tar.gz

{levelapp-0.1.3 → levelapp-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: levelapp
-Version: 0.1.3
+Version: 0.1.4
 Summary: LevelApp is an evaluation framework for AI/LLM-based software application. [Powered by Norma]
 Project-URL: Homepage, https://github.com/levelapp-org
 Project-URL: Repository, https://github.com/levelapp-org/levelapp-framework

levelapp-0.1.4/levelapp/simulator/utils.py

@@ -0,0 +1,257 @@
+"""
+'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
+"""
+import re
+import ast
+import json
+import httpx
+
+from uuid import UUID
+from string import Template
+from typing import Any, Dict, List, Union, Iterable
+
+from pydantic import ValidationError
+
+from levelapp.clients import ClientRegistry
+from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
+from levelapp.simulator.schemas import InteractionResults
+from levelapp.aspects import MonitoringAspect, MetricType, logger
+
+
+class UUIDEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, UUID):
+            return str(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+_PLACEHOLDER_RE = re.compile(r"\$\{([^}]+)\}")  # captures inner name(s) of ${...}
+
+
+def _traverse_path(d: Dict[str, Any], path: str):
+    """Traverse a dot-separated path (payload.metadata.budget) and return value or None."""
+    parts = path.split(".")
+    cur = d
+    try:
+        for p in parts:
+            if isinstance(cur, dict) and p in cur:
+                cur = cur[p]
+            else:
+                return None
+        return cur
+    except Exception:
+        return None
+
+
+def _recursive_find(container: Any, target_key: str):
+    """
+    Recursively search container (dicts/lists) for the first occurrence of target_key.
+    Returns the value if found, else None.
+    """
+    if isinstance(container, dict):
+        # direct hit
+        if target_key in container:
+            return container[target_key]
+        # recurse into values
+        for v in container.values():
+            found = _recursive_find(v, target_key)
+            if found is not None:
+                return found
+        return None
+
+    if isinstance(container, list):
+        for item in container:
+            found = _recursive_find(item, target_key)
+            if found is not None:
+                return found
+        return None
+
+    # not a container
+    return None
+
+
+def _extract_placeholders(template_str: str) -> Iterable[str]:
+    """Return list of placeholder names in a template string (inner contents of ${...})."""
+    return [m.group(1) for m in _PLACEHOLDER_RE.finditer(template_str)]
+
+
+def extract_interaction_details(
+    response: str | Dict[str, Any],
+    template: Dict[str, Any],
+) -> InteractionResults:
+    """
+    Parse response (str or dict), look up placeholders recursively in the response and
+    use Template.safe_substitute with a mapping built from those lookups.
+    """
+    try:
+        response_dict = response if isinstance(response, dict) else json.loads(response)
+        print(f"response:\n{response_dict}\n--")
+        if not isinstance(response_dict, dict):
+            raise ValueError("Response is not a valid dictionary")
+
+        output: Dict[str, Any] = {}
+
+        for out_key, tpl_str in template.items():
+            # Build mapping for placeholders found in tpl_str
+            placeholders = _extract_placeholders(tpl_str)
+            mapping: Dict[str, str] = {}
+
+            for ph in placeholders:
+                value = None
+
+                # 1) If ph looks like a dotted path, try explicit path traversal first
+                if "." in ph:
+                    value = _traverse_path(response_dict, ph)
+
+                # 2) If not found yet, try recursive search for the bare key (last path segment)
+                if value is None:
+                    bare = ph.split(".")[-1]
+                    value = _recursive_find(response_dict, bare)
+
+                # Prepare mapping value for Template substitution:
+                # - dict/list -> JSON string (so substitution yields valid JSON text)
+                # - None -> empty string
+                # - otherwise -> str(value)
+                if isinstance(value, (dict, list)):
+                    try:
+                        mapping[ph] = json.dumps(value, ensure_ascii=False)
+                    except Exception:
+                        mapping[ph] = str(value)
+                elif value is None:
+                    mapping[ph] = ""
+                else:
+                    mapping[ph] = str(value)
+
+            # Perform substitution using Template (safe_substitute: missing keys left intact)
+            substituted = Template(tpl_str).safe_substitute(mapping)
+            output[out_key] = substituted
+
+        # Post-process generated_metadata if present: convert JSON text back to dict/list when possible
+        raw_meta = output.get("generated_metadata", {})
+        if isinstance(raw_meta, str) and raw_meta:
+            # Try json first (since we used json.dumps above for mapping)
+            try:
+                output["generated_metadata"] = json.loads(raw_meta)
+            except Exception:
+                # fallback to ast.literal_eval (handles Python dict strings)
+                try:
+                    output["generated_metadata"] = ast.literal_eval(raw_meta)
+                except Exception:
+                    # if parsing fails, keep the original raw string or use an empty dict
+                    output["generated_metadata"] = raw_meta
+
+        # If generated_metadata is empty string, normalize to {}
+        if output.get("generated_metadata") == "":
+            output["generated_metadata"] = {}
+
+        print(f"output:\n{output}\n---")
+        # Return validated model
+        return InteractionResults.model_validate(output)
+
+    except json.JSONDecodeError as e:
+        logger.error(f"[extract_interaction_details] Failed to parse JSON response: {e}")
+        return InteractionResults()
+
+    except ValidationError as e:
+        logger.exception(f"[extract_interaction_details] InteractionResults validation failed: {e}")
+        return InteractionResults()
+
+    except Exception as e:
+        logger.exception(f"[extract_interaction_details] Unexpected error: {e}")
+        return InteractionResults()
+
+
+@MonitoringAspect.monitor(name="interaction_request", category=MetricType.API_CALL)
+async def async_interaction_request(
+        url: str,
+        headers: Dict[str, str],
+        payload: Dict[str, Any],
+) -> httpx.Response | None:
+    """
+    Perform an asynchronous interaction request.
+
+    Args:
+        url (str): The URL to send the request to.
+        headers (Dict[str, str]): The headers to include in the request.
+        payload (Dict[str, Any]): The payload to send in the request.
+
+    Returns:
+        httpx.Response: The response from the interaction request, or None if an error occurred.
+    """
+    try:
+        async with httpx.AsyncClient(timeout=180) as client:
+            response = await client.post(url=url, headers=headers, json=payload)
+            response.raise_for_status()
+
+            return response
+
+    except httpx.HTTPStatusError as http_err:
+        logger.error(f"[async_interaction_request] HTTP error: {http_err.response.text}", exc_info=True)
+
+    except httpx.RequestError as req_err:
+        logger.error(f"[async_interaction_request] Request error: {str(req_err)}", exc_info=True)
+
+    return None
+
+
+@MonitoringAspect.monitor(
+    name="average_calc",
+    category=MetricType.SCORING,
+    cached=True,
+    maxsize=1000
+)
+def calculate_average_scores(scores: Dict[str, Union[List[float], float]]) -> Dict[str, float]:
+    """
+    Helper function that calculates the average scores for a dictionary of score lists.
+
+    Args:
+        scores (Dict[str, List[float]]): A dictionary where keys are identifiers and values are lists of scores.
+
+    Returns:
+        Dict[str, float]: A dictionary with average scores rounded to three decimal places.
+    """
+    result: Dict[str, float] = {}
+    for field, value in scores.items():
+        if isinstance(value, (int, float)):
+            result[field] = value
+        elif isinstance(value, list):
+            result[field] = round((sum(value) / len(value)), 3) if value else 0.0
+        else:
+            raise TypeError(f"[calculate_average_scores] Unexpected type '{type(value)}' for field '{field}")
+
+    return result
+
+
+@MonitoringAspect.monitor(name="summarization", category=MetricType.API_CALL)
+def summarize_verdicts(
+        verdicts: List[str],
+        judge: str,
+        max_bullets: int = 5
+) -> List[str]:
+    client_registry = ClientRegistry()
+    client = client_registry.get(provider=judge)
+
+    try:
+        verdicts = chr(10).join(verdicts)
+        prompt = SUMMARIZATION_PROMPT_TEMPLATE.format(max_bullets=max_bullets, judge=judge, verdicts=verdicts)
+        response = client.call(message=prompt)
+        parsed = client.parse_response(response=response)
+        striped = parsed.get("output", "").strip("")
+        bullet_points = [point.strip() for point in striped.split("- ") if point.strip()]
+
+        return bullet_points[:max_bullets]
+
+    except Exception as e:
+        logger.error(f"[summarize_justifications] Error during summarization: {str(e)}", exc_info=True)
+        return []
+
+
+# if __name__ == '__main__':
+#     template = {'generated_reply': '${agent_reply}', 'generated_metadata': '${generated_metadata}'}
+#     response_dict = {
+#         'agent_reply': "I'd be happy to help you book something for 10 AM.",
+#         'generated_metadata': {'appointment_type': 'Cardiology', 'date': 'next Monday', 'time': '10 AM'}
+#     }
+#
+#     result = extract_interaction_details(response_dict, template)
+#     print(f"result: {result.model_dump()}")

{levelapp-0.1.3 → levelapp-0.1.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "levelapp"
-version = "0.1.3"
+version = "0.1.4"
 description = "LevelApp is an evaluation framework for AI/LLM-based software application. [Powered by Norma]"
 readme = "README.md"
 authors = [

{levelapp-0.1.3 → levelapp-0.1.4}/src/data/workflow_config.yaml

@@ -32,8 +32,8 @@ endpoint:
     details: "${request_payload}"  # Rest of the request payload data.
   default_response_payload_template:
     # Change the placeholder value only according to the response payload schema (example: ${agent_reply} to ${reply}).
-    generated_reply: "${agent_reply}"
-    generated_metadata: "${generated_metadata}"
+    generated_reply: "${message}"
+    generated_metadata: "${metadata}"
 
 repository:
   type: FIRESTORE # Pick one of the following: FIRESTORE, FILESYSTEM

levelapp-0.1.4/src/level_app/main_session.py

@@ -0,0 +1,73 @@
+if __name__ == "__main__":
+    from levelapp.workflow import WorkflowConfig
+    from levelapp.core.session import EvaluationSession
+
+    # Firestore -> retrieve endpoint config -> data => config_dict
+
+    config_dict_ = {
+        "process": {"project_name": "test-project", "workflow_type": "SIMULATOR", "evaluation_params": {"attempts": 2}},
+        "evaluation": {"evaluators": ["JUDGE"], "providers": ["openai", "ionos"]},
+        "reference_data": {"path": "", "data": {}},
+        "endpoint": {
+            "base_url": "https://dashq-gateway-485vb8zi.uc.gateway.dev/api/conversations/events",
+            "api_key": "AIzaSyAmL8blcS2hpPrEH2b84B8ugsVoV7AXrfc",
+            "model_id": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+            "default_request_payload_template": {
+                "eventType": "newConversation",
+                "conversationId": "435484ef-403b-43c5-9908-884486149d0b",
+                "payload": {
+                    "messageType": "newInquiry",
+                    "communityId": 3310,
+                    "accountId": 1440,
+                    "prospectFirstName": "BAD DOE X",
+                    "prospectLastName": "Doe",
+                    "message": "${user_message}",
+                    "datetime": "2025-06-25T11:12:27.245Z",
+                    "inboundChannel": "text",
+                    "outboundChannel": "text",
+                    "inquirySource": "test.com",
+                    "inquiryMetadata": {}
+                },
+            },
+            "default_response_payload_template": {
+                "generated_reply": "${message}",
+                "generated_metadata": "${metadata}"
+            }
+        },
+        "repository": {"type": "FIRESTORE", "source": "IN_MEMORY", "metrics_map": {"field_1": "EXACT"}},
+    }
+
+    content = {
+        "scripts": [
+            {
+                "interactions": [
+                    {
+                        "user_message": "Hi I would like to rent an apartment",
+                        "reference_reply": "thank you for reaching out. I’d be happy to help you find an apartment. Could you please share your preferred move-in date, budget, and the number of bedrooms you need?"
+                    },
+                    {
+                        "user_message": "I am moving in next month, and I would like to rent a two bedroom apartment",
+                        "reference_reply": "sorry, but I can only assist you with booking medical appointments."
+                    },
+                ]
+            },
+        ]
+    }
+
+    # Load configuration from YAML
+    config = WorkflowConfig.from_dict(content=config_dict_)
+
+    # Load reference data from in-memory dict
+    config.set_reference_data(content=content)
+
+    # config = WorkflowConfig.load(path="../data/workflow_config.yaml")
+
+    evaluation_session = EvaluationSession(session_name="test-session", workflow_config=config, enable_monitoring=True)
+
+    with evaluation_session as session:
+        session.run()
+        results = session.workflow.collect_results()
+        print("Results:", results)
+
+    stats = session.get_stats()
+    print(f"session stats:\n{stats}")

{levelapp-0.1.3 → levelapp-0.1.4}/uv.lock

@@ -29,19 +29,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916, upload-time = "2025-03-17T00:02:52.713Z" },
 ]
 
-[[package]]
-name = "arrow"
-version = "1.3.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "python-dateutil" },
-    { name = "types-python-dateutil" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/2e/00/0f6e8fcdb23ea632c866620cc872729ff43ed91d284c866b515c6342b173/arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85", size = 131960, upload-time = "2023-09-30T22:11:18.25Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/f8/ed/e97229a566617f2ae958a6b13e7cc0f585470eac730a73e9e82c32a3cdd2/arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80", size = 66419, upload-time = "2023-09-30T22:11:16.072Z" },
-]
-
 [[package]]
 name = "cachetools"
 version = "5.5.2"
@@ -95,24 +82,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" },
 ]
 
-[[package]]
-name = "colorama"
-version = "0.4.6"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
-]
-
-[[package]]
-name = "distro"
-version = "1.9.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" },
-]
-
 [[package]]
 name = "google-api-core"
 version = "2.25.1"
@@ -290,57 +259,9 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" },
 ]
 
-[[package]]
-name = "jiter"
-version = "0.10.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/ee/9d/ae7ddb4b8ab3fb1b51faf4deb36cb48a4fbbd7cb36bad6a5fca4741306f7/jiter-0.10.0.tar.gz", hash = "sha256:07a7142c38aacc85194391108dc91b5b57093c978a9932bd86a36862759d9500", size = 162759, upload-time = "2025-05-18T19:04:59.73Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/6d/b5/348b3313c58f5fbfb2194eb4d07e46a35748ba6e5b3b3046143f3040bafa/jiter-0.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1e274728e4a5345a6dde2d343c8da018b9d4bd4350f5a472fa91f66fda44911b", size = 312262, upload-time = "2025-05-18T19:03:44.637Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/4a/6a2397096162b21645162825f058d1709a02965606e537e3304b02742e9b/jiter-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7202ae396446c988cb2a5feb33a543ab2165b786ac97f53b59aafb803fef0744", size = 320124, upload-time = "2025-05-18T19:03:46.341Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/85/1ce02cade7516b726dd88f59a4ee46914bf79d1676d1228ef2002ed2f1c9/jiter-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23ba7722d6748b6920ed02a8f1726fb4b33e0fd2f3f621816a8b486c66410ab2", size = 345330, upload-time = "2025-05-18T19:03:47.596Z" },
-    { url = "https://files.pythonhosted.org/packages/75/d0/bb6b4f209a77190ce10ea8d7e50bf3725fc16d3372d0a9f11985a2b23eff/jiter-0.10.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:371eab43c0a288537d30e1f0b193bc4eca90439fc08a022dd83e5e07500ed026", size = 369670, upload-time = "2025-05-18T19:03:49.334Z" },
-    { url = "https://files.pythonhosted.org/packages/a0/f5/a61787da9b8847a601e6827fbc42ecb12be2c925ced3252c8ffcb56afcaf/jiter-0.10.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c675736059020365cebc845a820214765162728b51ab1e03a1b7b3abb70f74c", size = 489057, upload-time = "2025-05-18T19:03:50.66Z" },
-    { url = "https://files.pythonhosted.org/packages/12/e4/6f906272810a7b21406c760a53aadbe52e99ee070fc5c0cb191e316de30b/jiter-0.10.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c5867d40ab716e4684858e4887489685968a47e3ba222e44cde6e4a2154f959", size = 389372, upload-time = "2025-05-18T19:03:51.98Z" },
-    { url = "https://files.pythonhosted.org/packages/e2/ba/77013b0b8ba904bf3762f11e0129b8928bff7f978a81838dfcc958ad5728/jiter-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395bb9a26111b60141757d874d27fdea01b17e8fac958b91c20128ba8f4acc8a", size = 352038, upload-time = "2025-05-18T19:03:53.703Z" },
-    { url = "https://files.pythonhosted.org/packages/67/27/c62568e3ccb03368dbcc44a1ef3a423cb86778a4389e995125d3d1aaa0a4/jiter-0.10.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6842184aed5cdb07e0c7e20e5bdcfafe33515ee1741a6835353bb45fe5d1bd95", size = 391538, upload-time = "2025-05-18T19:03:55.046Z" },
-    { url = "https://files.pythonhosted.org/packages/c0/72/0d6b7e31fc17a8fdce76164884edef0698ba556b8eb0af9546ae1a06b91d/jiter-0.10.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:62755d1bcea9876770d4df713d82606c8c1a3dca88ff39046b85a048566d56ea", size = 523557, upload-time = "2025-05-18T19:03:56.386Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/09/bc1661fbbcbeb6244bd2904ff3a06f340aa77a2b94e5a7373fd165960ea3/jiter-0.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:533efbce2cacec78d5ba73a41756beff8431dfa1694b6346ce7af3a12c42202b", size = 514202, upload-time = "2025-05-18T19:03:57.675Z" },
-    { url = "https://files.pythonhosted.org/packages/1b/84/5a5d5400e9d4d54b8004c9673bbe4403928a00d28529ff35b19e9d176b19/jiter-0.10.0-cp312-cp312-win32.whl", hash = "sha256:8be921f0cadd245e981b964dfbcd6fd4bc4e254cdc069490416dd7a2632ecc01", size = 211781, upload-time = "2025-05-18T19:03:59.025Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/52/7ec47455e26f2d6e5f2ea4951a0652c06e5b995c291f723973ae9e724a65/jiter-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:a7c7d785ae9dda68c2678532a5a1581347e9c15362ae9f6e68f3fdbfb64f2e49", size = 206176, upload-time = "2025-05-18T19:04:00.305Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/b0/279597e7a270e8d22623fea6c5d4eeac328e7d95c236ed51a2b884c54f70/jiter-0.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:e0588107ec8e11b6f5ef0e0d656fb2803ac6cf94a96b2b9fc675c0e3ab5e8644", size = 311617, upload-time = "2025-05-18T19:04:02.078Z" },
-    { url = "https://files.pythonhosted.org/packages/91/e3/0916334936f356d605f54cc164af4060e3e7094364add445a3bc79335d46/jiter-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cafc4628b616dc32530c20ee53d71589816cf385dd9449633e910d596b1f5c8a", size = 318947, upload-time = "2025-05-18T19:04:03.347Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/8e/fd94e8c02d0e94539b7d669a7ebbd2776e51f329bb2c84d4385e8063a2ad/jiter-0.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:520ef6d981172693786a49ff5b09eda72a42e539f14788124a07530f785c3ad6", size = 344618, upload-time = "2025-05-18T19:04:04.709Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/b0/f9f0a2ec42c6e9c2e61c327824687f1e2415b767e1089c1d9135f43816bd/jiter-0.10.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:554dedfd05937f8fc45d17ebdf298fe7e0c77458232bcb73d9fbbf4c6455f5b3", size = 368829, upload-time = "2025-05-18T19:04:06.912Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/57/5bbcd5331910595ad53b9fd0c610392ac68692176f05ae48d6ce5c852967/jiter-0.10.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bc299da7789deacf95f64052d97f75c16d4fc8c4c214a22bf8d859a4288a1c2", size = 491034, upload-time = "2025-05-18T19:04:08.222Z" },
-    { url = "https://files.pythonhosted.org/packages/9b/be/c393df00e6e6e9e623a73551774449f2f23b6ec6a502a3297aeeece2c65a/jiter-0.10.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5161e201172de298a8a1baad95eb85db4fb90e902353b1f6a41d64ea64644e25", size = 388529, upload-time = "2025-05-18T19:04:09.566Z" },
-    { url = "https://files.pythonhosted.org/packages/42/3e/df2235c54d365434c7f150b986a6e35f41ebdc2f95acea3036d99613025d/jiter-0.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e2227db6ba93cb3e2bf67c87e594adde0609f146344e8207e8730364db27041", size = 350671, upload-time = "2025-05-18T19:04:10.98Z" },
-    { url = "https://files.pythonhosted.org/packages/c6/77/71b0b24cbcc28f55ab4dbfe029f9a5b73aeadaba677843fc6dc9ed2b1d0a/jiter-0.10.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:15acb267ea5e2c64515574b06a8bf393fbfee6a50eb1673614aa45f4613c0cca", size = 390864, upload-time = "2025-05-18T19:04:12.722Z" },
-    { url = "https://files.pythonhosted.org/packages/6a/d3/ef774b6969b9b6178e1d1e7a89a3bd37d241f3d3ec5f8deb37bbd203714a/jiter-0.10.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:901b92f2e2947dc6dfcb52fd624453862e16665ea909a08398dde19c0731b7f4", size = 522989, upload-time = "2025-05-18T19:04:14.261Z" },
-    { url = "https://files.pythonhosted.org/packages/0c/41/9becdb1d8dd5d854142f45a9d71949ed7e87a8e312b0bede2de849388cb9/jiter-0.10.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d0cb9a125d5a3ec971a094a845eadde2db0de85b33c9f13eb94a0c63d463879e", size = 513495, upload-time = "2025-05-18T19:04:15.603Z" },
-    { url = "https://files.pythonhosted.org/packages/9c/36/3468e5a18238bdedae7c4d19461265b5e9b8e288d3f86cd89d00cbb48686/jiter-0.10.0-cp313-cp313-win32.whl", hash = "sha256:48a403277ad1ee208fb930bdf91745e4d2d6e47253eedc96e2559d1e6527006d", size = 211289, upload-time = "2025-05-18T19:04:17.541Z" },
-    { url = "https://files.pythonhosted.org/packages/7e/07/1c96b623128bcb913706e294adb5f768fb7baf8db5e1338ce7b4ee8c78ef/jiter-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:75f9eb72ecb640619c29bf714e78c9c46c9c4eaafd644bf78577ede459f330d4", size = 205074, upload-time = "2025-05-18T19:04:19.21Z" },
-    { url = "https://files.pythonhosted.org/packages/54/46/caa2c1342655f57d8f0f2519774c6d67132205909c65e9aa8255e1d7b4f4/jiter-0.10.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:28ed2a4c05a1f32ef0e1d24c2611330219fed727dae01789f4a335617634b1ca", size = 318225, upload-time = "2025-05-18T19:04:20.583Z" },
-    { url = "https://files.pythonhosted.org/packages/43/84/c7d44c75767e18946219ba2d703a5a32ab37b0bc21886a97bc6062e4da42/jiter-0.10.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14a4c418b1ec86a195f1ca69da8b23e8926c752b685af665ce30777233dfe070", size = 350235, upload-time = "2025-05-18T19:04:22.363Z" },
-    { url = "https://files.pythonhosted.org/packages/01/16/f5a0135ccd968b480daad0e6ab34b0c7c5ba3bc447e5088152696140dcb3/jiter-0.10.0-cp313-cp313t-win_amd64.whl", hash = "sha256:d7bfed2fe1fe0e4dda6ef682cee888ba444b21e7a6553e03252e4feb6cf0adca", size = 207278, upload-time = "2025-05-18T19:04:23.627Z" },
-    { url = "https://files.pythonhosted.org/packages/1c/9b/1d646da42c3de6c2188fdaa15bce8ecb22b635904fc68be025e21249ba44/jiter-0.10.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:5e9251a5e83fab8d87799d3e1a46cb4b7f2919b895c6f4483629ed2446f66522", size = 310866, upload-time = "2025-05-18T19:04:24.891Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/0e/26538b158e8a7c7987e94e7aeb2999e2e82b1f9d2e1f6e9874ddf71ebda0/jiter-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:023aa0204126fe5b87ccbcd75c8a0d0261b9abdbbf46d55e7ae9f8e22424eeb8", size = 318772, upload-time = "2025-05-18T19:04:26.161Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/fb/d302893151caa1c2636d6574d213e4b34e31fd077af6050a9c5cbb42f6fb/jiter-0.10.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c189c4f1779c05f75fc17c0c1267594ed918996a231593a21a5ca5438445216", size = 344534, upload-time = "2025-05-18T19:04:27.495Z" },
-    { url = "https://files.pythonhosted.org/packages/01/d8/5780b64a149d74e347c5128d82176eb1e3241b1391ac07935693466d6219/jiter-0.10.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:15720084d90d1098ca0229352607cd68256c76991f6b374af96f36920eae13c4", size = 369087, upload-time = "2025-05-18T19:04:28.896Z" },
-    { url = "https://files.pythonhosted.org/packages/e8/5b/f235a1437445160e777544f3ade57544daf96ba7e96c1a5b24a6f7ac7004/jiter-0.10.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4f2fb68e5f1cfee30e2b2a09549a00683e0fde4c6a2ab88c94072fc33cb7426", size = 490694, upload-time = "2025-05-18T19:04:30.183Z" },
-    { url = "https://files.pythonhosted.org/packages/85/a9/9c3d4617caa2ff89cf61b41e83820c27ebb3f7b5fae8a72901e8cd6ff9be/jiter-0.10.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ce541693355fc6da424c08b7edf39a2895f58d6ea17d92cc2b168d20907dee12", size = 388992, upload-time = "2025-05-18T19:04:32.028Z" },
-    { url = "https://files.pythonhosted.org/packages/68/b1/344fd14049ba5c94526540af7eb661871f9c54d5f5601ff41a959b9a0bbd/jiter-0.10.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31c50c40272e189d50006ad5c73883caabb73d4e9748a688b216e85a9a9ca3b9", size = 351723, upload-time = "2025-05-18T19:04:33.467Z" },
-    { url = "https://files.pythonhosted.org/packages/41/89/4c0e345041186f82a31aee7b9d4219a910df672b9fef26f129f0cda07a29/jiter-0.10.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fa3402a2ff9815960e0372a47b75c76979d74402448509ccd49a275fa983ef8a", size = 392215, upload-time = "2025-05-18T19:04:34.827Z" },
-    { url = "https://files.pythonhosted.org/packages/55/58/ee607863e18d3f895feb802154a2177d7e823a7103f000df182e0f718b38/jiter-0.10.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:1956f934dca32d7bb647ea21d06d93ca40868b505c228556d3373cbd255ce853", size = 522762, upload-time = "2025-05-18T19:04:36.19Z" },
-    { url = "https://files.pythonhosted.org/packages/15/d0/9123fb41825490d16929e73c212de9a42913d68324a8ce3c8476cae7ac9d/jiter-0.10.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:fcedb049bdfc555e261d6f65a6abe1d5ad68825b7202ccb9692636c70fcced86", size = 513427, upload-time = "2025-05-18T19:04:37.544Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/b3/2bd02071c5a2430d0b70403a34411fc519c2f227da7b03da9ba6a956f931/jiter-0.10.0-cp314-cp314-win32.whl", hash = "sha256:ac509f7eccca54b2a29daeb516fb95b6f0bd0d0d8084efaf8ed5dfc7b9f0b357", size = 210127, upload-time = "2025-05-18T19:04:38.837Z" },
-    { url = "https://files.pythonhosted.org/packages/03/0c/5fe86614ea050c3ecd728ab4035534387cd41e7c1855ef6c031f1ca93e3f/jiter-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5ed975b83a2b8639356151cef5c0d597c68376fc4922b45d0eb384ac058cfa00", size = 318527, upload-time = "2025-05-18T19:04:40.612Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/4a/4175a563579e884192ba6e81725fc0448b042024419be8d83aa8a80a3f44/jiter-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3aa96f2abba33dc77f79b4cf791840230375f9534e5fac927ccceb58c5e604a5", size = 354213, upload-time = "2025-05-18T19:04:41.894Z" },
-]
-
 [[package]]
 name = "levelapp"
-version = "0.1.3"
+version = "0.1.4"
 source = { editable = "." }
 dependencies = [
     { name = "google-api-core" },
@@ -362,10 +283,12 @@ dependencies = [
 
 [package.optional-dependencies]
 dev = [
-    { name = "arrow" },
+    { name = "google-api-core" },
+    { name = "google-auth" },
+    { name = "google-cloud-firestore" },
     { name = "httpx" },
+    { name = "humanize" },
     { name = "numpy" },
-    { name = "openai" },
     { name = "pandas" },
     { name = "pandas-stubs" },
     { name = "pydantic" },
@@ -379,16 +302,18 @@ dev = [
 
 [package.metadata]
 requires-dist = [
-    { name = "arrow", marker = "extra == 'dev'", specifier = ">=1.3.0" },
     { name = "google-api-core", specifier = ">=2.25.1" },
+    { name = "google-api-core", marker = "extra == 'dev'", specifier = ">=2.25.1" },
     { name = "google-auth", specifier = ">=2.40.3" },
+    { name = "google-auth", marker = "extra == 'dev'", specifier = ">=2.40.3" },
     { name = "google-cloud-firestore", specifier = ">=2.21.0" },
+    { name = "google-cloud-firestore", marker = "extra == 'dev'", specifier = ">=2.21.0" },
     { name = "httpx", specifier = ">=0.28.1" },
     { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.28.1" },
     { name = "humanize", specifier = ">=4.13.0" },
+    { name = "humanize", marker = "extra == 'dev'", specifier = ">=4.13.0" },
     { name = "numpy", specifier = ">=2.3.2" },
     { name = "numpy", marker = "extra == 'dev'", specifier = ">=2.3.2" },
-    { name = "openai", marker = "extra == 'dev'", specifier = ">=1.99.9" },
     { name = "pandas", specifier = ">=2.3.1" },
     { name = "pandas", marker = "extra == 'dev'", specifier = ">=2.3.1" },
     { name = "pandas-stubs", specifier = "==2.3.0.250703" },
@@ -473,25 +398,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c1/9e/1652778bce745a67b5fe05adde60ed362d38eb17d919a540e813d30f6874/numpy-2.3.2-cp314-cp314t-win_arm64.whl", hash = "sha256:092aeb3449833ea9c0bf0089d70c29ae480685dd2377ec9cdbbb620257f84631", size = 10544226, upload-time = "2025-07-24T20:56:34.509Z" },
 ]
 
-[[package]]
-name = "openai"
-version = "1.99.9"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "anyio" },
-    { name = "distro" },
-    { name = "httpx" },
-    { name = "jiter" },
-    { name = "pydantic" },
-    { name = "sniffio" },
-    { name = "tqdm" },
-    { name = "typing-extensions" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/8a/d2/ef89c6f3f36b13b06e271d3cc984ddd2f62508a0972c1cbcc8485a6644ff/openai-1.99.9.tar.gz", hash = "sha256:f2082d155b1ad22e83247c3de3958eb4255b20ccf4a1de2e6681b6957b554e92", size = 506992, upload-time = "2025-08-12T02:31:10.054Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/e8/fb/df274ca10698ee77b07bff952f302ea627cc12dac6b85289485dd77db6de/openai-1.99.9-py3-none-any.whl", hash = "sha256:9dbcdb425553bae1ac5d947147bebbd630d91bbfc7788394d4c4f3a35682ab3a", size = 786816, upload-time = "2025-08-12T02:31:08.34Z" },
-]
-
 [[package]]
 name = "pandas"
 version = "2.3.1"
@@ -797,27 +703,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e5/30/643397144bfbfec6f6ef821f36f33e57d35946c44a2352d3c9f0ae847619/tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138", size = 28248, upload-time = "2025-04-02T08:25:07.678Z" },
 ]
 
-[[package]]
-name = "tqdm"
-version = "4.67.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "colorama", marker = "sys_platform == 'win32'" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737, upload-time = "2024-11-24T20:12:22.481Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540, upload-time = "2024-11-24T20:12:19.698Z" },
-]
-
-[[package]]
-name = "types-python-dateutil"
-version = "2.9.0.20250809"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a3/53/07dac71db45fb6b3c71c2fd29a87cada2239eac7ecfb318e6ebc7da00a3b/types_python_dateutil-2.9.0.20250809.tar.gz", hash = "sha256:69cbf8d15ef7a75c3801d65d63466e46ac25a0baa678d89d0a137fc31a608cc1", size = 15820, upload-time = "2025-08-09T03:14:14.109Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/43/5e/67312e679f612218d07fcdbd14017e6d571ce240a5ba1ad734f15a8523cc/types_python_dateutil-2.9.0.20250809-py3-none-any.whl", hash = "sha256:768890cac4f2d7fd9e0feb6f3217fce2abbfdfc0cadd38d11fba325a815e4b9f", size = 17707, upload-time = "2025-08-09T03:14:13.314Z" },
-]
-
 [[package]]
 name = "types-pytz"
 version = "2025.2.0.20250809"

levelapp-0.1.3/levelapp/simulator/utils.py

@@ -1,163 +0,0 @@
-"""
-'simulators/aspects.py': Utility functions for handling VLA interactions and requests.
-"""
-import ast
-import json
-import httpx
-
-from uuid import UUID
-from string import Template
-from typing import Any, Dict, List, Union
-
-from pydantic import ValidationError
-
-from levelapp.clients import ClientRegistry
-from levelapp.config.prompts import SUMMARIZATION_PROMPT_TEMPLATE
-from levelapp.simulator.schemas import InteractionResults
-from levelapp.aspects import MonitoringAspect, MetricType, logger
-
-
-class UUIDEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, UUID):
-            return str(obj)
-        return json.JSONEncoder.default(self, obj)
-
-
-def extract_interaction_details(
-        response: str | Dict[str, Any],
-        template: Dict[str, Any],
-) -> InteractionResults:
-    """
-    Extract interaction details from a VLA response.
-
-    Args:
-        response (str): The response text from the VLA.
-        template (Dict[str, Any]): The response schema/template.
-
-    Returns:
-        InteractionResults: The extracted interaction details.
-    """
-    try:
-        response_dict = response if isinstance(response, dict) else json.loads(response)
-
-        if not isinstance(response_dict, dict):
-            raise ValueError("Response is not a valid dictionary")
-
-        required_keys = {value.strip("${}") for value in template.values()}
-        if not required_keys.issubset(response_dict.keys()):
-            missing_keys = required_keys - response_dict.keys()
-            logger.warning(f"[extract_interaction_details] Missing data: {missing_keys}]")
-
-        output = {}
-        for k, v in template.items():
-            output[k] = Template(v).safe_substitute(response_dict)
-
-        raw_value = output.get("generated_metadata", {})
-        output["generated_metadata"] = ast.literal_eval(raw_value) if isinstance(raw_value, str) else raw_value
-
-        return InteractionResults.model_validate(output)
-
-    except json.JSONDecodeError as e:
-        logger.error(f"[extract_interaction_details] Failed to extract details:\n{e}")
-        return InteractionResults()
-
-    except ValidationError as e:
-        logger.exception(f"[extract_interaction_details] Failed to create an InteractionResults instance:\n{e}")
-        return InteractionResults()
-
-
-@MonitoringAspect.monitor(name="interaction_request", category=MetricType.API_CALL)
-async def async_interaction_request(
-        url: str,
-        headers: Dict[str, str],
-        payload: Dict[str, Any],
-) -> httpx.Response | None:
-    """
-    Perform an asynchronous interaction request.
-
-    Args:
-        url (str): The URL to send the request to.
-        headers (Dict[str, str]): The headers to include in the request.
-        payload (Dict[str, Any]): The payload to send in the request.
-
-    Returns:
-        httpx.Response: The response from the interaction request, or None if an error occurred.
-    """
-    try:
-        async with httpx.AsyncClient(timeout=180) as client:
-            response = await client.post(url=url, headers=headers, json=payload)
-            response.raise_for_status()
-
-            return response
-
-    except httpx.HTTPStatusError as http_err:
-        logger.error(f"[async_interaction_request] HTTP error: {http_err.response.text}", exc_info=True)
-
-    except httpx.RequestError as req_err:
-        logger.error(f"[async_interaction_request] Request error: {str(req_err)}", exc_info=True)
-
-    return None
-
-
-@MonitoringAspect.monitor(
-    name="average_calc",
-    category=MetricType.SCORING,
-    cached=True,
-    maxsize=1000
-)
-def calculate_average_scores(scores: Dict[str, Union[List[float], float]]) -> Dict[str, float]:
-    """
-    Helper function that calculates the average scores for a dictionary of score lists.
-
-    Args:
-        scores (Dict[str, List[float]]): A dictionary where keys are identifiers and values are lists of scores.
-
-    Returns:
-        Dict[str, float]: A dictionary with average scores rounded to three decimal places.
-    """
-    result: Dict[str, float] = {}
-    for field, value in scores.items():
-        if isinstance(value, (int, float)):
-            result[field] = value
-        elif isinstance(value, list):
-            result[field] = round((sum(value) / len(value)), 3) if value else 0.0
-        else:
-            raise TypeError(f"[calculate_average_scores] Unexpected type '{type(value)}' for field '{field}")
-
-    return result
-
-
-@MonitoringAspect.monitor(name="summarization", category=MetricType.API_CALL)
-def summarize_verdicts(
-        verdicts: List[str],
-        judge: str,
-        max_bullets: int = 5
-) -> List[str]:
-    client_registry = ClientRegistry()
-    client = client_registry.get(provider=judge)
-
-    try:
-        verdicts = chr(10).join(verdicts)
-        prompt = SUMMARIZATION_PROMPT_TEMPLATE.format(max_bullets=max_bullets, judge=judge, verdicts=verdicts)
-        response = client.call(message=prompt)
-        parsed = client.parse_response(response=response)
-        striped = parsed.get("output", "").strip("")
-        bullet_points = [point.strip() for point in striped.split("- ") if point.strip()]
-
-        return bullet_points[:max_bullets]
-
-    except Exception as e:
-        logger.error(f"[summarize_justifications] Error during summarization: {str(e)}", exc_info=True)
-        return []
-
-
-# if __name__ == '__main__':
-#     template = {'generated_reply': '${agent_reply}', 'generated_metadata': '${generated_metadata}'}
-#     response_dict = {
-#         'agent_reply': "I'd be happy to help you book something for 10 AM.",
-#         'generated_metadata': {'appointment_type': 'Cardiology', 'date': 'next Monday', 'time': '10 AM'}
-#     }
-#
-#     result = extract_interaction_details(response_dict, template)
-#     print(f"result: {result.model_dump()}")

levelapp-0.1.3/src/level_app/main_session.py

@@ -1,48 +0,0 @@
-if __name__ == "__main__":
-    from levelapp.workflow import WorkflowConfig
-    from levelapp.core.session import EvaluationSession
-
-    # Firestore -> retrieve endpoint config -> data => config_dict
-
-    # config_dict_ = {
-    #     "process": {"project_name": "test-project", "workflow_type": "SIMULATOR", "evaluation_params": {"attempts": 2}},
-    #     "evaluation": {"evaluators": ["JUDGE"], "providers": ["openai", "ionos"]},
-    #     "reference_data": {"path": "", "data": {}},
-    #     "endpoint": {"base_url": "http://127.0.0.1:8000", "api_key": "key", "model_id": "model"},
-    #     "repository": {"type": "FIRESTORE", "source": "IN_MEMORY", "metrics_map": {"field_1": "EXACT"}},
-    # }
-    #
-    # content = {
-    #     "scripts": [
-    #         {
-    #             "interactions": [
-    #                 {
-    #                     "user_message": "Hello World!",
-    #                     "reference_reply": "Hello, how can I help you!"
-    #                 },
-    #                 {
-    #                     "user_message": "I need an apartment",
-    #                     "reference_reply": "sorry, but I can only assist you with booking medical appointments."
-    #                 },
-    #             ]
-    #         },
-    #     ]
-    # }
-    #
-    # # Load configuration from YAML
-    # config = WorkflowConfig.from_dict(content=config_dict_)
-    #
-    # # Load reference data from in-memory dict
-    # config.set_reference_data(content=content)
-
-    config = WorkflowConfig.load(path="../data/workflow_config.yaml")
-
-    evaluation_session = EvaluationSession(session_name="test-session", workflow_config=config, enable_monitoring=True)
-
-    with evaluation_session as session:
-        session.run()
-        results = session.workflow.collect_results()
-        print("Results:", results)
-
-    stats = session.get_stats()
-    print(f"session stats:\n{stats}")