levelapp 0.1.4__py3-none-any.whl → 0.1.5__py3-none-any.whl

{levelapp-0.1.4.dist-info → levelapp-0.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: levelapp
-Version: 0.1.4
+Version: 0.1.5
 Summary: LevelApp is an evaluation framework for AI/LLM-based software application. [Powered by Norma]
 Project-URL: Homepage, https://github.com/levelapp-org
 Project-URL: Repository, https://github.com/levelapp-org/levelapp-framework
@@ -114,23 +114,54 @@ evaluation:
     field_2 : LEVENSHTEIN
 
 reference_data:
-  path: 
+  path: "../data/conversation_example_1.json"
   data:
 
 endpoint:
-  base_url: "http://127.0.0.1:8000"
-  url_path: ''
-  api_key: "<API-KEY>"
-  bearer_token: "<BEARER-TOKEN>"
-  model_id: "meta-llama/Meta-Llama-3.1-8B-Instruct"
-  default_request_payload_template:
-    # Change the user message field name only according to the request payload schema (example: 'prompt' to 'message').
-    prompt: "${user_message}"
-    details: "${request_payload}"  # Rest of the request payload data.
-  default_response_payload_template:
-    # Change the placeholder value only according to the response payload schema (example: ${agent_reply} to ${reply}).
-    agent_reply: "${agent_reply}"
-    generated_metadata: "${generated_metadata}"
+  name: conversational-agent
+  base_url: http://127.0.0.1:8000
+  path: /v1/chat
+  method: POST
+  timeout: 60
+  retry_count: 3
+  retry_backoff: 0.5
+  headers:
+    - name: model_id
+      value: meta-llama/Meta-Llama-3-8B-Instruct
+      secure: false
+    - name: x-api-key
+      value: API_KEY  # Load from .env file using python-dotenv.
+      secure: true
+    - name: Content-Type
+      value: application/json
+      secure: false
+  request_schema:
+    # Static field to be included in every request.
+    - field_path: message.source
+      value: system
+      value_type: static
+      required: true
+      
+    # Dynamic field to be populated from runtime context.
+    - field_path: message.text
+      value: message_text  # the key from the runtime context.
+      value_type: dynamic
+      required: true
+      
+    # Env-based field (from OS environment variables).
+    - field_path: metadata.env
+      value: ENV_VAR_NAME
+      value_type: env
+      required: false
+      
+  response_mapping:
+    # Map the response fields that will be extracted.
+    - field_path: reply.text
+      extract_as: agent_reply  # The simulator requires this key: 'agent_reply'.
+    - field_path: reply.metadata
+      extract_as: generated_metadata  # The simulator requires this key: 'generated_metadata'.
+    - field_path: reply.guardrail_flag
+      extract_as: guardrail_flag  # The simulator requires this key: 'guardrail_flag'.
 
 repository:
   type: FIRESTORE # Pick one of the following: FIRESTORE, FILESYSTEM
@@ -138,8 +169,8 @@ repository:
   database_name: ""
 ```
 
-- **Endpoint Configuration**: Define how to interact with your LLM-based system (base URL, auth, payload templates).
-- **Placeholders**: For the request payload, change the field names (e.g., 'prompt' to 'message') according to your API specs. For the response payload, change the place holders values (e.g., `${agent_reply}` to `${generated_reply}`).
+- **Endpoint Configuration**: Define how to interact with your LLM-based system (base URL, headers, request/response payload schema).
+- **Placeholders**: For dynamic request schema fields, use the values ('value') to dynamically populate these fields during runtime (e.g., `context = {'message_text': "Hello, world!"}`).
 - **Secrets**: Store API keys in `.env` and load via `python-dotenv` (e.g., `API_KEY=your_key_here`).
 
 For conversation scripts (used in Simulator), provide a JSON file with this schema:
@@ -154,16 +185,14 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
           "reference_reply": "Sure, I can help with that. Could you please specify the type of doctor you need to see?",
           "interaction_type": "initial",
           "reference_metadata": {},
-          "guardrail_flag": false,
-          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+          "guardrail_flag": false
         },
         {
           "user_message": "I need to see a cardiologist.",
           "reference_reply": "When would you like to schedule your appointment?",
           "interaction_type": "intermediate",
           "reference_metadata": {},
-          "guardrail_flag": false,
-          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+          "guardrail_flag": false
         },
         {
           "user_message": "I would like to book it for next Monday morning.",
@@ -174,8 +203,7 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
             "date": "next Monday",
             "time": "10 AM"
           },
-          "guardrail_flag": false,
-          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+          "guardrail_flag": false
         },
         {
           "id": "f4f2dd35-71d7-4b75-ba2b-93a4f546004a",
@@ -183,8 +211,7 @@ For conversation scripts (used in Simulator), provide a JSON file with this sche
           "reference_reply": "Your appointment with the cardiologist is booked for 10 AM next Monday. Is there anything else I can help you with?",
           "interaction_type": "final",
           "reference_metadata": {},
-          "guardrail_flag": false,
-          "request_payload": {"user_id":  "0001", "user_role": "ADMIN"}
+          "guardrail_flag": false
         }
       ],
       "description": "A conversation about booking a doctor appointment.",
@@ -245,11 +272,90 @@ if __name__ == "__main__":
 
     
     config_dict = {
-        "process": {"project_name": "test-project", "workflow_type": "SIMULATOR", "evaluation_params": {"attempts": 2}},
-        "evaluation": {"evaluators": ["JUDGE", "REFERENCE"], "providers": ["openai", "ionos"], "metrics_map": {"field_1": "EXACT"}},
-        "reference_data": {"path": "", "data": {}},
-        "endpoint": {"base_url": "http://127.0.0.1:8000", "api_key": "key", "model_id": "model"},
-        "repository": {"type": "FIRESTORE", "source": "IN_MEMORY"},
+        "process": {
+            "project_name": "test-project",
+            "workflow_type": "SIMULATOR",  # Pick one of the following workflows: SIMULATOR, COMPARATOR, ASSESSOR.
+            "evaluation_params": {
+                "attempts": 1,  # Add the number of simulation attempts.
+            }
+        },
+        "evaluation": {
+            "evaluators": ["JUDGE", "REFERENCE"],  # Select from the following: JUDGE, REFERENCE, RAG.
+            "providers": ["openai", "ionos"],
+            "metrics_map": {
+                "field_1": "EXACT",
+                "field_2": "LEVENSHTEIN"
+            }
+        },
+        "reference_data": {
+            "path": "../data/conversation_example_1.json",
+            "data": None
+        },
+        "endpoint": {
+            "name": "conversational-agent",
+            "base_url": "http://127.0.0.1:8000",
+            "path": "/v1/chat",
+            "method": "POST",
+            "timeout": 60,
+            "retry_count": 3,
+            "retry_backoff": 0.5,
+            "headers": [
+                {
+                    "name": "model_id",
+                    "value": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+                    "secure": False
+                },
+                {
+                    "name": "x-api-key",
+                    "value": "API_KEY",  # Load from .env file using python-dotenv.
+                    "secure": True 
+                },
+                {
+                    "name": "Content-Type",
+                    "value": "application/json",
+                    "secure": False
+                }
+            ],
+            "request_schema": [
+                {
+                    "field_path": "message.source",
+                    "value": "system",
+                    "value_type": "static",
+                    "required": True
+                },
+                {
+                    "field_path": "message.text",
+                    "value": "message_text",  # the key from the runtime context.
+                    "value_type": "dynamic",
+                    "required": True
+                },
+                {
+                    "field_path": "metadata.env",
+                    "value": "ENV_VAR_NAME",
+                    "value_type": "env",
+                    "required": False
+                }
+            ],
+            "response_mapping": [
+                {
+                    "field_path": "reply.text",
+                    "extract_as": "agent_reply"  # Remember that the simulator requires this key: 'agent_reply'.
+                },
+                {
+                    "field_path": "reply.metadata",
+                    "extract_as": "agent_reply"  # Remember that the simulator requires this key: 'agent_reply'.
+                },
+                {
+                    "field_path": "reply.guardrail_flag",
+                    "extract_as": "metadata"  # Remember that the simulator requires this key: 'agent_reply'.
+                }
+            ]
+        },
+        "repository": {
+            "type": "FIRESTORE",  # Pick one of the following: FIRESTORE, FILESYSTEM
+            "project_id": "(default)",
+            "database_name": ""
+        }
     }
 
     content = {
@@ -275,9 +381,18 @@ if __name__ == "__main__":
     # Load reference data from dict variable
     config.set_reference_data(content=content)
 
-    evaluation_session = EvaluationSession(session_name="test-session-2", workflow_config=config)
+    evaluation_session = EvaluationSession(
+        session_name="test-session", 
+        workflow_config=config, 
+        enable_monitoring=True  # To disable the monitoring aspect, set this to False.
+    )
 
     with evaluation_session as session:
+        # Optional: Run connectivity test before the full evaluation
+        test_results = session.run_connectivity_test(
+            context={"user_message": "I want to book an appointment with a dentist."}
+        )
+        print(f"Connectivity Test Results:\n{test_results}\n---")
         session.run()
         results = session.workflow.collect_results()
         print("Results:", results)

{levelapp-0.1.4.dist-info → levelapp-0.1.5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 levelapp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 levelapp/aspects/__init__.py,sha256=_OaPcjTWBizqcUdDVj5aYue7lG9ytjQGLhPvReriKnU,326
-levelapp/aspects/loader.py,sha256=xWpcWtS25zbVhZ0UnIJEcQA9klajKk10TLK4j1IStH0,9543
+levelapp/aspects/loader.py,sha256=IB2sZTmTdAvYHQZlH7PdZGQHh3r86P-zX3rIp0PyG2M,9577
 levelapp/aspects/logger.py,sha256=MJ9HphyHYkTE5-ajA_WuMUTM0qQzd0WIP243vF-pj3M,1698
 levelapp/aspects/monitor.py,sha256=ibUk01Y5y67_qBJRA5YzvjMX8QrRkMTJ-mN77ztuLlo,22113
 levelapp/aspects/sanitizer.py,sha256=zUqgb76tXJ8UUYtHp0Rz7q9PZjAHpSpHPPFfGTjjQNg,5229
@@ -16,15 +16,25 @@ levelapp/comparator/schemas.py,sha256=lUAQzEyStidt2ePQgV2zq-An5MLBrVSw6t8fB0FQKJ
 levelapp/comparator/scorer.py,sha256=LBRy8H11rXulSa-k40BcycPcMAHgdUm13qS7ibWHq6I,9032
 levelapp/comparator/utils.py,sha256=Eu48nDrNzFr0lwAJJS0aNhKsAWQ72syTEWYMNYfg764,4331
 levelapp/config/__init__.py,sha256=9oaajE5zW-OVWOszUzMAG6nHDSbLQWa3KT6bVoSvzRA,137
+levelapp/config/api_config.yaml,sha256=t5OGz2YyEWM6G6n5PX7erYLivnZJQ5AD61wnA7Ntd8k,4364
+levelapp/config/dashq_api.yaml,sha256=msQn-0pQOcNueb4A3funSt8mJqNjmOv0EIxhGeVmKRM,3019
 levelapp/config/endpoint.py,sha256=B-uIEKF-0_Y6Vo8MZ8eoCZocRkghijrdpwT3zq0FDLk,7647
-levelapp/config/endpoint_.py,sha256=-abrIYKbFPLxTqNst-zbCI4MpMCmCMe9VZ6O8OwNRiE,1629
+levelapp/config/endpoint_.py,sha256=KEVEgYvnB1UfVczWyvYJjtIafFAeqKT-mHE776hMxlE,13379
+levelapp/config/endpoints.yaml,sha256=9FyK9CvfKKG8Vp200_GrxYzm1ZUYw8g8Ad5VEPW2P6k,1101
 levelapp/config/prompts.py,sha256=NXOKRp5l1VQ9LO0pUojVH6TDJhWyZImsAvZEz2QiD9k,2206
 levelapp/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 levelapp/core/base.py,sha256=oh4OkKgwGxmw_jgjX6wrBoK0KPc1JvCMZfbZP_mGmIg,12453
 levelapp/core/schemas.py,sha256=E47d93MMOj4eRYZIqUyLBiE5Ye7WgwkOJPOWQ6swRmo,465
-levelapp/core/session.py,sha256=6utDbLdg6DjwHL5dP-4wGe4_f7gFgEukuNNeOnbCbtA,9035
+levelapp/core/session.py,sha256=-NXpJlwyQRnswaQU8sQ-Ozgi9YQOe17Rxay__ILrUHQ,9344
+levelapp/endpoint/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp/endpoint/client.py,sha256=QBCGAikGjZvlylvXiLKZLprQ4anZwAUpGSmsiOneCrQ,3496
+levelapp/endpoint/manager.py,sha256=xypEQqYxi1RiSaUbL0HlHNJwvCliz3Tx9u_TGKPELSo,4212
+levelapp/endpoint/parsers.py,sha256=13RewihOiWVt7gMQ3g-UOEz8kNpSZ-KY2NgAN44Il2E,4263
+levelapp/endpoint/schemas.py,sha256=V0tpXC8aawHpX5zatderYa0fB14_QEkPiKRvfsuGZRM,851
+levelapp/endpoint/tester.py,sha256=6ylVRwsS_BRo19pJh_J7UcH3NZIfDxxOoccfSqQ9tR4,2078
+levelapp/endpoint/usage_example.py,sha256=XKpUYffLIe81osmFdoSM5oYHUw9HkKdQr_0h4TIZxSg,1242
 levelapp/evaluator/__init__.py,sha256=K-P75Q1FXXLCNqH1wyhT9sf4y2R9a1qR5449AXEsY1k,109
-levelapp/evaluator/evaluator.py,sha256=JCRgQps9GKlJBDYw9xzVrC2_aGy0GhGAJ0ZkSC_IWWA,10806
+levelapp/evaluator/evaluator.py,sha256=kkWQg4GEqDyNeIVwFzkk36uDYfPFXRQQ7jsK51BDSA4,11000
 levelapp/metrics/__init__.py,sha256=x8iTaeDezJyQ9-NFe8GGvzwIBhyAJHWSRfBE3JRX-PE,1878
 levelapp/metrics/embedding.py,sha256=wvlT8Q5DjDT6GrAIFtc5aFbA_80hDLUXMP4RbSpSwHE,115
 levelapp/metrics/exact.py,sha256=Kb13nD2OVLrl3iYHaXrxDfrxDuhW0SMVvLAEXPaJtlY,6235
@@ -32,19 +42,20 @@ levelapp/metrics/fuzzy.py,sha256=Rg8ashzMxtQwKO-z_LLzdj2PDIRqL4CBw6PGRf9IBrI,259
 levelapp/metrics/token.py,sha256=yQi9hxT_fXTGjLiCCemDxQ4Uk2zD-wQYtSnDlI2AuuY,3521
 levelapp/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 levelapp/repository/__init__.py,sha256=hNmFRZ7kKJN1mMlOHeW9xf0j9Q7gqTXYJ3hMCzk9to4,79
+levelapp/repository/filesystem.py,sha256=-C2oVThZt16K41iZNSEaM2qO3tTPsFKVsDYQQiwo1Bk,7475
 levelapp/repository/firestore.py,sha256=K9JgxsNCelAKtzTDv19c1dHRlitMeRzo7H3caTlKuF8,10369
 levelapp/simulator/__init__.py,sha256=8Dz8g7rbpBZX3WoknVmMVoWm_VT72ZL9BABOF1xFpqs,83
-levelapp/simulator/schemas.py,sha256=YGprtuRZ4m33WBD35xj1Ib5EbMTdDCOp-wCykf-Iz-4,3700
-levelapp/simulator/simulator.py,sha256=yreado12XMVEJ4N4cYj5m_bYVKU3BsOv5oQycB7wCFw,19889
-levelapp/simulator/utils.py,sha256=xZXdF24rrOm5RCp5ELk0wQxxGd70CahDT79cIC-XmlE,9589
+levelapp/simulator/schemas.py,sha256=LvkXa8KGh8aGZAz8mB9-hUDFVbQjzD8zVzMXeoI6ZGI,3715
+levelapp/simulator/simulator.py,sha256=7rlovMrbwyRndDnltAG98ff8ojny6hoXdVFGEdBkm0U,20474
+levelapp/simulator/utils.py,sha256=smSrZ8praKINK0wFpKl3tmqr21OUz_dheUOTH0miTys,4882
 levelapp/workflow/__init__.py,sha256=27b2obG7ObhR43yd2uH-R0koRB7-DG8Emnvrq8EjsTA,193
-levelapp/workflow/base.py,sha256=1A_xKSBOmVjfMbRBcNhDK6G17SEjqRIm-XjMw45IPC4,5596
-levelapp/workflow/config.py,sha256=MlHt1PsXD09aukB93fvKTew0D8WD4_jdnO93Nn6b2U0,2923
-levelapp/workflow/context.py,sha256=gjAZXHEdlsXqWY6DbXOfKXNbxQbahRPSnNzyWDqryPU,2559
+levelapp/workflow/base.py,sha256=wkvVbxAhWH1D9YJOoHCE4xfC4Hnl4hYpFYCG8_v-8dk,6643
+levelapp/workflow/config.py,sha256=hgch9KV-TMtbesEFi00eibFdk5JJStJwb5TGO5m-o-M,3124
+levelapp/workflow/context.py,sha256=KOXm_5HJWYyWfl9C83BqT37X7QVuUYSS-ZoDpZgXFQw,2696
 levelapp/workflow/factory.py,sha256=z1ttJmI59sU9HgOvPo3ixUJ_oPv838XgehfuOorlTt8,1634
 levelapp/workflow/registration.py,sha256=VHUHjLHXad5kjcKukaEOIf7hBZ09bT3HAzVmIT08aLo,359
-levelapp/workflow/runtime.py,sha256=cFyXNWXSuURKbrMDHdkTcjeItM9wHP-5DPljntwYL5g,686
-levelapp-0.1.4.dist-info/METADATA,sha256=zCmgM_evZ9Y0xcAX-so7foD_auO0I9PSzLGw0pL2HUY,12572
-levelapp-0.1.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-levelapp-0.1.4.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-levelapp-0.1.4.dist-info/RECORD,,
+levelapp/workflow/runtime.py,sha256=a3REqikh3-QHj0uYikqx0b4xQjq-w6VNyiUandL5GWw,690
+levelapp-0.1.5.dist-info/METADATA,sha256=rs-J5XSEWR2WNUWP6wVJkbIgwOooLPul3POZZZdsjUQ,16130
+levelapp-0.1.5.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+levelapp-0.1.5.dist-info/licenses/LICENSE,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+levelapp-0.1.5.dist-info/RECORD,,