codemie-test-harness 0.1.223__py3-none-any.whl → 0.1.225__py3-none-any.whl

codemie_test_harness/tests/enums/model_types.py

@@ -37,6 +37,7 @@ class ModelTypes(str, Enum):
     CLAUDE_4_OPUS = "claude-4-opus"
     CLAUDE_4_1_OPUS = "claude-4-1-opus"
     CLAUDE_4_SONNET_1M = "claude-4-sonnet-1m"
+    CLAUDE_4_5_HAIKU = "claude-4-5-haiku"
 
     # Other
     RLAB_QWQ_32B = "rlab-qwq-32b"

codemie_test_harness/tests/test_data/llm_test_data.py

@@ -56,6 +56,7 @@ MODEL_RESPONSES = [
     LlmResponseData(ModelTypes.CLAUDE_4_1_OPUS, AWS_ENVS),
     LlmResponseData(ModelTypes.CLAUDE_4_SONNET_1M, AWS_ENVS),
     LlmResponseData(ModelTypes.CLAUDE_4_5_SONNET, AWS_ENVS),
+    LlmResponseData(ModelTypes.CLAUDE_4_5_HAIKU, AWS_ENVS),
     # Other LLMs test data
     LlmResponseData(ModelTypes.RLAB_QWQ_32B, OTHER_ENVS),
     LlmResponseData(ModelTypes.DEEPSEEK_R1, OTHER_ENVS),

codemie_test_harness/tests/test_data/vendor_workflow_test_data.py

@@ -27,6 +27,89 @@ vendor_workflow_test_data = [
         marks=[pytest.mark.aws, pytest.mark.bedrock],
         id="AWS_Bedrock_Object_Input",
     ),
+    pytest.param(
+        VendorType.AWS,
+        CredentialTypes.AWS,
+        CredentialsManager.aws_credentials(),
+        "codemie-autotests-flow-string-input",
+        "pop",
+        """
+            # Pop Playlist (3 Songs)
+            
+            1. "Blinding Lights" - The Weeknd
+               *Upbeat synth-pop track with irresistible 80s vibes and a catchy chorus*
+            
+            2. "As It Was" - Harry Styles
+               *Melodic pop anthem with nostalgic undertones and danceable rhythm*
+            
+            3. "Levitating" - Dua Lipa
+               *Disco-infused pop banger with infectious grooves and stellar vocals*
+            
+            Enjoy these modern pop hits! Would you like any specific recommendations or adjustments to the playlist?
+        """,
+        marks=[pytest.mark.aws, pytest.mark.bedrock],
+        id="AWS_Bedrock_String_Input",
+    ),
+    pytest.param(
+        VendorType.AWS,
+        CredentialTypes.AWS,
+        CredentialsManager.aws_credentials(),
+        "codemie-autotests-flow-number-input",
+        json.dumps(3),
+        """
+            # Pop Playlist (3 Songs)
+            
+            1. "Shape of You" - Ed Sheeran
+            2. "Blinding Lights" - The Weeknd
+            3. "Dance Monkey" - Tones and I
+            
+            Each of these songs has been a massive pop hit with catchy hooks and radio-friendly production. Enjoy your playlist!
+        """,
+        marks=[pytest.mark.aws, pytest.mark.bedrock],
+        id="AWS_Bedrock_Number_Input",
+    ),
+    pytest.param(
+        VendorType.AWS,
+        CredentialTypes.AWS,
+        CredentialsManager.aws_credentials(),
+        "codemie-autotests-flow-boolean-input",
+        json.dumps(True),
+        """
+            # Pop Playlist (3 Songs)
+            
+            1. "Bad Guy" by Billie Eilish
+            2. "Blinding Lights" by The Weeknd
+            3. "As It Was" by Harry Styles
+            
+            These are three popular pop songs from recent years that feature catchy melodies, modern production, and have achieved significant commercial success.
+        """,
+        marks=[pytest.mark.aws, pytest.mark.bedrock],
+        id="AWS_Bedrock_Boolean_Input",
+    ),
+    pytest.param(
+        VendorType.AWS,
+        CredentialTypes.AWS,
+        CredentialsManager.aws_credentials(),
+        "codemie-autotests-flow-array-input",
+        json.dumps(["pop", "rock"]),
+        """
+            # Playlist Creation
+            
+            Here are two playlists with 2 songs each:
+            
+            ## Pop Playlist (2 songs)
+            1. "Shape of You" by Ed Sheeran
+            2. "Bad Guy" by Billie Eilish
+            
+            ## Rock Playlist (2 songs)
+            1. "Bohemian Rhapsody" by Queen
+            2. "Sweet Child O' Mine" by Guns N' Roses
+            
+            Would you like me to expand either playlist with more song suggestions or provide information about any of these tracks?
+        """,
+        marks=[pytest.mark.aws, pytest.mark.bedrock],
+        id="AWS_Bedrock_Array_Input",
+    ),
     # pytest.param(
     #     VendorType.AZURE,
     #     CredentialTypes.AZURE,

codemie_test_harness/tests/vendor/workflows/test_vendor_workflows.py

@@ -54,7 +54,7 @@ def test_vendor_workflow_installation_and_execution(
         user_input=user_input,
     )
 
-    similarity_check.check_similarity(response, expected_response)
+    similarity_check.check_similarity(response, expected_response, similarity_rank=50)
 
     uninstall_response = vendor_workflow_utils.uninstall_workflow(
         vendor=vendor_type,

{codemie_test_harness-0.1.223.dist-info → codemie_test_harness-0.1.225.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.223
+Version: 0.1.225
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.223)
+Requires-Dist: codemie-sdk-python (==0.1.225)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-repeat (>=0.9.3,<0.10.0)

{codemie_test_harness-0.1.223.dist-info → codemie_test_harness-0.1.225.dist-info}/RECORD

@@ -73,7 +73,7 @@ codemie_test_harness/tests/e2e/test_e2e.py,sha256=BjBzyCeZCSGE1iGhjdYKBo6pMwimZ2
 codemie_test_harness/tests/enums/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/enums/environment.py,sha256=p1qAX5kGZ83eZVZf1xE2OMaJ6Br8AWcudKrPQCY1Zms,3099
 codemie_test_harness/tests/enums/integrations.py,sha256=L4sItzBjFBR65gMCebSYchADjwxmTtpEWQZvxd9aSnU,188
-codemie_test_harness/tests/enums/model_types.py,sha256=aAT7v0B3i4ryXbh1__xaUAsYHE54Kdn488Pklz6YqaE,1316
+codemie_test_harness/tests/enums/model_types.py,sha256=Kc1uxjivCaHMdHex4SYde5wopD6GjDsr44db_K0KIPs,1358
 codemie_test_harness/tests/enums/tools.py,sha256=y-FMN0K4jtG9vlZMnwqLqtuJyfIzFm3D548SGF-cUqw,6587
 codemie_test_harness/tests/integrations/__init__.py,sha256=5vnZbxvYQ1Y91O8DJG3QfBHWcdYsMii59cMBzsvHBCg,237
 codemie_test_harness/tests/integrations/project/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -157,7 +157,7 @@ codemie_test_harness/tests/test_data/google_datasource_test_data.py,sha256=9hO8K
 codemie_test_harness/tests/test_data/index_test_data.py,sha256=VZJC_Fmko32EHaybUwCy0mWMiwAeUAAmhRa0Xt4oTHQ,1115
 codemie_test_harness/tests/test_data/integrations_test_data.py,sha256=qfsJG2wmYPcZPZ5JpIJPqhnyTqLQlFEd4uOEzK4264A,38795
 codemie_test_harness/tests/test_data/keycloak_tool_test_data.py,sha256=uF8YqHGgLpQVxKdpKXLe-7F-ipEGQiHHh28nZvvVGM8,1244
-codemie_test_harness/tests/test_data/llm_test_data.py,sha256=srhtBg8vU4IgN2hpmtk2m8Uxu9KaVQNFGfQM9XcuJx8,2535
+codemie_test_harness/tests/test_data/llm_test_data.py,sha256=NJvBLl-PlWFn_HzQ3yYckJ7FG5o2epGVzZ6jcgpuHcY,2595
 codemie_test_harness/tests/test_data/mcp_server_test_data.py,sha256=m6PImS_J2gPNY5ijf9MG_eOX_LxJjTZ23AXQDgaK_Oc,7151
 codemie_test_harness/tests/test_data/notification_tools_test_data.py,sha256=D7KLP-cyQljU2Io21CukphnzMadJ7VJ0FhXFdU3GfGk,794
 codemie_test_harness/tests/test_data/open_api_tools_test_data.py,sha256=mvf0legyzLy1Ng-gqtQntR9HskFwVCDaLqPzDRXV8rM,3093
@@ -172,7 +172,7 @@ codemie_test_harness/tests/test_data/servicenow_tools_test_data.py,sha256=PKw9zE
 codemie_test_harness/tests/test_data/vcs_tools_test_data.py,sha256=ZJdday96uOKOy_kPRBPp0w9JZbf0il7Y_hPfwRmUkuM,4518
 codemie_test_harness/tests/test_data/vendor_endpoint_test_data.py,sha256=Fti3ZbpS3m06Heyh5AAcoAKZuimX709iZQFPhzKvrC4,1075
 codemie_test_harness/tests/test_data/vendor_test_data.py,sha256=FFS35ikzMzSHShnfmdIme-WjueSjj0qBwa0OU6ooKfc,1409
-codemie_test_harness/tests/test_data/vendor_workflow_test_data.py,sha256=hZX_GpIz0RIjSuqvgCh_Sv5zwk4-iatP1J-SIKCL8cg,1588
+codemie_test_harness/tests/test_data/vendor_workflow_test_data.py,sha256=xcemVCa2i5SR1qJMNGQ2iND1e0kizdtG4Qh1M_xy9bA,4649
 codemie_test_harness/tests/test_data/workflow/invalid_config/invalid_assistant_id.yaml,sha256=_cioQNq3icemob9u0i-hXkTy2nflzyP0Ce8FWiPG14M,265
 codemie_test_harness/tests/test_data/workflow/invalid_config/invalid_assistant_in_state.yaml,sha256=t_W95zD5bfdGf3F6p64-2qBHz7SkL_7mFT675uieWZg,209
 codemie_test_harness/tests/test_data/workflow/invalid_config/invalid_data_source.yaml,sha256=Vwx3HyrQkL8sWNtfwL6d0qiJhru6X3ojKBASAzJeY9w,252
@@ -309,7 +309,7 @@ codemie_test_harness/tests/vendor/assistants/test_vendor_assistants.py,sha256=8u
 codemie_test_harness/tests/vendor/assistants/test_vendor_assistants_endpoints.py,sha256=XA9idTm6ZycoKCTGq31QpL_BeybgQa7dFC3RSqAPtec,13271
 codemie_test_harness/tests/vendor/workflows/__init__.py,sha256=qcP6VHdlR6qbNhjrqRGHE5JReIj4CH0WGlMvpKgm1aM,38
 codemie_test_harness/tests/vendor/workflows/test_vendor_workflow_endpoints.py,sha256=dkCmFPKj-sIVgbuaqQ2Ef7JBKUWGTftW9xZB3D1KCrU,11236
-codemie_test_harness/tests/vendor/workflows/test_vendor_workflows.py,sha256=BVHWU9RN5Wn6XzXbRxo5Lod28GV7mPkbQSkErC-OUY4,1813
+codemie_test_harness/tests/vendor/workflows/test_vendor_workflows.py,sha256=sHMbfc6-uG0nYFtp0CBIyyGFs8DbRdux-WfD4lWsKB8,1833
 codemie_test_harness/tests/webhook/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/webhook/test_webhook_service.py,sha256=POmxQG0tpcNW9-yKQ62CcnQpUEFYlTOs0_4H9MijIHY,8127
 codemie_test_harness/tests/workflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -417,7 +417,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
 codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=D835gaRbCnB4va5mi9TdA_u9StSpGXQ_fgzwW0S2pwo,1173
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=Se9imIiBYuJU78m1pLu0g4ZmHygKZjr6JjIWkGXTy1Q,1364
-codemie_test_harness-0.1.223.dist-info/METADATA,sha256=Gj4xWyOTQ8ZIJzhSOK2Ku_rtgHlZgM8bgp6zIZze1Oc,27184
-codemie_test_harness-0.1.223.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-codemie_test_harness-0.1.223.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
-codemie_test_harness-0.1.223.dist-info/RECORD,,
+codemie_test_harness-0.1.225.dist-info/METADATA,sha256=lq5XwAC5TPCoJzyws_eg29NgefHBrD0tCTSws_zwaSs,27184
+codemie_test_harness-0.1.225.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+codemie_test_harness-0.1.225.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
+codemie_test_harness-0.1.225.dist-info/RECORD,,