codemie-test-harness 0.1.160__py3-none-any.whl → 0.1.162__py3-none-any.whl

codemie_test_harness/tests/llm/assistants/test_lite_llm.py

@@ -0,0 +1,96 @@
+import pytest
+from codemie_sdk.models.integration import CredentialTypes
+from hamcrest import assert_that, has_item
+
+from codemie_test_harness.tests.enums.model_types import ModelTypes
+from codemie_test_harness.tests.test_data.llm_test_data import MODEL_RESPONSES
+from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.utils.client_factory import get_client
+from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.env_resolver import get_environment
+from codemie_test_harness.tests.utils.pytest_utils import check_mark
+
+
+@pytest.fixture(scope="module")
+def lite_llm_integration(integration_utils):
+    credential_values = CredentialsUtil.lite_llm_credentials()
+    integration = integration_utils.create_user_integration(
+        CredentialTypes.LITE_LLM,
+        credential_values,
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
+@pytest.fixture(scope="function")
+def invalid_lite_llm_integration(integration_utils):
+    credential_values = CredentialsUtil.invalid_lite_llm_credentials()
+    integration = integration_utils.create_user_integration(
+        CredentialTypes.LITE_LLM,
+        credential_values,
+        project_name=test_project_name,
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
+def pytest_generate_tests(metafunc):
+    if "model_type" in metafunc.fixturenames:
+        is_smoke = check_mark(metafunc, "smoke")
+        test_data = []
+        env = get_environment()
+        if is_smoke:
+            available_models = get_client().llms.list()
+            for model in available_models:
+                test_data.append(pytest.param(model.base_name))
+        else:
+            for model_data in MODEL_RESPONSES:
+                test_data.append(
+                    pytest.param(
+                        model_data.model_type,
+                        marks=pytest.mark.skipif(
+                            env not in model_data.environments,
+                            reason=f"Skip on non {'/'.join(str(env) for env in model_data.environments[:-1])} envs",
+                        ),
+                    )
+                )
+
+        metafunc.parametrize("model_type", test_data)
+
+
+@pytest.mark.assistant
+@pytest.mark.lite_llm
+@pytest.mark.regression
+def test_assistant_with_different_models_in_lite_llm(
+    llm_utils,
+    lite_llm_integration,
+    assistant_utils,
+    model_type,
+    similarity_check,
+):
+    assert_that(
+        [row.base_name for row in llm_utils.list_llm_models()],
+        has_item(model_type),
+        f"{model_type} is missing in backend response",
+    )
+    assistant = assistant_utils.create_assistant(model_type)
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+
+    if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
+        response = "\n".join(response.split("\n")[-3:])
+    similarity_check.check_similarity(response, "Hello")
+
+
+@pytest.mark.assistant
+@pytest.mark.lite_llm
+@pytest.mark.regression
+def test_assistant_with_invalid_lite_llm(
+    invalid_lite_llm_integration,
+    assistant,
+    assistant_utils,
+):
+    assistant = assistant(project_name=test_project_name)
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    assert_that(response.startswith("AI Agent run failed with error: Error code: 401"))

codemie_test_harness/tests/llm/assistants/test_llm.py

@@ -36,10 +36,10 @@ def pytest_generate_tests(metafunc):
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -57,10 +57,10 @@ def test_assistant_with_different_models(
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models_with_top_p_parameter(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -78,10 +78,10 @@ def test_assistant_with_different_models_with_top_p_parameter(
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models_with_temperature_parameter(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )

codemie_test_harness/tests/service/test_task_service.py

@@ -3,6 +3,14 @@ import uuid
 from datetime import datetime
 
 import pytest
+from codemie_sdk.models.assistant import (
+    AssistantChatRequest,
+    ToolKitDetails,
+    ToolDetails,
+    ChatMessage,
+    ChatRole,
+)
+from codemie_sdk.models.task import BackgroundTaskEntity
 from hamcrest import (
     assert_that,
     has_length,
@@ -13,14 +21,6 @@ from hamcrest import (
     is_not,
 )
 
-from codemie_sdk.models.assistant import (
-    AssistantChatRequest,
-    ToolKitDetails,
-    ToolDetails,
-    ChatMessage,
-    ChatRole,
-)
-from codemie_sdk.models.task import BackgroundTaskEntity
 from codemie_test_harness.tests import PROJECT
 
 
@@ -135,5 +135,12 @@ def test_get_task_not_found(assistant_utils):
     """Test getting a non-existent task."""
     with pytest.raises(Exception) as exc_info:
         assistant_utils.get_tasks("non-existent-task-id")
-    # assert "404" in str(exc_info.value) or "not found" in str(exc_info.value).lower(), Need to fix API to produce error
-    assert_that(str(exc_info.value), contains_string("500"))
+
+        assert_that(exc_info.value.response.status_code, equal_to(404))
+
+        assert_that(
+            exc_info.value.response.json()["error"]["details"],
+            equal_to(
+                "The task with ID [non-existent-task-id] could not be found in the system."
+            ),
+        )

codemie_test_harness/tests/service/test_workflow_execution_service.py

@@ -1,6 +1,7 @@
 from time import sleep
 
 import pytest
+from codemie_sdk.models.workflow import ExecutionStatus, WorkflowMode
 from hamcrest import (
     assert_that,
     equal_to,
@@ -12,10 +13,8 @@ from hamcrest import (
     is_not,
 )
 
-from codemie_sdk.models.workflow import ExecutionStatus, WorkflowMode
 from codemie_test_harness.tests import PROJECT
 from codemie_test_harness.tests.utils.base_utils import get_random_name
-
 from codemie_test_harness.tests.utils.yaml_utils import (
     AssistantModel,
     StateModel,
@@ -133,14 +132,16 @@ def test_run_workflow(workflow_utils, search_utils, test_workflow: str):
         )
 
 
-@pytest.mark.skip(reason="Bug: https://jiraeu.epam.com/browse/EPMCDME-7883")
 def test_list_executions_nonexistent_workflow(search_utils):
     with pytest.raises(Exception) as exc_info:
         search_utils.list_workflow_executions("non-existent-id")
 
+    assert_that(exc_info.value.response.status_code, equal_to(404))
     assert_that(
-        str(exc_info.value).lower(),
-        contains_string("No workflows found with id non-existent-id"),
+        str(exc_info.value.response.json()["error"]["details"]),
+        equal_to(
+            "The workflow with ID [non-existent-id] could not be found in the system."
+        ),
     )
 
 

codemie_test_harness/tests/utils/aws_parameters_store.py

@@ -623,6 +623,24 @@ class CredentialsUtil:
                 cred.value = "wrong_key"
         return credentials
 
+    @staticmethod
+    def lite_llm_credentials() -> List[CredentialValues]:
+        lite_llm_creds = AwsParameterStore.get_cloud_provider_credentials("litellm")
+        return [
+            CredentialValues(
+                key="api_key",
+                value=lite_llm_creds.get("api_key"),
+            ),
+        ]
+
+    @staticmethod
+    def invalid_lite_llm_credentials() -> List[CredentialValues]:
+        credentials = CredentialsUtil.lite_llm_credentials()
+        for cred in credentials:
+            if cred.key == "api_key":
+                cred.value = "wrong_key"
+        return credentials
+
     @staticmethod
     def jira_cloud_jql() -> str:
         jira_creds = AwsParameterStore.get_cloud_provider_credentials("jira")

{codemie_test_harness-0.1.160.dist-info → codemie_test_harness-0.1.162.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.160
+Version: 0.1.162
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.160)
+Requires-Dist: codemie-sdk-python (==0.1.162)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)

{codemie_test_harness-0.1.160.dist-info → codemie_test_harness-0.1.162.dist-info}/RECORD

@@ -78,7 +78,8 @@ codemie_test_harness/tests/integrations/user/test_default_integrations.py,sha256
 codemie_test_harness/tests/integrations/user/test_user_integrations.py,sha256=lGOoyGyKby1vOvXbz9de-PfhS0eUEX6vS7tg6vixtEE,8120
 codemie_test_harness/tests/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/llm/assistants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-codemie_test_harness/tests/llm/assistants/test_llm.py,sha256=06oFZikafBC8YrFxgm5RJqIw7or0h8CDWroZ-LvX1Dw,3495
+codemie_test_harness/tests/llm/assistants/test_lite_llm.py,sha256=-X6QMjFw1wuTWQ1vTLM-Ae_Ha9jLCCHlFY7LgUttqHM,3485
+codemie_test_harness/tests/llm/assistants/test_llm.py,sha256=rpS-rsRqR13HPKPDeT_PvgnlSg1_KtbXTAzQpfShL8E,3531
 codemie_test_harness/tests/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/providers/test_providers_endpoints.py,sha256=SfZVCaYcsk_mG2b9zVu7w2wOY54loQHdmRS8eK5PNo4,8120
 codemie_test_harness/tests/search/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -91,9 +92,9 @@ codemie_test_harness/tests/service/test_assistant_service.py,sha256=NSQUNMv_Dk7x
 codemie_test_harness/tests/service/test_datasource_service.py,sha256=_mER2wtvZr4SqCd6rJnRBG6q6-kDtS63xY5ZsDCdNt4,15444
 codemie_test_harness/tests/service/test_integration_service.py,sha256=ODQJhBRELTIy75aFpZt_-axJHbB1wwrX6oz7JGNjtus,7529
 codemie_test_harness/tests/service/test_llm_service.py,sha256=aBJz6Z82JzeWeTuhHTVaRm1Cm7lJMKGRMQ1f6rIalTQ,543
-codemie_test_harness/tests/service/test_task_service.py,sha256=kxbZBbxyTg4YsRhteMBEoaqiLwbcF9h03fQ6e9m9xTY,4562
+codemie_test_harness/tests/service/test_task_service.py,sha256=q2RaZbNooWraONLFCbebYqdgecs4aXUCbA9HOTzUIGw,4677
 codemie_test_harness/tests/service/test_user_service.py,sha256=NqSsxOLTGg7p7juHLFP6StEs2Qx_wfct9R7F83pe2JU,1135
-codemie_test_harness/tests/service/test_workflow_execution_service.py,sha256=dUNon4lHbzPIVM5Y9jwZw7PNTK1c9g-X4m98XQtUYDA,5356
+codemie_test_harness/tests/service/test_workflow_execution_service.py,sha256=pHV7WXvQKzpw0wMn6aHVZ01g7yZupmU5NXLVE9rIKjo,5419
 codemie_test_harness/tests/service/test_workflow_service.py,sha256=QyxtorhaCI1oE2D1OLx7X7jAlBv0kwwFpQztvV1nUus,8152
 codemie_test_harness/tests/test_data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/test_data/ado_test_plan_tools_test_data.py,sha256=Al5u4HNfrcoj-b072uEGsqUqjKqwXLGJXKQ0QeJT3PI,5778
@@ -220,7 +221,7 @@ codemie_test_harness/tests/ui/test_workflow_templates.py,sha256=pVuF98d3eEfinb5j
 codemie_test_harness/tests/ui/test_workflows.py,sha256=a2VY8BAsw3Po7e10r4g7S798GhpA4KYh2ZjnYEhU1PY,3664
 codemie_test_harness/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/utils/assistant_utils.py,sha256=jy3dFF4ZiT22Wippl1a5jIEAAyJ71P0ss8AIHPefz6k,6511
-codemie_test_harness/tests/utils/aws_parameters_store.py,sha256=AUAJkK9XBx4IPfV8Gr8fn00yERrQFMjm7-2CXldOKU0,22734
+codemie_test_harness/tests/utils/aws_parameters_store.py,sha256=1iwqGN5DbUTrJcCmDM2wLnNkak59SjxWyYFFX-gSBAk,23337
 codemie_test_harness/tests/utils/base_utils.py,sha256=tfishCUxO3iEuasWOifoF9_fXspm4uIHS26Ryqu-NxA,5998
 codemie_test_harness/tests/utils/client_factory.py,sha256=Yyg2iXe7g7fIfIUbOH8z_8qgVo_lB-nYLOfCV5ONXFw,641
 codemie_test_harness/tests/utils/constants.py,sha256=ZNyM5wERHFN-c8TCvBcxCdFf0As9TOpr22YvLCMHArE,1116
@@ -347,7 +348,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
 codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=vq6tucNBxiNIQSmIj_pYiiPm0lipU9X3kzeCd6xEbRM,966
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=uD2qs361j6Egp4UumfoQ4gC24-NioXfiW0IF53N9hVA,1175
-codemie_test_harness-0.1.160.dist-info/METADATA,sha256=aWQcHiEkFpoo7Fd1zEJOjpSIS-enVOPlQmrG49ZRj7c,8998
-codemie_test_harness-0.1.160.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-codemie_test_harness-0.1.160.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
-codemie_test_harness-0.1.160.dist-info/RECORD,,
+codemie_test_harness-0.1.162.dist-info/METADATA,sha256=P1Q4a4YnD6COFOY-PfS_-GBgT9mPzD-w_9kIBheM_gQ,8998
+codemie_test_harness-0.1.162.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+codemie_test_harness-0.1.162.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
+codemie_test_harness-0.1.162.dist-info/RECORD,,