codemie-test-harness 0.1.160__py3-none-any.whl → 0.1.161__py3-none-any.whl

codemie_test_harness/tests/llm/assistants/test_lite_llm.py

@@ -0,0 +1,96 @@
+import pytest
+from codemie_sdk.models.integration import CredentialTypes
+from hamcrest import assert_that, has_item
+
+from codemie_test_harness.tests.enums.model_types import ModelTypes
+from codemie_test_harness.tests.test_data.llm_test_data import MODEL_RESPONSES
+from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.utils.client_factory import get_client
+from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.env_resolver import get_environment
+from codemie_test_harness.tests.utils.pytest_utils import check_mark
+
+
+@pytest.fixture(scope="module")
+def lite_llm_integration(integration_utils):
+    credential_values = CredentialsUtil.lite_llm_credentials()
+    integration = integration_utils.create_user_integration(
+        CredentialTypes.LITE_LLM,
+        credential_values,
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
+@pytest.fixture(scope="function")
+def invalid_lite_llm_integration(integration_utils):
+    credential_values = CredentialsUtil.invalid_lite_llm_credentials()
+    integration = integration_utils.create_user_integration(
+        CredentialTypes.LITE_LLM,
+        credential_values,
+        project_name=test_project_name,
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
+def pytest_generate_tests(metafunc):
+    if "model_type" in metafunc.fixturenames:
+        is_smoke = check_mark(metafunc, "smoke")
+        test_data = []
+        env = get_environment()
+        if is_smoke:
+            available_models = get_client().llms.list()
+            for model in available_models:
+                test_data.append(pytest.param(model.base_name))
+        else:
+            for model_data in MODEL_RESPONSES:
+                test_data.append(
+                    pytest.param(
+                        model_data.model_type,
+                        marks=pytest.mark.skipif(
+                            env not in model_data.environments,
+                            reason=f"Skip on non {'/'.join(str(env) for env in model_data.environments[:-1])} envs",
+                        ),
+                    )
+                )
+
+        metafunc.parametrize("model_type", test_data)
+
+
+@pytest.mark.assistant
+@pytest.mark.lite_llm
+@pytest.mark.regression
+def test_assistant_with_different_models_in_lite_llm(
+    llm_utils,
+    lite_llm_integration,
+    assistant_utils,
+    model_type,
+    similarity_check,
+):
+    assert_that(
+        [row.base_name for row in llm_utils.list_llm_models()],
+        has_item(model_type),
+        f"{model_type} is missing in backend response",
+    )
+    assistant = assistant_utils.create_assistant(model_type)
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+
+    if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
+        response = "\n".join(response.split("\n")[-3:])
+    similarity_check.check_similarity(response, "Hello")
+
+
+@pytest.mark.assistant
+@pytest.mark.lite_llm
+@pytest.mark.regression
+def test_assistant_with_invalid_lite_llm(
+    invalid_lite_llm_integration,
+    assistant,
+    assistant_utils,
+):
+    assistant = assistant(project_name=test_project_name)
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    assert_that(response.startswith("AI Agent run failed with error: Error code: 401"))

codemie_test_harness/tests/llm/assistants/test_llm.py

@@ -36,10 +36,10 @@ def pytest_generate_tests(metafunc):
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -57,10 +57,10 @@ def test_assistant_with_different_models(
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models_with_top_p_parameter(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -78,10 +78,10 @@ def test_assistant_with_different_models_with_top_p_parameter(
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models_with_temperature_parameter(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )

codemie_test_harness/tests/utils/aws_parameters_store.py

@@ -623,6 +623,24 @@ class CredentialsUtil:
                 cred.value = "wrong_key"
         return credentials
 
+    @staticmethod
+    def lite_llm_credentials() -> List[CredentialValues]:
+        lite_llm_creds = AwsParameterStore.get_cloud_provider_credentials("litellm")
+        return [
+            CredentialValues(
+                key="api_key",
+                value=lite_llm_creds.get("api_key"),
+            ),
+        ]
+
+    @staticmethod
+    def invalid_lite_llm_credentials() -> List[CredentialValues]:
+        credentials = CredentialsUtil.lite_llm_credentials()
+        for cred in credentials:
+            if cred.key == "api_key":
+                cred.value = "wrong_key"
+        return credentials
+
     @staticmethod
     def jira_cloud_jql() -> str:
         jira_creds = AwsParameterStore.get_cloud_provider_credentials("jira")

{codemie_test_harness-0.1.160.dist-info → codemie_test_harness-0.1.161.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.160
+Version: 0.1.161
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.160)
+Requires-Dist: codemie-sdk-python (==0.1.161)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)

{codemie_test_harness-0.1.160.dist-info → codemie_test_harness-0.1.161.dist-info}/RECORD

@@ -78,7 +78,8 @@ codemie_test_harness/tests/integrations/user/test_default_integrations.py,sha256
 codemie_test_harness/tests/integrations/user/test_user_integrations.py,sha256=lGOoyGyKby1vOvXbz9de-PfhS0eUEX6vS7tg6vixtEE,8120
 codemie_test_harness/tests/llm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/llm/assistants/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-codemie_test_harness/tests/llm/assistants/test_llm.py,sha256=06oFZikafBC8YrFxgm5RJqIw7or0h8CDWroZ-LvX1Dw,3495
+codemie_test_harness/tests/llm/assistants/test_lite_llm.py,sha256=-X6QMjFw1wuTWQ1vTLM-Ae_Ha9jLCCHlFY7LgUttqHM,3485
+codemie_test_harness/tests/llm/assistants/test_llm.py,sha256=rpS-rsRqR13HPKPDeT_PvgnlSg1_KtbXTAzQpfShL8E,3531
 codemie_test_harness/tests/providers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/providers/test_providers_endpoints.py,sha256=SfZVCaYcsk_mG2b9zVu7w2wOY54loQHdmRS8eK5PNo4,8120
 codemie_test_harness/tests/search/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
@@ -220,7 +221,7 @@ codemie_test_harness/tests/ui/test_workflow_templates.py,sha256=pVuF98d3eEfinb5j
 codemie_test_harness/tests/ui/test_workflows.py,sha256=a2VY8BAsw3Po7e10r4g7S798GhpA4KYh2ZjnYEhU1PY,3664
 codemie_test_harness/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/utils/assistant_utils.py,sha256=jy3dFF4ZiT22Wippl1a5jIEAAyJ71P0ss8AIHPefz6k,6511
-codemie_test_harness/tests/utils/aws_parameters_store.py,sha256=AUAJkK9XBx4IPfV8Gr8fn00yERrQFMjm7-2CXldOKU0,22734
+codemie_test_harness/tests/utils/aws_parameters_store.py,sha256=1iwqGN5DbUTrJcCmDM2wLnNkak59SjxWyYFFX-gSBAk,23337
 codemie_test_harness/tests/utils/base_utils.py,sha256=tfishCUxO3iEuasWOifoF9_fXspm4uIHS26Ryqu-NxA,5998
 codemie_test_harness/tests/utils/client_factory.py,sha256=Yyg2iXe7g7fIfIUbOH8z_8qgVo_lB-nYLOfCV5ONXFw,641
 codemie_test_harness/tests/utils/constants.py,sha256=ZNyM5wERHFN-c8TCvBcxCdFf0As9TOpr22YvLCMHArE,1116
@@ -347,7 +348,7 @@ codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/__init__.
 codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py,sha256=vq6tucNBxiNIQSmIj_pYiiPm0lipU9X3kzeCd6xEbRM,966
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py,sha256=uD2qs361j6Egp4UumfoQ4gC24-NioXfiW0IF53N9hVA,1175
-codemie_test_harness-0.1.160.dist-info/METADATA,sha256=aWQcHiEkFpoo7Fd1zEJOjpSIS-enVOPlQmrG49ZRj7c,8998
-codemie_test_harness-0.1.160.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
-codemie_test_harness-0.1.160.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
-codemie_test_harness-0.1.160.dist-info/RECORD,,
+codemie_test_harness-0.1.161.dist-info/METADATA,sha256=8CsQCb2vq2XdzkZC8ZpOW8-PMgPMrW6mdpYPWszjG_Q,8998
+codemie_test_harness-0.1.161.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+codemie_test_harness-0.1.161.dist-info/entry_points.txt,sha256=n98t-EOM5M1mnMl_j2X4siyeO9zr0WD9a5LF7JyElIM,73
+codemie_test_harness-0.1.161.dist-info/RECORD,,