codemie-test-harness 0.1.160__tar.gz → 0.1.161__tar.gz

{codemie_test_harness-0.1.160 → codemie_test_harness-0.1.161}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: codemie-test-harness
-Version: 0.1.160
+Version: 0.1.161
 Summary: Autotest for CodeMie backend and UI
 Author: Anton Yeromin
 Author-email: anton_yeromin@epam.com
@@ -13,7 +13,7 @@ Requires-Dist: aws-assume-role-lib (>=2.10.0,<3.0.0)
 Requires-Dist: boto3 (>=1.39.8,<2.0.0)
 Requires-Dist: click (>=8.1.7,<9.0.0)
 Requires-Dist: codemie-plugins (>=0.1.123,<0.2.0)
-Requires-Dist: codemie-sdk-python (==0.1.160)
+Requires-Dist: codemie-sdk-python (==0.1.161)
 Requires-Dist: pytest (>=8.4.1,<9.0.0)
 Requires-Dist: pytest-playwright (>=0.7.0,<0.8.0)
 Requires-Dist: pytest-reportportal (>=5.5.2,<6.0.0)

codemie_test_harness-0.1.161/codemie_test_harness/tests/llm/assistants/test_lite_llm.py

@@ -0,0 +1,96 @@
+import pytest
+from codemie_sdk.models.integration import CredentialTypes
+from hamcrest import assert_that, has_item
+
+from codemie_test_harness.tests.enums.model_types import ModelTypes
+from codemie_test_harness.tests.test_data.llm_test_data import MODEL_RESPONSES
+from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
+from codemie_test_harness.tests.utils.client_factory import get_client
+from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.env_resolver import get_environment
+from codemie_test_harness.tests.utils.pytest_utils import check_mark
+
+
+@pytest.fixture(scope="module")
+def lite_llm_integration(integration_utils):
+    credential_values = CredentialsUtil.lite_llm_credentials()
+    integration = integration_utils.create_user_integration(
+        CredentialTypes.LITE_LLM,
+        credential_values,
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
+@pytest.fixture(scope="function")
+def invalid_lite_llm_integration(integration_utils):
+    credential_values = CredentialsUtil.invalid_lite_llm_credentials()
+    integration = integration_utils.create_user_integration(
+        CredentialTypes.LITE_LLM,
+        credential_values,
+        project_name=test_project_name,
+    )
+    yield integration
+    if integration:
+        integration_utils.delete_integration(integration)
+
+
+def pytest_generate_tests(metafunc):
+    if "model_type" in metafunc.fixturenames:
+        is_smoke = check_mark(metafunc, "smoke")
+        test_data = []
+        env = get_environment()
+        if is_smoke:
+            available_models = get_client().llms.list()
+            for model in available_models:
+                test_data.append(pytest.param(model.base_name))
+        else:
+            for model_data in MODEL_RESPONSES:
+                test_data.append(
+                    pytest.param(
+                        model_data.model_type,
+                        marks=pytest.mark.skipif(
+                            env not in model_data.environments,
+                            reason=f"Skip on non {'/'.join(str(env) for env in model_data.environments[:-1])} envs",
+                        ),
+                    )
+                )
+
+        metafunc.parametrize("model_type", test_data)
+
+
+@pytest.mark.assistant
+@pytest.mark.lite_llm
+@pytest.mark.regression
+def test_assistant_with_different_models_in_lite_llm(
+    llm_utils,
+    lite_llm_integration,
+    assistant_utils,
+    model_type,
+    similarity_check,
+):
+    assert_that(
+        [row.base_name for row in llm_utils.list_llm_models()],
+        has_item(model_type),
+        f"{model_type} is missing in backend response",
+    )
+    assistant = assistant_utils.create_assistant(model_type)
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+
+    if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
+        response = "\n".join(response.split("\n")[-3:])
+    similarity_check.check_similarity(response, "Hello")
+
+
+@pytest.mark.assistant
+@pytest.mark.lite_llm
+@pytest.mark.regression
+def test_assistant_with_invalid_lite_llm(
+    invalid_lite_llm_integration,
+    assistant,
+    assistant_utils,
+):
+    assistant = assistant(project_name=test_project_name)
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+    assert_that(response.startswith("AI Agent run failed with error: Error code: 401"))

{codemie_test_harness-0.1.160 → codemie_test_harness-0.1.161}/codemie_test_harness/tests/llm/assistants/test_llm.py

@@ -36,10 +36,10 @@ def pytest_generate_tests(metafunc):
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -57,10 +57,10 @@ def test_assistant_with_different_models(
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models_with_top_p_parameter(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
@@ -78,10 +78,10 @@ def test_assistant_with_different_models_with_top_p_parameter(
 @pytest.mark.regression
 @pytest.mark.smoke
 def test_assistant_with_different_models_with_temperature_parameter(
-    client, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check
 ):
     assert_that(
-        [row.base_name for row in client.llms.list()],
+        [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )

{codemie_test_harness-0.1.160 → codemie_test_harness-0.1.161}/codemie_test_harness/tests/utils/aws_parameters_store.py

@@ -623,6 +623,24 @@ class CredentialsUtil:
                 cred.value = "wrong_key"
         return credentials
 
+    @staticmethod
+    def lite_llm_credentials() -> List[CredentialValues]:
+        lite_llm_creds = AwsParameterStore.get_cloud_provider_credentials("litellm")
+        return [
+            CredentialValues(
+                key="api_key",
+                value=lite_llm_creds.get("api_key"),
+            ),
+        ]
+
+    @staticmethod
+    def invalid_lite_llm_credentials() -> List[CredentialValues]:
+        credentials = CredentialsUtil.lite_llm_credentials()
+        for cred in credentials:
+            if cred.key == "api_key":
+                cred.value = "wrong_key"
+        return credentials
+
     @staticmethod
     def jira_cloud_jql() -> str:
         jira_creds = AwsParameterStore.get_cloud_provider_credentials("jira")

{codemie_test_harness-0.1.160 → codemie_test_harness-0.1.161}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemie-test-harness"
-version = "0.1.160"
+version = "0.1.161"
 description = "Autotest for CodeMie backend and UI"
 authors = [
     "Anton Yeromin <anton_yeromin@epam.com>",
@@ -12,7 +12,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "^3.12"
-codemie-sdk-python = "0.1.160"
+codemie-sdk-python = "0.1.161"
 pytest = "^8.4.1"
 python-dotenv = "^1.1.0"
 PyHamcrest = "^2.1.0"