codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.198__py3-none-any.whl

codemie_test_harness/tests/integrations/user/test_default_integrations.py

@@ -37,15 +37,9 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
 from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
     rp_test_data,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.enums.environment import Environment
 from codemie_test_harness.tests.utils.constants import test_project_name
-from codemie_test_harness.tests.utils.env_resolver import get_environment
-
-pytestmark = pytest.mark.skipif(
-    get_environment() in [Environment.LOCALHOST, Environment.GCP],
-    reason="Skipping this test on local environment",
-)
 
 
 @pytest.mark.assistant
@@ -87,7 +81,11 @@ def test_assistant_with_default_integration_cloud(
     # create an assistant
     cloud_assistant = assistant(toolkit, tool_name, project_name=test_project_name)
 
-    response = assistant_utils.ask_assistant(cloud_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        cloud_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -126,7 +124,11 @@ def test_assistant_with_default_integration_ado(
     # create an assistant
     ado_assistant = assistant(toolkit, tool_name, project_name=test_project_name)
 
-    response = assistant_utils.ask_assistant(ado_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -170,7 +172,11 @@ def test_assistant_with_default_integration_codebase(
         toolkit, CodeBaseTool.SONAR, project_name=test_project_name
     )
 
-    response = assistant_utils.ask_assistant(sonar_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        sonar_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -223,7 +229,11 @@ def test_assistant_with_default_integration_git(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -262,7 +272,11 @@ def test_assistant_with_default_integration_jira(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
@@ -299,7 +313,11 @@ def test_assistant_with_default_integration_email(
         Toolkit.NOTIFICATION, NotificationTool.EMAIL, project_name=test_project_name
     )
 
-    response = assistant_utils.ask_assistant(email_assistant, EMAIL_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        email_assistant, EMAIL_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(NotificationTool.EMAIL, triggered_tools)
 
     similarity_check.check_similarity(response, EMAIL_RESPONSE)
 
@@ -336,7 +354,11 @@ def test_assistant_with_default_integration_keycloak(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        keycloak_assistant, KEYCLOAK_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
 
@@ -376,6 +398,10 @@ def test_assistant_with_default_integration_report_portal(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(report_portal_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        report_portal_assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(ReportPortalTool.GET_DASHBOARD_DATA, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/llm/assistants/test_llm.py

@@ -1,6 +1,8 @@
 import pytest
+from codemie_sdk.models.assistant import ToolKitDetails, ToolDetails
 from hamcrest import assert_that, has_item
 from codemie_test_harness.tests.enums.model_types import ModelTypes
+from codemie_test_harness.tests.enums.tools import Toolkit, FileManagementTool
 from codemie_test_harness.tests.test_data.llm_test_data import MODEL_RESPONSES
 from codemie_test_harness.tests.utils.client_factory import get_client
 from codemie_test_harness.tests.utils.env_resolver import get_environment
@@ -36,14 +38,25 @@ def pytest_generate_tests(metafunc):
 @pytest.mark.api
 @pytest.mark.smoke
 def test_assistant_with_different_models(
-    llm_utils, assistant_utils, model_type, similarity_check
+    llm_utils, assistant_utils, model_type, similarity_check, filesystem_integration
 ):
     assert_that(
         [row.base_name for row in llm_utils.list_llm_models()],
         has_item(model_type),
         f"{model_type} is missing in backend response",
     )
-    assistant = assistant_utils.create_assistant(model_type)
+
+    tool = ToolKitDetails(
+        toolkit=Toolkit.FILE_MANAGEMENT,
+        tools=[
+            ToolDetails(
+                name=FileManagementTool.GENERATE_IMAGE, settings=filesystem_integration
+            )
+        ],
+        settings=filesystem_integration,
+    )
+
+    assistant = assistant_utils.create_assistant(model_type, toolkits=[tool])
     response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
 
     if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
@@ -91,3 +104,33 @@ def test_assistant_with_different_models_with_temperature_parameter(
     if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
         response = "\n".join(response.split("\n")[-3:])
     similarity_check.check_similarity(response, "Hello")
+
+
+@pytest.mark.assistant
+@pytest.mark.llm
+@pytest.mark.api
+@pytest.mark.smoke
+def test_assistant_with_different_models_with_datasource_attached(
+    llm_utils,
+    assistant_utils,
+    model_type,
+    similarity_check,
+    datasource_utils,
+    default_embedding_llm,
+    kb_context,
+    file_datasource,
+):
+    assert_that(
+        [row.base_name for row in llm_utils.list_llm_models()],
+        has_item(model_type),
+        f"{model_type} is missing in backend response",
+    )
+
+    assistant = assistant_utils.create_assistant(
+        model_type, context=[kb_context(file_datasource)]
+    )
+    response = assistant_utils.ask_assistant(assistant, "Just say one word: 'Hello'")
+
+    if model_type in [ModelTypes.DEEPSEEK_R1, ModelTypes.RLAB_QWQ_32B]:
+        response = "\n".join(response.split("\n")[-3:])
+    similarity_check.check_similarity(response, "Hello")