codemie-test-harness 0.1.179__py3-none-any.whl → 0.1.181__py3-none-any.whl

codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py

@@ -19,6 +19,7 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 pytestmark = pytest.mark.skipif(
     EnvironmentResolver.is_localhost(),
@@ -28,7 +29,7 @@ pytestmark = pytest.mark.skipif(
 
 @pytest.mark.assistant
 @pytest.mark.elastic
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6132")
 def test_create_assistant_with_elastic_tool(
     integration_utils, assistant, assistant_utils, similarity_check
@@ -43,14 +44,17 @@ def test_create_assistant_with_elastic_tool(
         settings=settings,
     )
 
-    response = assistant_utils.ask_assistant(assistant, ELASTIC_TOOL_TASK)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, ELASTIC_TOOL_TASK, minimal_response=False
+    )
 
+    assert_tool_triggered(DataManagementTool.ELASTIC, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_ELASTIC)
 
 
 @pytest.mark.assistant
 @pytest.mark.sql
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6132")
 @pytest.mark.parametrize(
     "db_dialect",
@@ -71,19 +75,36 @@ def test_create_assistant_with_sql_tool(
 
     conversation_id = str(uuid.uuid4())
 
-    assistant_utils.ask_assistant(
-        assistant, SQL_TOOL_CREATE_TABLE_TASK, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant,
+        SQL_TOOL_CREATE_TABLE_TASK,
+        conversation_id=conversation_id,
+        minimal_response=False,
     )
-    assistant_utils.ask_assistant(
-        assistant, SQL_TOOL_INSERT_TABLE_TASK, conversation_id=conversation_id
+    assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
+
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant,
+        SQL_TOOL_INSERT_TABLE_TASK,
+        conversation_id=conversation_id,
+        minimal_response=False,
     )
+    assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
 
-    response = assistant_utils.ask_assistant(
-        assistant, SQL_TOOL_QUERY_TABLE_TASK, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant,
+        SQL_TOOL_QUERY_TABLE_TASK,
+        conversation_id=conversation_id,
+        minimal_response=False,
     )
+    assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
 
-    assistant_utils.ask_assistant(
-        assistant, SQL_TOOL_DELETE_TABLE_TASK, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant,
+        SQL_TOOL_DELETE_TABLE_TASK,
+        conversation_id=conversation_id,
+        minimal_response=False,
     )
+    assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_SQL)

codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py

@@ -14,13 +14,16 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
     file_editing_tools_test_data,
     show_diff_task,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
 @pytest.mark.assistant
 @pytest.mark.file_management
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6103")
 @pytest.mark.parametrize(
     "tool_name,prompt,expected_response",
@@ -39,14 +42,17 @@ def test_create_assistant_with_file_management_tool(
         Toolkit.FILE_MANAGEMENT, tool_name, settings=filesystem_integration
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.file_management
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6103")
 @pytest.mark.skipif(
     EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
@@ -60,7 +66,11 @@ def test_create_assistant_with_file_management_generate_image_tool(
         settings=filesystem_integration,
     )
 
-    response = assistant_utils.ask_assistant(assistant, GENERATE_IMAGE_TOOL_TASK)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, GENERATE_IMAGE_TOOL_TASK, minimal_response=False
+    )
+
+    assert_tool_triggered(FileManagementTool.GENERATE_IMAGE, triggered_tools)
 
     assert_that(
         response.lower(),
@@ -73,7 +83,7 @@ def test_create_assistant_with_file_management_generate_image_tool(
 
 @pytest.mark.assistant
 @pytest.mark.file_management
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6103")
 def test_create_assistant_with_file_management_read_file_tool(
     filesystem_integration, assistant, assistant_utils, similarity_check
@@ -84,9 +94,15 @@ def test_create_assistant_with_file_management_read_file_tool(
         settings=filesystem_integration,
     )
 
-    assistant_utils.ask_assistant(assistant, WRITE_FILE_TASK)
+    _, write_triggered_tools = assistant_utils.ask_assistant(
+        assistant, WRITE_FILE_TASK, minimal_response=False
+    )
+    assert_tool_triggered(FileManagementTool.WRITE_FILE, write_triggered_tools)
 
-    response = assistant_utils.ask_assistant(assistant, READ_FILE_TOOL_TASK)
+    response, read_triggered_tools = assistant_utils.ask_assistant(
+        assistant, READ_FILE_TOOL_TASK, minimal_response=False
+    )
+    assert_tool_triggered(FileManagementTool.READ_FILE, read_triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_READ_FILE_TASK)
 
@@ -94,7 +110,7 @@ def test_create_assistant_with_file_management_read_file_tool(
 @pytest.mark.skip(reason="Tests are flaky, tools work unstable")
 @pytest.mark.assistant
 @pytest.mark.file_management
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "tool_name,expected_response",
     file_editing_tools_test_data,
@@ -115,9 +131,15 @@ def test_create_assistant_with_file_management_file_editing_tool(
         settings=filesystem_integration,
     )
     file_to_update = f"sum_{get_random_name()}.py"
-    assistant_utils.ask_assistant(assistant, create_file_task(file_to_update))
+    _, create_triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_file_task(file_to_update), minimal_response=False
+    )
+    assert_tool_triggered(tool_name, create_triggered_tools)
 
-    assistant_utils.ask_assistant(assistant, insert_to_file_task(file_to_update))
+    _, insert_triggered_tools = assistant_utils.ask_assistant(
+        assistant, insert_to_file_task(file_to_update), minimal_response=False
+    )
+    assert_tool_triggered(tool_name, insert_triggered_tools)
 
     prompt = (
         show_file_task(file_to_update)
@@ -125,6 +147,9 @@ def test_create_assistant_with_file_management_file_editing_tool(
         else show_diff_task(file_to_update)
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/assistant/tools/git/test_assistant_with_git_tools.py

@@ -10,12 +10,16 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
     get_merge_request_changes_test_data,
     update_file_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name, to_camel_case
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    to_camel_case,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
     list_branches_set_active_branch_test_data,
@@ -40,13 +44,16 @@ def test_assistant_with_list_branch_set_active_branch_tools(
         context=code_context(code_datasource),
         settings=git_integration,
     )
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template",
     create_branch_test_data,
@@ -76,7 +83,10 @@ def test_assistant_with_create_branch_tool(
             context=code_context(code_datasource),
             settings=git_integration,
         )
-        response = assistant_utils.ask_assistant(assistant, prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant, prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
 
         assert_that(
             git_utils.branch_exists(branch_name),
@@ -92,7 +102,7 @@ def test_assistant_with_create_branch_tool(
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template,expected_content_template",
     create_file_test_data,
@@ -126,7 +136,10 @@ def test_assistant_with_create_file_tool(
             context=code_context(code_datasource),
             settings=git_integration,
         )
-        response = assistant_utils.ask_assistant(assistant, prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant, prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
 
         file_content = git_utils.get_file_content(file_name, "main")
 
@@ -140,7 +153,7 @@ def test_assistant_with_create_file_tool(
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template",
     create_merge_request_test_data,
@@ -172,7 +185,10 @@ def test_assistant_with_create_merge_request_tool(
             context=code_context(code_datasource),
             settings=git_integration,
         )
-        response = assistant_utils.ask_assistant(assistant, prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant, prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
         mr_id = git_utils.get_merge_request_id_by_title(mr_name)
 
         expected = expected_template(source_branch, mr_name, mr_id)
@@ -184,7 +200,7 @@ def test_assistant_with_create_merge_request_tool(
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template,file_content_template",
     delete_file_test_data,
@@ -231,7 +247,10 @@ def test_assistant_with_delete_file_tool(
             context=code_context(code_datasource),
             settings=git_integration,
         )
-        response = assistant_utils.ask_assistant(assistant, prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant, prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
 
         assert_that(
             git_utils.file_exists(file_name, "main"),
@@ -247,7 +266,7 @@ def test_assistant_with_delete_file_tool(
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,create_mr_prompt_template,create_mr_expected_template,get_mr_changes_prompt_template,get_mr_changes_expected_template",
     get_merge_request_changes_test_data,
@@ -282,7 +301,10 @@ def test_assistant_with_get_merge_request_changes_tool(
 
         create_mr_prompt = create_mr_prompt_template(source_branch, class_name, mr_name)
 
-        response = assistant_utils.ask_assistant(assistant, create_mr_prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant, create_mr_prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name[:-1], triggered_tools)
         mr_id = git_utils.get_merge_request_id_by_title(mr_name)
 
         create_mr_expected = create_mr_expected_template(source_branch, mr_name, mr_id)
@@ -295,7 +317,10 @@ def test_assistant_with_get_merge_request_changes_tool(
         get_changes_prompt = get_mr_changes_prompt_template(mr_id)
         get_changes_expected = get_mr_changes_expected_template(class_name)
 
-        response = assistant_utils.ask_assistant(assistant, get_changes_prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant, get_changes_prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name[-1], triggered_tools)
         similarity_check.check_similarity(response, get_changes_expected)
     finally:
         if git_utils.branch_exists(source_branch):
@@ -304,7 +329,7 @@ def test_assistant_with_get_merge_request_changes_tool(
 
 @pytest.mark.assistant
 @pytest.mark.gitlab
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,create_prompt_template,create_expected_template,created_content_template,update_prompt_template,update_expected_template,updated_content_template",
     update_file_test_data,
@@ -340,7 +365,10 @@ def test_assistant_with_update_file_tools(
         )
 
         create_prompt = create_prompt_template(class_name)
-        response = assistant_utils.ask_assistant(assistant, create_prompt)
+        response, create_triggered_tools = assistant_utils.ask_assistant(
+            assistant, create_prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name[0], create_triggered_tools)
 
         create_expected = create_expected_template(class_name)
         similarity_check.check_similarity(response, create_expected)
@@ -355,7 +383,10 @@ def test_assistant_with_update_file_tools(
         )
 
         update_prompt = update_prompt_template(class_name)
-        response = assistant_utils.ask_assistant(assistant, update_prompt)
+        response, update_triggered_tools = assistant_utils.ask_assistant(
+            assistant, update_prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name[1], update_triggered_tools)
 
         update_expected = update_expected_template(class_name)
         similarity_check.check_similarity(response, update_expected)

codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py

@@ -7,21 +7,24 @@ from hamcrest import assert_that, contains_string
 from codemie_sdk.models.integration import CredentialTypes
 from codemie_test_harness.tests import autotest_entity_prefix
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import CliMcpServer
 from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     cli_mcp_server_test_data,
     CLI_MCP_SERVER,
 )
 
-pytestmark = pytest.mark.skipif(
-    EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
-)
+# pytestmark = pytest.mark.skipif(
+#     EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
+# )
 
 
 @pytest.mark.assistant
 @pytest.mark.mcp
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "command,expected_answer",
     cli_mcp_server_test_data,
@@ -31,17 +34,19 @@ def test_cli_mcp_server(
     assistant_utils, assistant, similarity_check, command, expected_answer
 ):
     assistant = assistant(mcp_server=CLI_MCP_SERVER)
-    response = assistant_utils.ask_assistant(
+    response, triggered_tools = assistant_utils.ask_assistant(
         assistant,
         f"execute command: '{command}'. In case of error just explain the issue and do not suggest "
         "any workarounds and do not try to run command with other parameters.",
+        minimal_response=False,
     )
+    assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
     similarity_check.check_similarity(response, expected_answer)
 
 
 @pytest.mark.assistant
 @pytest.mark.mcp
-@pytest.mark.regression
+@pytest.mark.api
 def test_env_var_in_mcp_server(
     assistant_utils, assistant, similarity_check, integration_utils
 ):
@@ -64,10 +69,12 @@ def test_env_var_in_mcp_server(
     assistant = assistant(mcp_server=cli_mcp_server_with_integration)
 
     dir_name = f"{autotest_entity_prefix}{get_random_name()}"
-    response = assistant_utils.ask_assistant(
+    response, triggered_tools = assistant_utils.ask_assistant(
         assistant,
         f"Execute commands sequentially: 'mkdir {dir_name}' then 'ls'. In the end return output of the second command.",
+        minimal_response=False,
     )
+    assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
     assert_that(
         response,
         contains_string(dir_name),

codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py

@@ -1,4 +1,5 @@
-from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
+from codemie_test_harness.tests.enums.tools import McpServerTime, McpServerFetch
 
 import pytest
 
@@ -11,14 +12,14 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     fetch_server_prompt,
 )
 
-pytestmark = pytest.mark.skipif(
-    EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
-)
+# pytestmark = pytest.mark.skipif(
+#     EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
+# )
 
 
 @pytest.mark.assistant
 @pytest.mark.mcp
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "mcp_server",
     time_mcp_server_test_data,
@@ -31,20 +32,25 @@ def test_creation_mcp_server_with_form_configuration(
 ):
     assistant = assistant(mcp_server=mcp_server)
 
-    response = assistant_utils.ask_assistant(
+    response, triggered_tools = assistant_utils.ask_assistant(
         assistant,
         time_server_prompt,
+        minimal_response=False,
     )
 
+    assert_tool_triggered(McpServerTime.CONVERT_TIME, triggered_tools)
     similarity_check.check_similarity(response, time_expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.mcp
-@pytest.mark.regression
+@pytest.mark.api
 def test_fetch_mcp_server(assistant_utils, assistant, similarity_check):
     assistant = assistant(mcp_server=FETCH_MCP_SERVER)
 
-    response = assistant_utils.ask_assistant(assistant, fetch_server_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, fetch_server_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(McpServerFetch.FETCH, triggered_tools)
     similarity_check.check_similarity(response, fetch_expected_response)

codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py

@@ -6,6 +6,7 @@ from codemie_test_harness.tests.enums.tools import Toolkit, NotificationTool
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.enums.environment import Environment
 from codemie_test_harness.tests.utils.env_resolver import get_environment
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -19,7 +20,7 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
 @pytest.mark.assistant
 @pytest.mark.notification
 @pytest.mark.email
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.skipif(
     get_environment() in [Environment.LOCALHOST, Environment.GCP],
     reason="Skipping this test on local environment",
@@ -40,7 +41,10 @@ def test_assistant_with_email_tool(
         Toolkit.NOTIFICATION, NotificationTool.EMAIL, settings=settings
     )
 
-    response = assistant_utils.ask_assistant(assistant, EMAIL_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, EMAIL_TOOL_PROMPT, minimal_response=False
+    )
+    assert_tool_triggered(NotificationTool.EMAIL, triggered_tools)
     similarity_check.check_similarity(response, EMAIL_RESPONSE)
 
     message_data = gmail_message_operator.get_messages_list()
@@ -65,7 +69,7 @@ def test_assistant_with_email_tool(
 @pytest.mark.assistant
 @pytest.mark.notification
 @pytest.mark.telegram
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_telegram_tool(
     assistant, assistant_utils, similarity_check, integration_utils
 ):
@@ -78,5 +82,8 @@ def test_assistant_with_telegram_tool(
         Toolkit.NOTIFICATION, NotificationTool.TELEGRAM, settings=settings
     )
 
-    response = assistant_utils.ask_assistant(assistant, TELEGRAM_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, TELEGRAM_TOOL_PROMPT, minimal_response=False
+    )
+    assert_tool_triggered(NotificationTool.TELEGRAM, triggered_tools)
     similarity_check.check_similarity(response, TELEGRAM_RESPONSE)

codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py

@@ -1,4 +1,5 @@
 from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 import pytest
 
@@ -10,7 +11,7 @@ from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
 
 @pytest.mark.assistant
 @pytest.mark.openapi
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.testcase("EPMCDME-6127")
 @pytest.mark.skipif(
     EnvironmentResolver.is_azure(),
@@ -34,6 +35,9 @@ def test_create_assistant_with_open_api_tool(
         Toolkit.OPEN_API, tool_name, settings=open_api_integration
     )
 
-    response = assistant_utils.ask_assistant(assistant_instance, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant_instance, prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_development_plugin.py

@@ -2,18 +2,21 @@ import os
 
 import pytest
 
-from codemie_test_harness.tests.enums.tools import Toolkit
+from codemie_test_harness.tests.enums.tools import Toolkit, PluginTool
 from codemie_test_harness.tests.test_data.plugin_tools_test_data import (
     list_files_plugin_tools_test_data,
     CREATE_READ_DELETE_FILE_TEST_DATA,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import TESTS_PATH
 
 
 @pytest.mark.assistant
 @pytest.mark.plugin
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "prompt,expected_response,tool_name",
     list_files_plugin_tools_test_data,
@@ -31,13 +34,18 @@ def test_assistant_with_list_files_plugin_tools(
 ):
     assistant = assistant(Toolkit.PLUGIN, Toolkit.PLUGIN, settings=development_plugin)
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.plugin
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.plugin
 def test_assistant_with_modify_files_plugin_tools(
     assistant_utils,
@@ -51,12 +59,16 @@ def test_assistant_with_modify_files_plugin_tools(
             Toolkit.PLUGIN, Toolkit.PLUGIN, settings=development_plugin
         )
 
-        response = assistant_utils.ask_assistant(
+        response, triggered_tools = assistant_utils.ask_assistant(
             assistant,
             CREATE_READ_DELETE_FILE_TEST_DATA["create_file_prompt"].format(
                 file_name, file_name
             ),
+            minimal_response=False,
         )
+
+        assert_tool_triggered(PluginTool.WRITE_FILE_TO_FILE_SYSTEM, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["create_file_response"].format(
@@ -64,21 +76,28 @@ def test_assistant_with_modify_files_plugin_tools(
             ),
         )
 
-        response = assistant_utils.ask_assistant(
+        response, triggered_tools = assistant_utils.ask_assistant(
             assistant,
             CREATE_READ_DELETE_FILE_TEST_DATA["git_command_prompt"].format(file_name),
+            minimal_response=False,
         )
+        assert_tool_triggered(PluginTool.GENERIC_GIT_TOOL, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["git_command_response"].format(file_name),
         )
 
-        response = assistant_utils.ask_assistant(
+        response, triggered_tools = assistant_utils.ask_assistant(
             assistant,
             CREATE_READ_DELETE_FILE_TEST_DATA["show_file_content_prompt"].format(
                 file_name
             ),
+            minimal_response=False,
         )
+
+        assert_tool_triggered(PluginTool.READ_FILE_FROM_FILE_SYSTEM, triggered_tools)
+
         similarity_check.check_similarity(
             response,
             CREATE_READ_DELETE_FILE_TEST_DATA["show_file_content_response"].format(