codemie-test-harness 0.1.183__py3-none-any.whl → 0.1.197__py3-none-any.whl

codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py

@@ -12,7 +12,10 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     RESPONSE_FOR_JIRA_TOOL,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import test_project_name
 
 
@@ -77,6 +80,12 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=JIRA_TOOL_PROMPT
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
+
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -124,6 +133,12 @@ def test_assistant_in_workflow_with_global_and_project_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=JIRA_TOOL_PROMPT
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
+
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -162,6 +177,12 @@ def test_assistant_in_workflow_with_project_integration_only(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=JIRA_TOOL_PROMPT
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
+
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -208,6 +229,12 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=JIRA_TOOL_PROMPT
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
+
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -254,4 +281,10 @@ def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=JIRA_TOOL_PROMPT
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
+
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)

codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py

@@ -11,7 +11,10 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
     ado_wiki_get_test_data,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import test_project_name
 
 ado_wiki_prompt = ado_wiki_get_test_data[1][2]
@@ -81,6 +84,12 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=ado_wiki_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
+
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -129,6 +138,12 @@ def test_assistant_in_workflow_with_global_and_project_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=ado_wiki_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
+
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -169,6 +184,12 @@ def test_assistant_in_workflow_with_project_integration_only(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=ado_wiki_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
+
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -217,6 +238,12 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=ado_wiki_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
+
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -265,4 +292,10 @@ def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=ado_wiki_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
+
     similarity_check.check_similarity(response, ado_wiki_answer)

codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py

@@ -11,7 +11,10 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
     list_branches_set_active_branch_test_data,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import test_project_name
 
 git_tool_prompt = list_branches_set_active_branch_test_data[0][2]
@@ -92,6 +95,12 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=git_tool_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
+
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -155,6 +164,12 @@ def test_assistant_in_workflow_with_global_and_project_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=git_tool_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
+
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -205,6 +220,12 @@ def test_assistant_in_workflow_with_project_integration_only(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=git_tool_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
+
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -267,6 +288,12 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=git_tool_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
+
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -329,4 +356,10 @@ def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, user_input=git_tool_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
+
     similarity_check.check_similarity(response, git_tool_answer)

codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py

@@ -15,7 +15,10 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
     RESPONSE_FOR_DIFF_UPDATE,
     RESPONSE_FOR_FILE_EDITOR,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
@@ -48,6 +51,10 @@ def test_workflow_with_file_management_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)
 
@@ -73,6 +80,10 @@ def test_workflow_with_generate_image_tool(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(FileManagementTool.GENERATE_IMAGE, triggered_tools)
 
     assert_that(
         response.lower(),
@@ -96,25 +107,36 @@ def test_workflow_with_read_file_tool(
 ):
     assistant_and_state_name = get_random_name()
     # Step 1: Write file
+    write_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        FileManagementTool.WRITE_FILE,
+        integration=filesystem_integration,
+        task=WRITE_FILE_TASK,
+    )
     workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            FileManagementTool.WRITE_FILE,
-            integration=filesystem_integration,
-            task=WRITE_FILE_TASK,
-        ).id,
+        write_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        write_workflow
+    )
+    assert_tool_triggered(FileManagementTool.WRITE_FILE, triggered_tools)
+
     # Step 2: Read file
+    read_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        FileManagementTool.READ_FILE,
+        integration=filesystem_integration,
+        task=READ_FILE_TOOL_TASK,
+    )
     response = workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            FileManagementTool.READ_FILE,
-            integration=filesystem_integration,
-            task=READ_FILE_TOOL_TASK,
-        ).id,
+        read_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        read_workflow
+    )
+    assert_tool_triggered(FileManagementTool.READ_FILE, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_READ_FILE_TASK)
 
 
@@ -135,37 +157,52 @@ def test_workflow_with_file_editing_tool(
     tool_names = (FileManagementTool.WRITE_FILE, FileManagementTool.DIFF_UPDATE)
 
     # Create the file
+    create_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        tool_names,
+        integration=filesystem_integration,
+        task=create_file_task(file_to_update),
+    )
     workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            tool_names,
-            integration=filesystem_integration,
-            task=create_file_task(file_to_update),
-        ).id,
+        create_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        create_workflow
+    )
+    assert_tool_triggered(FileManagementTool.WRITE_FILE, triggered_tools)
 
     # Insert to the file
+    insert_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        tool_names,
+        integration=filesystem_integration,
+        task=insert_to_file_task(file_to_update),
+    )
     workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            tool_names,
-            integration=filesystem_integration,
-            task=insert_to_file_task(file_to_update),
-        ).id,
+        insert_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        insert_workflow
+    )
+    assert_tool_triggered(FileManagementTool.DIFF_UPDATE, triggered_tools)
 
     # Show the diff
+    diff_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        tool_names,
+        integration=filesystem_integration,
+        task=show_diff_task(file_to_update),
+    )
     response = workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            tool_names,
-            integration=filesystem_integration,
-            task=show_diff_task(file_to_update),
-        ).id,
+        diff_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        diff_workflow
+    )
+    assert_tool_triggered(FileManagementTool.DIFF_UPDATE, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_DIFF_UPDATE)
 
@@ -186,36 +223,51 @@ def test_workflow_with_filesystem_tool(
     file_to_update = f"sum_{get_random_name()}.py"
 
     # Create the file
+    create_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        FileManagementTool.FILESYSTEM_EDITOR,
+        integration=filesystem_integration,
+        task=create_file_task(file_to_update),
+    )
     workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            FileManagementTool.FILESYSTEM_EDITOR,
-            integration=filesystem_integration,
-            task=create_file_task(file_to_update),
-        ).id,
+        create_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        create_workflow
+    )
+    assert_tool_triggered(FileManagementTool.FILESYSTEM_EDITOR, triggered_tools)
 
     # Insert to the file
+    insert_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        FileManagementTool.FILESYSTEM_EDITOR,
+        integration=filesystem_integration,
+        task=insert_to_file_task(file_to_update),
+    )
     workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            FileManagementTool.FILESYSTEM_EDITOR,
-            integration=filesystem_integration,
-            task=insert_to_file_task(file_to_update),
-        ).id,
+        insert_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        insert_workflow
+    )
+    assert_tool_triggered(FileManagementTool.FILESYSTEM_EDITOR, triggered_tools)
 
     # Show the file content
+    show_workflow = workflow_with_virtual_assistant(
+        assistant_and_state_name,
+        FileManagementTool.FILESYSTEM_EDITOR,
+        integration=filesystem_integration,
+        task=show_file_task(file_to_update),
+    )
     response = workflow_utils.execute_workflow(
-        workflow_with_virtual_assistant(
-            assistant_and_state_name,
-            FileManagementTool.FILESYSTEM_EDITOR,
-            integration=filesystem_integration,
-            task=show_file_task(file_to_update),
-        ).id,
+        show_workflow.id,
         assistant_and_state_name,
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        show_workflow
+    )
+    assert_tool_triggered(FileManagementTool.FILESYSTEM_EDITOR, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_FILE_EDITOR)

codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py

@@ -10,7 +10,11 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
     get_merge_request_changes_test_data,
     update_file_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name, to_camel_case
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    to_camel_case,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.workflow
@@ -45,6 +49,10 @@ def test_workflow_with_list_branch_set_active_branch_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -85,6 +93,10 @@ def test_workflow_with_create_branch_tool(
         response = workflow_utils.execute_workflow(
             test_workflow.id, assistant_and_state_name
         )
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            test_workflow
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
 
         assert_that(
             git_utils.branch_exists(name),
@@ -139,6 +151,10 @@ def test_workflow_with_create_file_tool(
         response = workflow_utils.execute_workflow(
             test_workflow.id, assistant_and_state_name
         )
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            test_workflow
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
 
         file_content = git_utils.get_file_content(file_name, "main")
 
@@ -189,6 +205,10 @@ def test_workflow_with_create_merge_request_tool(
         response = workflow_utils.execute_workflow(
             test_workflow.id, assistant_and_state_name
         )
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            test_workflow
+        )
+        assert_tool_triggered(tool_name[0], triggered_tools)
         mr_id = git_utils.get_merge_request_id_by_title(name)
 
         expected = expected_template(source_branch, name, mr_id)
@@ -252,6 +272,10 @@ def test_workflow_with_delete_file_tool(
         response = workflow_utils.execute_workflow(
             test_workflow.id, assistant_and_state_name
         )
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            test_workflow
+        )
+        assert_tool_triggered(tool_name, triggered_tools)
 
         assert_that(
             git_utils.file_exists(file_name, "main"),
@@ -304,7 +328,11 @@ def test_workflow_with_get_merge_request_changes_tool(
 
         create_mr_prompt = create_mr_prompt_template(source_branch, class_name, mr_name)
 
-        response = assistant_utils.ask_assistant(assistant_instance, create_mr_prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant_instance, create_mr_prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name[:-1], triggered_tools)
+
         mr_id = git_utils.get_merge_request_id_by_title(mr_name)
 
         create_mr_expected = create_mr_expected_template(source_branch, mr_name, mr_id)
@@ -327,6 +355,10 @@ def test_workflow_with_get_merge_request_changes_tool(
         response = workflow_utils.execute_workflow(
             test_workflow.id, assistant_and_state_name
         )
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            test_workflow
+        )
+        assert_tool_triggered(tool_name[-1], triggered_tools)
 
         similarity_check.check_similarity(response, get_changes_expected)
     finally:
@@ -373,7 +405,10 @@ def test_workflow_with_update_file_tools(
         )
 
         create_prompt = create_prompt_template(class_name)
-        response = assistant_utils.ask_assistant(assistant_instance, create_prompt)
+        response, triggered_tools = assistant_utils.ask_assistant(
+            assistant_instance, create_prompt, minimal_response=False
+        )
+        assert_tool_triggered(tool_name[0], triggered_tools)
 
         create_expected = create_expected_template(class_name)
         similarity_check.check_similarity(response, create_expected)
@@ -400,6 +435,10 @@ def test_workflow_with_update_file_tools(
         response = workflow_utils.execute_workflow(
             test_workflow.id, assistant_and_state_name
         )
+        triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+            test_workflow
+        )
+        assert_tool_triggered(tool_name[1], triggered_tools)
 
         similarity_check.check_similarity(response, update_expected)
 

codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py

@@ -1,5 +1,10 @@
 import pytest
 
+from codemie_test_harness.tests.enums.tools import (
+    CliMcpServer,
+    McpServerFetch,
+    McpServerTime,
+)
 from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     FETCH_MCP_SERVER,
     fetch_expected_response,
@@ -12,11 +17,10 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
     cli_mcp_server_test_data,
     CLI_MCP_SERVER,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
-
-# pytestmark = pytest.mark.skipif(
-#     EnvironmentResolver.is_localhost(), reason="Skipping this test on local environment"
-# )
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.workflow
@@ -42,6 +46,12 @@ def test_workflow_with_time_mcp_server(
         assistant_and_state_name,
         time_server_prompt,
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(McpServerTime.CONVERT_TIME, triggered_tools)
+
     similarity_check.check_similarity(response, time_expected_response)
 
 
@@ -74,6 +84,12 @@ def test_workflow_with_cli_mcp_server(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, f"execute the command: '{command}'"
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
+
     similarity_check.check_similarity(response, expected_answer)
 
 
@@ -96,4 +112,10 @@ def test_workflow_with_fetch_mcp_server(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name, fetch_server_prompt
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(McpServerFetch.FETCH, triggered_tools)
+
     similarity_check.check_similarity(response, fetch_expected_response)

codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py

@@ -2,6 +2,7 @@ import pytest
 from hamcrest import assert_that, equal_to
 
 from codemie_test_harness.tests.enums.tools import NotificationTool
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 from codemie_test_harness.tests.test_data.notification_tools_test_data import (
     EMAIL_TOOL_PROMPT,
     EMAIL_RESPONSE,
@@ -42,6 +43,12 @@ def test_workflow_with_notification_email_tool(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(NotificationTool.EMAIL, triggered_tools)
+
     similarity_check.check_similarity(response, EMAIL_RESPONSE)
 
     message_data = gmail_message_operator.get_messages_list()
@@ -85,4 +92,10 @@ def test_workflow_with_notification_telegram_tool(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(NotificationTool.TELEGRAM, triggered_tools)
+
     similarity_check.check_similarity(response, TELEGRAM_RESPONSE)

codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py

@@ -3,7 +3,10 @@ import pytest
 from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
     open_api_tools_test_data,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
 
 
@@ -40,4 +43,10 @@ def test_workflow_with_open_api_tools(
     response = workflow_utils.execute_workflow(
         test_workflow.id, assistant_and_state_name
     )
+
+    triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
+        test_workflow
+    )
+    assert_tool_triggered(tool_name, triggered_tools)
+
     similarity_check.check_similarity(response, expected_response)