codemie-test-harness 0.1.179__py3-none-any.whl → 0.1.181__py3-none-any.whl

codemie_test_harness/tests/assistant/test_assistants.py

@@ -24,7 +24,11 @@ from codemie_test_harness.tests.test_data.file_test_data import (
     RESPONSE_FOR_TWO_FILES_UPLOADED,
 )
 from codemie_test_harness.tests.test_data.output_schema_test_data import output_schema
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.utils.constants import FILES_PATH
 from codemie_test_harness.tests.utils.file_utils import save_file
 
@@ -36,7 +40,7 @@ system_prompt_for_chatting_with_files = """Your task is to show the content of t
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_create_assistant(assistant_utils, default_llm):
     response = assistant_utils.send_create_assistant_request()
@@ -47,7 +51,7 @@ def test_create_assistant(assistant_utils, default_llm):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_all_assistants(search_utils):
     response = search_utils.list_assistants(filters=None)
@@ -57,7 +61,7 @@ def test_get_all_assistants(search_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_prebuilt_assistants(assistant_utils):
     response = assistant_utils.get_prebuilt_assistant()
@@ -67,7 +71,7 @@ def test_get_prebuilt_assistants(assistant_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_users_assistants(assistant_utils, search_utils):
     assistant_utils.send_create_assistant_request()
@@ -78,7 +82,7 @@ def test_get_users_assistants(assistant_utils, search_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_assistant_context(assistant_utils):
     response = assistant_utils.get_assistant_context(PROJECT)
@@ -87,7 +91,7 @@ def test_get_assistant_context(assistant_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_available_tools(assistant_utils):
     response = assistant_utils.get_assistant_tools()
@@ -96,7 +100,7 @@ def test_get_available_tools(assistant_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_assistant_by_id(assistant_utils):
     assistant_name = get_random_name()
@@ -112,7 +116,7 @@ def test_get_assistant_by_id(assistant_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_get_assistant_by_slug(assistant_utils):
     assistant_name = get_random_name()
@@ -128,7 +132,7 @@ def test_get_assistant_by_slug(assistant_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_update_assistant(assistant_utils, default_llm):
     assistant_name = get_random_name()
@@ -166,7 +170,7 @@ def test_update_assistant(assistant_utils, default_llm):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_delete_assistant(assistant_utils, search_utils):
     assistant_name = get_random_name()
@@ -184,7 +188,7 @@ def test_delete_assistant(assistant_utils, search_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_ask_assistant(assistant_utils):
     test_assistant = assistant_utils.create_assistant()
@@ -203,7 +207,7 @@ def test_ask_assistant(assistant_utils):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.skip(reason="Not implemented yet")
 def test_export_assistant(assistant_utils):
     assistant_name = get_random_name()
@@ -220,11 +224,11 @@ def test_export_assistant(assistant_utils):
 
 @pytest.mark.assistant
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
-@pytest.mark.tescase("EPMCDME-4001, EPMCDME-4002, EPMCDME-2527")
+@pytest.mark.testcase("EPMCDME-4001, EPMCDME-4002, EPMCDME-2527")
 @pytest.mark.parametrize(
-    "file_name,expected_response",
+    "file_name,expected_response,expected_tool",
     file_test_data,
     ids=[f"{row[0]}" for row in file_test_data],
 )
@@ -234,6 +238,7 @@ def test_create_assistant_and_prompt_with_file(
     similarity_check,
     file_name,
     expected_response,
+    expected_tool,
 ):
     prompt = (
         f"What is the content/text of the {file_name}. Show information from ALL pages. "
@@ -249,19 +254,25 @@ def test_create_assistant_and_prompt_with_file(
 
     conversation_id = str(uuid.uuid4())
 
-    response = assistant_utils.ask_assistant(
-        assistant, prompt, file_urls=[file_url], conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant,
+        prompt,
+        file_urls=[file_url],
+        conversation_id=conversation_id,
+        minimal_response=False,
     )
+    assert_tool_triggered(expected_tool, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
-    response = assistant_utils.ask_assistant(
-        assistant, prompt, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, conversation_id=conversation_id, minimal_response=False
     )
+    assert_tool_triggered(expected_tool, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_assistant_has_an_access_to_the_history(
     assistant_utils,
@@ -288,7 +299,7 @@ def test_assistant_has_an_access_to_the_history(
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_passing_history_to_chat(
     assistant_utils,
@@ -315,7 +326,7 @@ def test_passing_history_to_chat(
 
 @pytest.mark.assistant
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 @pytest.mark.parametrize(
     "file_name",
@@ -333,7 +344,7 @@ def test_upload_file_to_chat(
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_chat_with_pydantic_output_schema(assistant_utils, assistant):
     class OutputSchema(BaseModel):
@@ -347,7 +358,7 @@ def test_chat_with_pydantic_output_schema(assistant_utils, assistant):
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_chat_with_json_output_schema(assistant_utils, assistant):
     response = assistant_utils.ask_assistant(
@@ -359,7 +370,7 @@ def test_chat_with_json_output_schema(assistant_utils, assistant):
 
 @pytest.mark.assistant
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_create_assistant_and_prompt_with_multiple_files(
     assistant_utils,
@@ -382,23 +393,28 @@ def test_create_assistant_and_prompt_with_multiple_files(
 
     conversation_id = str(uuid.uuid4())
 
-    response = assistant_utils.ask_assistant(
+    response, triggered_tools = assistant_utils.ask_assistant(
         assistant,
         prompt,
         file_urls=[docx_file_url, ini_file_url],
         conversation_id=conversation_id,
+        minimal_response=False,
     )
+
+    assert_tool_triggered((Default.DOCX_TOOL, Default.FILE_ANALYSIS), triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_TWO_FILES_UPLOADED)
 
-    response = assistant_utils.ask_assistant(
-        assistant, prompt, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, conversation_id=conversation_id, minimal_response=False
     )
+
+    assert_tool_triggered((Default.DOCX_TOOL, Default.FILE_ANALYSIS), triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_TWO_FILES_UPLOADED)
 
 
 @pytest.mark.assistant
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 @pytest.mark.parametrize("prompt,expected_response", EXCEL_TOOL_TEST_DATA)
 def test_excel_tool_extended_functionality(
@@ -428,16 +444,17 @@ def test_excel_tool_extended_functionality(
     file_url = uploaded_file.get("file_url")
 
     # Send the prompt with the uploaded file
-    response = assistant_utils.ask_assistant(
-        assistant_instance, prompt, file_urls=[file_url]
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant_instance, prompt, file_urls=[file_url], minimal_response=False
     )
 
+    assert_tool_triggered(Default.EXCEL_TOOL, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.file
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 @pytest.mark.parametrize("prompt,expected_response", DOCX_TOOL_TEST_DATA)
 def test_docx_tool_extended_functionality(
@@ -467,8 +484,9 @@ def test_docx_tool_extended_functionality(
     file_url = uploaded_file.get("file_url")
 
     # Send the prompt with the uploaded file
-    response = assistant_utils.ask_assistant(
-        assistant_instance, prompt, file_urls=[file_url]
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant_instance, prompt, file_urls=[file_url], minimal_response=False
     )
 
+    assert_tool_triggered(Default.DOCX_TOOL, triggered_tools)
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py

@@ -5,11 +5,12 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
     KEYCLOAK_TOOL_PROMPT,
     KEYCLOAK_TOOL_RESPONSE,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 @pytest.mark.assistant
 @pytest.mark.keycloak
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_keycloak_tool(
     assistant_utils,
     assistant,
@@ -22,6 +23,10 @@ def test_assistant_with_keycloak_tool(
         settings=keycloak_integration,
     )
 
-    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        keycloak_assistant, KEYCLOAK_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_test_plan_tools.py

@@ -11,14 +11,17 @@ from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ADO_TEST_PLAN_DELETE_PLAN,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import ID_PATTERN
 from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
 
 
 @pytest.mark.assistant
 @pytest.mark.ado
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
     ado_test_plan_get_test_data,
@@ -44,13 +47,17 @@ def test_assistant_with_ado_test_plan_get_tools(
         settings=settings,
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.ado
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_ado_test_plan_tools(
     assistant_utils,
     assistant,
@@ -78,13 +85,18 @@ def test_assistant_with_ado_test_plan_tools(
     create_prompt = ADO_TEST_PLAN_CREATE_TEST_PLAN["prompt_to_assistant"].format(
         test_plan_title
     )
-    create_response = assistant_utils.ask_assistant(assistant, create_prompt)
+    create_response, create_triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_prompt, minimal_response=False
+    )
 
     # Extract the test plan ID from the response
     test_plan_id = extract_id_from_ado_response(create_response, ID_PATTERN)
     create_expected = ADO_TEST_PLAN_CREATE_TEST_PLAN["expected_llm_answer"].format(
         test_plan_title, test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.CREATE_TEST_PLAN, create_triggered_tools
+    )
     similarity_check.check_similarity(create_response, create_expected)
 
     # Calculate root suite ID (based on the Java implementation)
@@ -94,8 +106,8 @@ def test_assistant_with_ado_test_plan_tools(
     create_suite_prompt = ADO_TEST_PLAN_CREATE_SUITE["prompt_to_assistant"].format(
         root_suite_id, test_plan_id
     )
-    create_suite_response = assistant_utils.ask_assistant(
-        assistant, create_suite_prompt
+    create_suite_response, create_suite_triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_suite_prompt, minimal_response=False
     )
 
     # Extract the test suite ID from the response
@@ -103,36 +115,52 @@ def test_assistant_with_ado_test_plan_tools(
     create_suite_expected = ADO_TEST_PLAN_CREATE_SUITE["expected_llm_answer"].format(
         test_suite_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.CREATE_TEST_SUITE, create_suite_triggered_tools
+    )
     similarity_check.check_similarity(create_suite_response, create_suite_expected)
 
     # 3. Add test case to the suite
     add_case_prompt = ADO_TEST_PLAN_ADD_TEST_CASE["prompt_to_assistant"].format(
         test_suite_id, test_plan_id
     )
-    add_case_response = assistant_utils.ask_assistant(assistant, add_case_prompt)
+    add_case_response, add_case_triggered_tools = assistant_utils.ask_assistant(
+        assistant, add_case_prompt, minimal_response=False
+    )
     add_case_expected = ADO_TEST_PLAN_ADD_TEST_CASE["expected_llm_answer"].format(
         test_suite_id, test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.ADD_TEST_CASE, add_case_triggered_tools
+    )
     similarity_check.check_similarity(add_case_response, add_case_expected)
 
     # 4. Delete test suite
     delete_suite_prompt = ADO_TEST_PLAN_DELETE_SUITE["prompt_to_assistant"].format(
         test_suite_id, test_plan_id
     )
-    delete_suite_response = assistant_utils.ask_assistant(
-        assistant, delete_suite_prompt
+    delete_suite_response, delete_suite_triggered_tools = assistant_utils.ask_assistant(
+        assistant, delete_suite_prompt, minimal_response=False
     )
     delete_suite_expected = ADO_TEST_PLAN_DELETE_SUITE["expected_llm_answer"].format(
         test_suite_id, test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.DELETE_TEST_SUITE, delete_suite_triggered_tools
+    )
     similarity_check.check_similarity(delete_suite_response, delete_suite_expected)
 
     # 5. Delete test plan
     delete_plan_prompt = ADO_TEST_PLAN_DELETE_PLAN["prompt_to_assistant"].format(
         test_plan_id
     )
-    delete_plan_response = assistant_utils.ask_assistant(assistant, delete_plan_prompt)
+    delete_plan_response, delete_plan_triggered_tools = assistant_utils.ask_assistant(
+        assistant, delete_plan_prompt, minimal_response=False
+    )
     delete_plan_expected = ADO_TEST_PLAN_DELETE_PLAN["expected_llm_answer"].format(
         test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.DELETE_TEST_PLAN, delete_plan_triggered_tools
+    )
     similarity_check.check_similarity(delete_plan_response, delete_plan_expected)

codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_wiki_tools.py

@@ -10,12 +10,15 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
     ADO_WIKI_DELETE_PAGE,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.assistant
 @pytest.mark.ado
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
     ado_wiki_get_test_data,
@@ -41,13 +44,17 @@ def test_assistant_with_ado_wiki_get_tools(
         settings=settings,
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.ado
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_ado_wiki_modify_tools(
     assistant_utils,
     assistant,
@@ -72,7 +79,10 @@ def test_assistant_with_ado_wiki_modify_tools(
     # 1. Create the page
     create_prompt = ADO_WIKI_CREATE_PAGE["prompt_to_assistant"].format(page_title)
     create_expected = ADO_WIKI_CREATE_PAGE["expected_llm_answer"].format(page_title)
-    response = assistant_utils.ask_assistant(assistant, create_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_prompt, minimal_response=False
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, triggered_tools)
     similarity_check.check_similarity(response, create_expected)
 
     # 2. Rename the page
@@ -82,7 +92,10 @@ def test_assistant_with_ado_wiki_modify_tools(
     rename_expected = ADO_WIKI_RENAME_PAGE["expected_llm_answer"].format(
         page_title, page_title
     )
-    response = assistant_utils.ask_assistant(assistant, rename_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, rename_prompt, minimal_response=False
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.RENAME_WIKI_PAGE, triggered_tools)
     similarity_check.check_similarity(response, rename_expected)
 
     # 3. Modify the page
@@ -92,7 +105,10 @@ def test_assistant_with_ado_wiki_modify_tools(
     modify_expected = ADO_WIKI_MODIFY_PAGE["expected_llm_answer"].format(
         page_title + "-Updated"
     )
-    response = assistant_utils.ask_assistant(assistant, modify_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, modify_prompt, minimal_response=False
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, triggered_tools)
     similarity_check.check_similarity(response, modify_expected)
 
     # 4. Delete the page
@@ -102,5 +118,8 @@ def test_assistant_with_ado_wiki_modify_tools(
     delete_expected = ADO_WIKI_DELETE_PAGE["expected_llm_answer"].format(
         page_title + "-Updated"
     )
-    response = assistant_utils.ask_assistant(assistant, delete_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, delete_prompt, minimal_response=False
+    )
+    assert_tool_triggered(AzureDevOpsWikiTool.DELETE_WIKI_PAGE_BY_PATH, triggered_tools)
     similarity_check.check_similarity(response, delete_expected)

codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_work_item_tools.py

@@ -9,14 +9,17 @@ from codemie_test_harness.tests.test_data.ado_work_item_tools_test_data import (
     ADO_WORK_ITEM_LINK,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import WORK_ITEM_ID_PATTERN
 from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
 
 
 @pytest.mark.assistant
 @pytest.mark.ado
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
     ado_work_item_get_test_data,
@@ -42,13 +45,17 @@ def test_assistant_with_ado_work_item_get_tools(
         settings=settings,
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
 @pytest.mark.ado
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_ado_work_item_modify_tools(
     assistant_utils,
     assistant,
@@ -72,29 +79,42 @@ def test_assistant_with_ado_work_item_modify_tools(
     work_item_title = f"Autotest Task {get_random_name()}"
 
     create_prompt = ADO_WORK_ITEM_CREATE["prompt_to_assistant"].format(work_item_title)
-    create_response = assistant_utils.ask_assistant(assistant, create_prompt)
+    create_response, create_triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_prompt, minimal_response=False
+    )
 
     work_item_id = extract_id_from_ado_response(create_response, WORK_ITEM_ID_PATTERN)
     create_expected = ADO_WORK_ITEM_CREATE["expected_llm_answer"].format(
         work_item_title
     )
+    assert_tool_triggered(
+        AzureDevOpsWorkItemTool.CREATE_WORK_ITEM, create_triggered_tools
+    )
     similarity_check.check_similarity(create_response, create_expected)
 
     new_title = f"Autotest Epic {get_random_name()}"
     update_prompt = ADO_WORK_ITEM_UPDATE["prompt_to_assistant"].format(
         work_item_id, new_title
     )
-    update_response = assistant_utils.ask_assistant(assistant, update_prompt)
+    update_response, update_triggered_tools = assistant_utils.ask_assistant(
+        assistant, update_prompt, minimal_response=False
+    )
 
     update_expected = ADO_WORK_ITEM_UPDATE["expected_llm_answer"].format(
         work_item_id, new_title
     )
+    assert_tool_triggered(
+        AzureDevOpsWorkItemTool.UPDATE_WORK_ITEM, update_triggered_tools
+    )
     similarity_check.check_similarity(update_response, update_expected)
 
     link_prompt = ADO_WORK_ITEM_LINK["prompt_to_assistant"].format(
         work_item_id, work_item_id
     )
-    link_response = assistant_utils.ask_assistant(assistant, link_prompt)
+    link_response, link_triggered_tools = assistant_utils.ask_assistant(
+        assistant, link_prompt, minimal_response=False
+    )
 
     link_expected = ADO_WORK_ITEM_LINK["expected_llm_answer"].format(work_item_id)
+    assert_tool_triggered(AzureDevOpsWorkItemTool.LINK_WORK_ITEMS, link_triggered_tools)
     similarity_check.check_similarity(link_response, link_expected)

codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py

@@ -2,10 +2,12 @@ import pytest
 
 from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
 
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
+
 
 @pytest.mark.assistant
 @pytest.mark.cloud
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,credential_type,credentials,prompt,expected_response",
     cloud_test_data,
@@ -24,8 +26,12 @@ def test_assistant_with_cloud_tools(
 ):
     settings = integration_utils.create_integration(credential_type, credentials)
 
-    aws_assistant = assistant(toolkit, tool_name, settings=settings)
+    cloud_assistant = assistant(toolkit, tool_name, settings=settings)
+
+    response, triggered_tools = assistant_utils.ask_assistant(
+        cloud_assistant, prompt, minimal_response=False
+    )
 
-    response = assistant_utils.ask_assistant(aws_assistant, prompt)
+    assert_tool_triggered(tool_name, triggered_tools)
 
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py

@@ -8,11 +8,12 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
     code_tools_test_data,
     sonar_tools_test_data,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 @pytest.mark.assistant
 @pytest.mark.codebase
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
     code_tools_test_data,
@@ -30,12 +31,16 @@ def test_assistant_with_codebase_tools(
     expected_response,
 ):
     assistant = assistant(toolkit, tool_name, context=code_context(code_datasource))
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.assistant
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "toolkit, tool_name, credentials, prompt, expected",
     sonar_tools_test_data,
@@ -60,5 +65,9 @@ def test_assistant_with_sonar_tools(
         settings=settings,
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected)