codemie-test-harness 0.1.180__py3-none-any.whl → 0.1.181__py3-none-any.whl

codemie_test_harness/tests/assistant/datasource/test_code_datasource.py

@@ -4,8 +4,13 @@ import pytest
 from hamcrest import assert_that, equal_to
 from requests import HTTPError
 
+from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.test_data.index_test_data import index_test_data
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
 
 
 @pytest.mark.datasource
@@ -35,16 +40,24 @@ def test_create_index_application_with_embedding_model(
         setting_id=git_integration.id, embeddings_model=embedding_model
     )
 
-    assistant = assistant(context=code_context(datasource))
+    assistant = assistant(
+        context=code_context(datasource), system_prompt="Run tools on each user prompt"
+    )
 
-    answer = assistant_utils.ask_assistant(assistant, question)
+    answer, triggered_tools = assistant_utils.ask_assistant(
+        assistant, question, minimal_response=False
+    )
+    assert_tool_triggered(Default.GET_REPOSITORY_FILE_TREE, triggered_tools)
     similarity_check.check_similarity(answer, expected_answer)
 
     datasource_utils.update_code_datasource(
         datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    answer = assistant_utils.ask_assistant(assistant, question)
+    answer, triggered_tools = assistant_utils.ask_assistant(
+        assistant, question, minimal_response=False
+    )
+    assert_tool_triggered(Default.GET_REPOSITORY_FILE_TREE, triggered_tools)
     similarity_check.check_similarity(answer, expected_answer)
 
 

codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py

@@ -13,7 +13,12 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 
 
 @pytest.fixture(scope="function")
@@ -61,16 +66,24 @@ def test_create_datasource_with_confluence_and_confluence_cloud_integration(
     expected_response,
 ):
     datasource = request.getfixturevalue(datasource_fixture)
-    assistant = assistant(context=kb_context(datasource))
+    assistant = assistant(
+        context=kb_context(datasource), system_prompt="Run tools on each user prompt"
+    )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
     datasource_utils.update_confluence_datasource(
         datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 

codemie_test_harness/tests/assistant/datasource/test_file_indexing.py

@@ -6,12 +6,17 @@ from hamcrest import (
 )
 from requests import HTTPError
 
+from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.test_data.file_test_data import (
     file_test_data,
     large_files_test_data,
     RESPONSE_FOR_TWO_FILES_INDEXED,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.test_data.index_test_data import index_test_data
 from codemie_test_harness.tests.utils.client_factory import get_client
 from codemie_test_harness.tests.utils.constants import FILES_PATH
@@ -82,7 +87,10 @@ def test_create_assistant_with_file_datasource(
     test_assistant = assistant(context=kb_context(datasource))
 
     prompt = "Show KB context. Return all information available in the context. Query may be 'Show content of the KB'"
-    response = assistant_utils.ask_assistant(test_assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        test_assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
 
     similarity_check.check_similarity(
         response, expected_response, assistant_name=test_assistant.name
@@ -168,8 +176,11 @@ def test_create_file_datasource_with_two_files(
 
     test_assistant = assistant(context=kb_context(datasource))
 
-    response = assistant_utils.ask_assistant(
-        test_assistant, "What types of data do we have available?"
+    response, triggered_tools = assistant_utils.ask_assistant(
+        test_assistant,
+        "What types of data do we have available?",
+        minimal_response=False,
     )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
 
     similarity_check.check_similarity(response, RESPONSE_FOR_TWO_FILES_INDEXED)

codemie_test_harness/tests/assistant/datasource/test_google_datasource.py

@@ -9,7 +9,12 @@ from codemie_test_harness.tests.test_data.google_datasource_test_data import (
     RESPONSE_FOR_GOOGLE_DOC,
     USER_PROMPT,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 
 
 @pytest.mark.datasource
@@ -24,16 +29,25 @@ def test_create_datasource_with_google_integration(
     kb_context,
     google_doc_datasource,
 ):
-    assistant = assistant(context=kb_context(google_doc_datasource))
+    assistant = assistant(
+        context=kb_context(google_doc_datasource),
+        system_prompt="Run tools on each user prompt",
+    )
 
-    response_from_llm = assistant_utils.ask_assistant(assistant, USER_PROMPT)
+    response_from_llm, triggered_tools = assistant_utils.ask_assistant(
+        assistant, USER_PROMPT, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response_from_llm, RESPONSE_FOR_GOOGLE_DOC)
 
     datasource_utils.update_google_doc_datasource(
         google_doc_datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response_from_llm = assistant_utils.ask_assistant(assistant, USER_PROMPT)
+    response_from_llm, triggered_tools = assistant_utils.ask_assistant(
+        assistant, USER_PROMPT, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response_from_llm, RESPONSE_FOR_GOOGLE_DOC)
 
 

codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py

@@ -12,7 +12,12 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     RESPONSE_FOR_JIRA_CLOUD_TOOL,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 
 
 @pytest.fixture(scope="function")
@@ -59,17 +64,25 @@ def test_create_datasource_with_jira_and_jira_cloud_integration(
     expected_response,
 ):
     datasource = request.getfixturevalue(datasource_fixture)
-    assistant = assistant(context=kb_context(datasource))
+    assistant = assistant(
+        context=kb_context(datasource), system_prompt="Run tools on each user prompt"
+    )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
-    similarity_check.check_similarity(response, response)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
 
     datasource_utils.update_jira_datasource(
         datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
-    similarity_check.check_similarity(response, response)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.datasource

codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool.py

@@ -13,6 +13,7 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 @pytest.mark.integration
@@ -71,8 +72,11 @@ def test_assistant_should_use_user_integration_by_default(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -115,8 +119,11 @@ def test_assistant_with_global_and_project_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -150,8 +157,11 @@ def test_assistant_with_project_integration_only(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -190,8 +200,11 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
@@ -230,6 +243,9 @@ def test_assistant_with_project_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)

codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py

@@ -12,6 +12,7 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 ado_wiki_prompt = ado_wiki_get_test_data[1][2]
 ado_wiki_answer = ado_wiki_get_test_data[1][3]
@@ -75,8 +76,11 @@ def test_assistant_should_use_user_integration_by_default(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -120,8 +124,11 @@ def test_assistant_with_global_and_project_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -157,8 +164,11 @@ def test_assistant_with_project_integration_only(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -199,8 +209,11 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
@@ -241,6 +254,9 @@ def test_assistant_with_project_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)

codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py

@@ -12,6 +12,7 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 git_tool_prompt = list_branches_set_active_branch_test_data[0][2]
@@ -86,8 +87,11 @@ def test_assistant_should_use_user_integration_by_default(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -145,8 +149,11 @@ def test_assistant_with_global_and_project_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -191,8 +198,11 @@ def test_assistant_with_project_integration_only(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -249,8 +259,11 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
@@ -307,6 +320,9 @@ def test_assistant_with_project_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)

codemie_test_harness/tests/assistant/test_assistants.py

@@ -24,7 +24,11 @@ from codemie_test_harness.tests.test_data.file_test_data import (
     RESPONSE_FOR_TWO_FILES_UPLOADED,
 )
 from codemie_test_harness.tests.test_data.output_schema_test_data import output_schema
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 from codemie_test_harness.tests.utils.constants import FILES_PATH
 from codemie_test_harness.tests.utils.file_utils import save_file
 
@@ -224,7 +228,7 @@ def test_export_assistant(assistant_utils):
 @pytest.mark.smoke
 @pytest.mark.testcase("EPMCDME-4001, EPMCDME-4002, EPMCDME-2527")
 @pytest.mark.parametrize(
-    "file_name,expected_response",
+    "file_name,expected_response,expected_tool",
     file_test_data,
     ids=[f"{row[0]}" for row in file_test_data],
 )
@@ -234,6 +238,7 @@ def test_create_assistant_and_prompt_with_file(
     similarity_check,
     file_name,
     expected_response,
+    expected_tool,
 ):
     prompt = (
         f"What is the content/text of the {file_name}. Show information from ALL pages. "
@@ -249,14 +254,20 @@ def test_create_assistant_and_prompt_with_file(
 
     conversation_id = str(uuid.uuid4())
 
-    response = assistant_utils.ask_assistant(
-        assistant, prompt, file_urls=[file_url], conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant,
+        prompt,
+        file_urls=[file_url],
+        conversation_id=conversation_id,
+        minimal_response=False,
     )
+    assert_tool_triggered(expected_tool, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
-    response = assistant_utils.ask_assistant(
-        assistant, prompt, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, conversation_id=conversation_id, minimal_response=False
     )
+    assert_tool_triggered(expected_tool, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -382,17 +393,22 @@ def test_create_assistant_and_prompt_with_multiple_files(
 
     conversation_id = str(uuid.uuid4())
 
-    response = assistant_utils.ask_assistant(
+    response, triggered_tools = assistant_utils.ask_assistant(
         assistant,
         prompt,
         file_urls=[docx_file_url, ini_file_url],
         conversation_id=conversation_id,
+        minimal_response=False,
     )
+
+    assert_tool_triggered((Default.DOCX_TOOL, Default.FILE_ANALYSIS), triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_TWO_FILES_UPLOADED)
 
-    response = assistant_utils.ask_assistant(
-        assistant, prompt, conversation_id=conversation_id
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, conversation_id=conversation_id, minimal_response=False
     )
+
+    assert_tool_triggered((Default.DOCX_TOOL, Default.FILE_ANALYSIS), triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_TWO_FILES_UPLOADED)
 
 
@@ -428,10 +444,11 @@ def test_excel_tool_extended_functionality(
     file_url = uploaded_file.get("file_url")
 
     # Send the prompt with the uploaded file
-    response = assistant_utils.ask_assistant(
-        assistant_instance, prompt, file_urls=[file_url]
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant_instance, prompt, file_urls=[file_url], minimal_response=False
     )
 
+    assert_tool_triggered(Default.EXCEL_TOOL, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -467,8 +484,9 @@ def test_docx_tool_extended_functionality(
     file_url = uploaded_file.get("file_url")
 
     # Send the prompt with the uploaded file
-    response = assistant_utils.ask_assistant(
-        assistant_instance, prompt, file_urls=[file_url]
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant_instance, prompt, file_urls=[file_url], minimal_response=False
     )
 
+    assert_tool_triggered(Default.DOCX_TOOL, triggered_tools)
     similarity_check.check_similarity(response, expected_response)

codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py

@@ -5,6 +5,7 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
     KEYCLOAK_TOOL_PROMPT,
     KEYCLOAK_TOOL_RESPONSE,
 )
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 @pytest.mark.assistant
@@ -22,6 +23,10 @@ def test_assistant_with_keycloak_tool(
         settings=keycloak_integration,
     )
 
-    response = assistant_utils.ask_assistant(keycloak_assistant, KEYCLOAK_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        keycloak_assistant, KEYCLOAK_TOOL_PROMPT, minimal_response=False
+    )
+
+    assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
 
     similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)

codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_test_plan_tools.py

@@ -11,7 +11,10 @@ from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
     ADO_TEST_PLAN_DELETE_PLAN,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_tool_triggered,
+)
 from codemie_test_harness.tests.utils.constants import ID_PATTERN
 from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
 
@@ -44,7 +47,11 @@ def test_assistant_with_ado_test_plan_get_tools(
         settings=settings,
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+
+    assert_tool_triggered(tool_name, triggered_tools)
     similarity_check.check_similarity(response, expected_response)
 
 
@@ -78,13 +85,18 @@ def test_assistant_with_ado_test_plan_tools(
     create_prompt = ADO_TEST_PLAN_CREATE_TEST_PLAN["prompt_to_assistant"].format(
         test_plan_title
     )
-    create_response = assistant_utils.ask_assistant(assistant, create_prompt)
+    create_response, create_triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_prompt, minimal_response=False
+    )
 
     # Extract the test plan ID from the response
     test_plan_id = extract_id_from_ado_response(create_response, ID_PATTERN)
     create_expected = ADO_TEST_PLAN_CREATE_TEST_PLAN["expected_llm_answer"].format(
         test_plan_title, test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.CREATE_TEST_PLAN, create_triggered_tools
+    )
     similarity_check.check_similarity(create_response, create_expected)
 
     # Calculate root suite ID (based on the Java implementation)
@@ -94,8 +106,8 @@ def test_assistant_with_ado_test_plan_tools(
     create_suite_prompt = ADO_TEST_PLAN_CREATE_SUITE["prompt_to_assistant"].format(
         root_suite_id, test_plan_id
     )
-    create_suite_response = assistant_utils.ask_assistant(
-        assistant, create_suite_prompt
+    create_suite_response, create_suite_triggered_tools = assistant_utils.ask_assistant(
+        assistant, create_suite_prompt, minimal_response=False
     )
 
     # Extract the test suite ID from the response
@@ -103,36 +115,52 @@ def test_assistant_with_ado_test_plan_tools(
     create_suite_expected = ADO_TEST_PLAN_CREATE_SUITE["expected_llm_answer"].format(
         test_suite_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.CREATE_TEST_SUITE, create_suite_triggered_tools
+    )
     similarity_check.check_similarity(create_suite_response, create_suite_expected)
 
     # 3. Add test case to the suite
     add_case_prompt = ADO_TEST_PLAN_ADD_TEST_CASE["prompt_to_assistant"].format(
         test_suite_id, test_plan_id
     )
-    add_case_response = assistant_utils.ask_assistant(assistant, add_case_prompt)
+    add_case_response, add_case_triggered_tools = assistant_utils.ask_assistant(
+        assistant, add_case_prompt, minimal_response=False
+    )
     add_case_expected = ADO_TEST_PLAN_ADD_TEST_CASE["expected_llm_answer"].format(
         test_suite_id, test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.ADD_TEST_CASE, add_case_triggered_tools
+    )
     similarity_check.check_similarity(add_case_response, add_case_expected)
 
     # 4. Delete test suite
     delete_suite_prompt = ADO_TEST_PLAN_DELETE_SUITE["prompt_to_assistant"].format(
         test_suite_id, test_plan_id
     )
-    delete_suite_response = assistant_utils.ask_assistant(
-        assistant, delete_suite_prompt
+    delete_suite_response, delete_suite_triggered_tools = assistant_utils.ask_assistant(
+        assistant, delete_suite_prompt, minimal_response=False
     )
     delete_suite_expected = ADO_TEST_PLAN_DELETE_SUITE["expected_llm_answer"].format(
         test_suite_id, test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.DELETE_TEST_SUITE, delete_suite_triggered_tools
+    )
     similarity_check.check_similarity(delete_suite_response, delete_suite_expected)
 
     # 5. Delete test plan
     delete_plan_prompt = ADO_TEST_PLAN_DELETE_PLAN["prompt_to_assistant"].format(
         test_plan_id
     )
-    delete_plan_response = assistant_utils.ask_assistant(assistant, delete_plan_prompt)
+    delete_plan_response, delete_plan_triggered_tools = assistant_utils.ask_assistant(
+        assistant, delete_plan_prompt, minimal_response=False
+    )
     delete_plan_expected = ADO_TEST_PLAN_DELETE_PLAN["expected_llm_answer"].format(
         test_plan_id
     )
+    assert_tool_triggered(
+        AzureDevOpsTestPlanTool.DELETE_TEST_PLAN, delete_plan_triggered_tools
+    )
     similarity_check.check_similarity(delete_plan_response, delete_plan_expected)