codemie-test-harness 0.1.179__py3-none-any.whl → 0.1.181__py3-none-any.whl

codemie_test_harness/tests/assistant/datasource/test_google_datasource.py

@@ -9,13 +9,18 @@ from codemie_test_harness.tests.test_data.google_datasource_test_data import (
     RESPONSE_FOR_GOOGLE_DOC,
     USER_PROMPT,
 )
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 
 
 @pytest.mark.datasource
 @pytest.mark.google
 @pytest.mark.smoke
-@pytest.mark.regression
+@pytest.mark.api
 def test_create_datasource_with_google_integration(
     datasource_utils,
     assistant,
@@ -24,22 +29,31 @@ def test_create_datasource_with_google_integration(
     kb_context,
     google_doc_datasource,
 ):
-    assistant = assistant(context=kb_context(google_doc_datasource))
+    assistant = assistant(
+        context=kb_context(google_doc_datasource),
+        system_prompt="Run tools on each user prompt",
+    )
 
-    response_from_llm = assistant_utils.ask_assistant(assistant, USER_PROMPT)
+    response_from_llm, triggered_tools = assistant_utils.ask_assistant(
+        assistant, USER_PROMPT, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response_from_llm, RESPONSE_FOR_GOOGLE_DOC)
 
     datasource_utils.update_google_doc_datasource(
         google_doc_datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response_from_llm = assistant_utils.ask_assistant(assistant, USER_PROMPT)
+    response_from_llm, triggered_tools = assistant_utils.ask_assistant(
+        assistant, USER_PROMPT, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
     similarity_check.check_similarity(response_from_llm, RESPONSE_FOR_GOOGLE_DOC)
 
 
 @pytest.mark.datasource
 @pytest.mark.google
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_edit_description_for_google_data_source(
     client, datasource_utils, google_doc_datasource
@@ -56,7 +70,7 @@ def test_edit_description_for_google_data_source(
 
 @pytest.mark.datasource
 @pytest.mark.google
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.smoke
 def test_create_google_doc_datasource_with_existing_name(
     google_doc_datasource, datasource_utils

codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py

@@ -12,7 +12,12 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
     RESPONSE_FOR_JIRA_CLOUD_TOOL,
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
-from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
+from codemie_test_harness.tests.utils.base_utils import (
+    get_random_name,
+    assert_response,
+    assert_tool_triggered,
+)
+from codemie_test_harness.tests.enums.tools import Default
 
 
 @pytest.fixture(scope="function")
@@ -27,7 +32,7 @@ def jira_cloud_datasource(datasource_utils, jira_cloud_integration):
 
 @pytest.mark.datasource
 @pytest.mark.project_management
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.parametrize(
     "datasource_fixture, prompt, expected_response",
     [
@@ -59,22 +64,30 @@ def test_create_datasource_with_jira_and_jira_cloud_integration(
     expected_response,
 ):
     datasource = request.getfixturevalue(datasource_fixture)
-    assistant = assistant(context=kb_context(datasource))
+    assistant = assistant(
+        context=kb_context(datasource), system_prompt="Run tools on each user prompt"
+    )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
-    similarity_check.check_similarity(response, response)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
 
     datasource_utils.update_jira_datasource(
         datasource.id, full_reindex=True, skip_reindex=False
     )
 
-    response = assistant_utils.ask_assistant(assistant, prompt)
-    similarity_check.check_similarity(response, response)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        assistant, prompt, minimal_response=False
+    )
+    assert_tool_triggered(Default.SEARCH_KB, triggered_tools)
+    similarity_check.check_similarity(response, expected_response)
 
 
 @pytest.mark.datasource
 @pytest.mark.jira
-@pytest.mark.regression
+@pytest.mark.api
 def test_edit_description_for_jira_data_source(
     client, datasource_utils, jira_datasource
 ):
@@ -90,7 +103,7 @@ def test_edit_description_for_jira_data_source(
 
 @pytest.mark.datasource
 @pytest.mark.jira
-@pytest.mark.regression
+@pytest.mark.api
 def test_create_jira_datasource_with_existing_name(datasource_utils, jira_datasource):
     datasource = datasource_utils.get_datasource(jira_datasource.id)
 

codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool.py

@@ -13,13 +13,14 @@ from codemie_test_harness.tests.test_data.pm_tools_test_data import (
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 @pytest.mark.parametrize(
     "user_integration, is_global, project_integration",
     integrations,
@@ -71,16 +72,19 @@ def test_assistant_should_use_user_integration_by_default(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 def test_assistant_with_global_and_project_integration(
     integration_utils,
     assistant,
@@ -115,16 +119,19 @@ def test_assistant_with_global_and_project_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 def test_assistant_with_project_integration_only(
     integration_utils,
     assistant,
@@ -150,15 +157,18 @@ def test_assistant_with_project_integration_only(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_global_valid_and_user_invalid_integration(
     assistant, assistant_utils, integration_utils, similarity_check
 ):
@@ -190,15 +200,18 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_project_valid_and_user_invalid_integration(
     assistant, assistant_utils, integration_utils, similarity_check
 ):
@@ -230,6 +243,9 @@ def test_assistant_with_project_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(jira_assistant, JIRA_TOOL_PROMPT)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        jira_assistant, JIRA_TOOL_PROMPT, minimal_response=False
+    )
 
+    assert_tool_triggered(ProjectManagementTool.JIRA, triggered_tools)
     similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)

codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py

@@ -12,6 +12,7 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 ado_wiki_prompt = ado_wiki_get_test_data[1][2]
 ado_wiki_answer = ado_wiki_get_test_data[1][3]
@@ -20,8 +21,8 @@ ado_wiki_answer = ado_wiki_get_test_data[1][3]
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 @pytest.mark.parametrize(
     "user_integration, is_global, project_integration",
     integrations,
@@ -75,16 +76,19 @@ def test_assistant_should_use_user_integration_by_default(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
-@pytest.mark.regression
+@pytest.mark.api
 @pytest.mark.not_for_parallel_run
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.testcase("EPMCDME-6708")
 def test_assistant_with_global_and_project_integration(
     integration_utils,
     assistant,
@@ -120,16 +124,19 @@ def test_assistant_with_global_and_project_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 def test_assistant_with_project_integration_only(
     integration_utils,
     assistant,
@@ -157,15 +164,18 @@ def test_assistant_with_project_integration_only(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_global_valid_and_user_invalid_integration(
     assistant, assistant_utils, integration_utils, similarity_check
 ):
@@ -199,15 +209,18 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_project_valid_and_user_invalid_integration(
     assistant, assistant_utils, integration_utils, similarity_check
 ):
@@ -241,6 +254,9 @@ def test_assistant_with_project_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(ado_assistant, ado_wiki_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        ado_assistant, ado_wiki_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(AzureDevOpsWikiTool.GET_WIKI, triggered_tools)
     similarity_check.check_similarity(response, ado_wiki_answer)

codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py

@@ -12,6 +12,7 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
 )
 from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
 from codemie_test_harness.tests.utils.constants import test_project_name
+from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
 
 
 git_tool_prompt = list_branches_set_active_branch_test_data[0][2]
@@ -21,8 +22,8 @@ git_tool_answer = list_branches_set_active_branch_test_data[0][3]
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 @pytest.mark.parametrize(
     "user_integration, is_global, project_integration",
     integrations,
@@ -86,16 +87,19 @@ def test_assistant_should_use_user_integration_by_default(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 def test_assistant_with_global_and_project_integration(
     datasource_utils,
     default_embedding_llm,
@@ -145,16 +149,19 @@ def test_assistant_with_global_and_project_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
-@pytest.mark.tescase("EPMCDME-6708")
+@pytest.mark.api
+@pytest.mark.testcase("EPMCDME-6708")
 def test_assistant_with_project_integration_only(
     datasource_utils,
     default_embedding_llm,
@@ -191,15 +198,18 @@ def test_assistant_with_project_integration_only(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_global_valid_and_user_invalid_integration(
     assistant,
     assistant_utils,
@@ -249,15 +259,18 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)
 
 
 @pytest.mark.integration
 @pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
-@pytest.mark.regression
+@pytest.mark.api
 def test_assistant_with_project_valid_and_user_invalid_integration(
     assistant,
     assistant_utils,
@@ -307,6 +320,9 @@ def test_assistant_with_project_valid_and_user_invalid_integration(
         project_name=test_project_name,
     )
 
-    response = assistant_utils.ask_assistant(git_assistant, git_tool_prompt)
+    response, triggered_tools = assistant_utils.ask_assistant(
+        git_assistant, git_tool_prompt, minimal_response=False
+    )
 
+    assert_tool_triggered(GitTool.LIST_BRANCHES_IN_REPO, triggered_tools)
     similarity_check.check_similarity(response, git_tool_answer)