codemie-test-harness 0.1.156__py3-none-any.whl → 0.1.157__py3-none-any.whl

codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py

@@ -17,6 +17,10 @@ ado_wiki_prompt = ado_wiki_get_test_data[0][2]
 ado_wiki_answer = ado_wiki_get_test_data[0][3]
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -82,6 +86,10 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -129,6 +137,10 @@ def test_assistant_in_workflow_with_global_and_project_integration(
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -168,6 +180,10 @@ def test_assistant_in_workflow_with_project_integration_only(
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
@@ -215,6 +231,10 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
     similarity_check.check_similarity(response, ado_wiki_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(

codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py

@@ -18,6 +18,11 @@ git_tool_prompt = list_branches_set_active_branch_test_data[0][2]
 git_tool_answer = list_branches_set_active_branch_test_data[0][3]
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
+@pytest.mark.datasource
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -93,6 +98,11 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
     similarity_check.check_similarity(response, git_tool_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
+@pytest.mark.datasource
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -154,6 +164,11 @@ def test_assistant_in_workflow_with_global_and_project_integration(
     similarity_check.check_similarity(response, git_tool_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
+@pytest.mark.datasource
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -202,6 +217,11 @@ def test_assistant_in_workflow_with_project_integration_only(
     similarity_check.check_similarity(response, git_tool_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
+@pytest.mark.datasource
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
@@ -262,6 +282,11 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
     similarity_check.check_similarity(response, git_tool_answer)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.integration
+@pytest.mark.default_integration
+@pytest.mark.datasource
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(

codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py

@@ -20,6 +20,9 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.file_management
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "tool_name, prompt, expected_response",
@@ -50,6 +53,9 @@ def test_workflow_with_assistant_with_file_management_tools(
     similarity_check.check_similarity(response, expected_response)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.file_management
 @pytest.mark.regression
 @pytest.mark.skipif(
     os.getenv("ENV") == "local", reason="Skipping this test on local environment"
@@ -82,6 +88,9 @@ def test_workflow_with_assistant_with_generate_image_tool(
     )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.file_management
 @pytest.mark.regression
 def test_workflow_with_assistant_with_read_file_tool(
     assistant,
@@ -109,6 +118,9 @@ def test_workflow_with_assistant_with_read_file_tool(
     similarity_check.check_similarity(response, RESPONSE_FOR_READ_FILE_TASK)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.file_management
 @pytest.mark.skip(reason="Test are flaky, tools work unstable")
 @pytest.mark.regression
 def test_workflow_with_assistant_with_file_editing_tool(
@@ -145,6 +157,9 @@ def test_workflow_with_assistant_with_file_editing_tool(
     similarity_check.check_similarity(response, RESPONSE_FOR_DIFF_UPDATE)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.file_management
 @pytest.mark.skip(reason="Test are flaky, tools work unstable")
 @pytest.mark.regression
 def test_workflow_with_assistant_with_filesystem_tool(

codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py

@@ -12,6 +12,9 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
 )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
@@ -44,6 +47,9 @@ def test_workflow_with_list_branch_set_active_branch_tools(
     similarity_check.check_similarity(response, expected_response)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template",
@@ -92,6 +98,9 @@ def test_workflow_with_create_branch_tool(
             git_utils.delete_branch(branch_name)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template,expected_content_template",
@@ -142,6 +151,9 @@ def test_workflow_with_create_file_tool(
             git_utils.delete_file(file_name, "main")
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template",
@@ -188,6 +200,9 @@ def test_workflow_with_create_merge_request_tool(
             git_utils.delete_branch(source_branch)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt_template,expected_template,file_content_template",
@@ -253,6 +268,9 @@ def test_workflow_with_delete_file_tool(
             git_utils.delete_file(file_name, "main")
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,create_mr_prompt_template,create_mr_expected_template,get_mr_changes_prompt_template,get_mr_changes_expected_template",
@@ -315,6 +333,9 @@ def test_workflow_with_get_merge_request_changes_tool(
             git_utils.delete_branch(source_branch)
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.gitlab
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,create_prompt_template,create_expected_template,created_content_template,update_prompt_template,update_expected_template,updated_content_template",

codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py

@@ -16,8 +16,10 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
 )
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.mcp
+@pytest.mark.regression
 @pytest.mark.skipif(
     os.getenv("ENV") == "local", reason="Skipping this test on local environment"
 )
@@ -38,8 +40,10 @@ def test_workflow_with_assistant_with_time_mcp_server(
     similarity_check.check_similarity(response, time_expected_response)
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.mcp
+@pytest.mark.regression
 @pytest.mark.parametrize(
     "command, expected_answer",
     cli_mcp_server_test_data,
@@ -69,8 +73,10 @@ def test_workflow_with_assistant_with_cli_mcp_server(
     similarity_check.check_similarity(response, expected_answer)
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.mcp
+@pytest.mark.regression
 @pytest.mark.skipif(
     os.getenv("ENV") == "local", reason="Skipping this test on local environment"
 )

codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py

@@ -14,6 +14,10 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
 )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.notification
+@pytest.mark.email
 @pytest.mark.regression
 @pytest.mark.skipif(
     os.getenv("ENV") in ["local", "gcp"],
@@ -58,6 +62,10 @@ def test_workflow_with_email_tool(
     assert_that(message_title, equal_to(EMAIL_SUBJECT))
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.notification
+@pytest.mark.telegram
 @pytest.mark.regression
 def test_workflow_with_telegram_tool(
     assistant,

codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py

@@ -8,6 +8,9 @@ from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
 )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.openapi
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "tool_name, prompt, expected_response",

codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py

@@ -11,8 +11,10 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
 from codemie_test_harness.tests.utils.constants import TESTS_PATH
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.plugin
+@pytest.mark.regression
 @pytest.mark.parametrize(
     "prompt,expected_response,tool_name",
     list_files_plugin_tools_test_data,
@@ -37,8 +39,10 @@ def test_workflow_with_assistant_with_list_files_plugin_tools(
     similarity_check.check_similarity(response, expected_response)
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.plugin
+@pytest.mark.regression
 def test_workflow_with_assistant_with_modify_files_plugin_tools(
     assistant,
     workflow_utils,

codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py

@@ -10,8 +10,11 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
 from codemie_test_harness.tests.utils.constants import TESTS_PATH
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.plugin
+@pytest.mark.mcp
+@pytest.mark.regression
 @pytest.mark.parametrize(
     "prompt,expected_response",
     cli_mcp_server_with_plugin_test_data,
@@ -36,8 +39,11 @@ def test_workflow_with_assistant_with_plugin_and_cli_mcp_server(
     similarity_check.check_similarity(response, expected_response)
 
 
-@pytest.mark.regression
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
 @pytest.mark.plugin
+@pytest.mark.mcp
+@pytest.mark.regression
 @pytest.mark.parametrize(
     "prompt,expected_response,tool_name",
     filesystem_mcp_server_with_plugin_test_data,

codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py

@@ -9,6 +9,9 @@ from codemie_test_harness.tests.utils.constants import (
 )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.project_management
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "tool_name, integration_type, prompt, expected_response",

codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py

@@ -9,6 +9,9 @@ from codemie_test_harness.tests.test_data.research_tools_test_data import (
 from codemie_test_harness.tests.utils.base_utils import percent_of_relevant_titles
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.research
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "tool_name,prompt,expected_percentage",
@@ -38,6 +41,9 @@ def test_workflow_with_search_tools(
     )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.research
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "tool_name,prompt,expected_response",

codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py

@@ -7,6 +7,9 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
 )
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.servicenow
 @pytest.mark.regression
 def test_workflow_with_assistant_with_servicenow_tools(
     service_now_integration,

codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py

@@ -9,6 +9,9 @@ from codemie_test_harness.tests.test_data.vcs_tools_test_data import (
 from codemie_test_harness.tests.utils.constants import vcs_integrations
 
 
+@pytest.mark.workflow
+@pytest.mark.workflow_with_assistant
+@pytest.mark.vcs
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "tool_name,prompt,expected_response",

codemie_test_harness/tests/workflow/config_validation/test_config_validation.py

@@ -82,14 +82,15 @@ VALIDATION_TEST_DATA = [
 INVALID_CONFIG_PATH = "test_data/workflow/invalid_config/"
 
 
+@pytest.mark.workflow
+@pytest.mark.regression
+@pytest.mark.smoke
+@pytest.mark.testcase("EPMCDME-5458")
 @pytest.mark.parametrize(
     "file_name, expected_errors",
     VALIDATION_TEST_DATA,
     ids=[f"{row[0]}" for row in VALIDATION_TEST_DATA],
 )
-@pytest.mark.testcase("EPMCDME-5458")
-@pytest.mark.regression
-@pytest.mark.smoke
 def test_create_workflow_with_invalid_config(
     workflow_utils, file_name, expected_errors
 ):

codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool.py

@@ -17,10 +17,13 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
 from codemie_test_harness.tests.utils.constants import test_project_name
 
 
-jira_tool_prompt = project_management_tools_data[0][3]
-jira_tool_response = project_management_tools_data[0][4]
+jira_tool_prompt = project_management_tools_data[0].values[3]
+jira_tool_response = project_management_tools_data[0].values[4]
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -85,6 +88,9 @@ def test_tool_in_workflow_should_use_user_integration_by_default(
     similarity_check.check_similarity(response, jira_tool_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -132,6 +138,9 @@ def test_tool_in_workflow_with_global_and_project_integration(
     similarity_check.check_similarity(response, jira_tool_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -170,6 +179,9 @@ def test_tool_in_workflow_with_project_integration_only(
     similarity_check.check_similarity(response, jira_tool_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_tool_in_workflow_with_global_valid_and_user_invalid_integration(
@@ -216,6 +228,9 @@ def test_tool_in_workflow_with_global_valid_and_user_invalid_integration(
     similarity_check.check_similarity(response, jira_tool_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_tool_in_workflow_with_project_valid_and_user_invalid_integration(

codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool_kit.py

@@ -22,6 +22,9 @@ ado_wiki_prompt = ado_wiki_get_test_data[0][2]
 ado_wiki_answer = ado_wiki_get_test_data[0][3]
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -87,6 +90,9 @@ def test_tool_in_workflow_should_use_user_integration_by_default(
     assert_that(response, equal_to(ado_wiki_answer))
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -134,6 +140,9 @@ def test_tool_in_workflow_with_global_and_project_integration(
     assert_that(response, equal_to(ado_wiki_answer))
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 @pytest.mark.tescase("EPMCDME-6708")
@@ -173,6 +182,9 @@ def test_tool_in_workflow_with_project_integration_only(
     assert_that(response, equal_to(ado_wiki_answer))
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_tool_in_workflow_with_global_valid_and_user_invalid_integration(
@@ -220,6 +232,9 @@ def test_tool_in_workflow_with_global_valid_and_user_invalid_integration(
     assert_that(response, equal_to(ado_wiki_answer))
 
 
+@pytest.mark.workflow
+@pytest.mark.integration
+@pytest.mark.default_integration
 @pytest.mark.not_for_parallel_run
 @pytest.mark.regression
 def test_tool_in_workflow_with_project_valid_and_user_invalid_integration(

codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_access_management_tool.py

@@ -12,6 +12,9 @@ from codemie_test_harness.tests.test_data.direct_tools.keycloak_tool_test_data i
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.keycloak
 @pytest.mark.regression
 def test_workflow_with_keycloak_tool_direct(
     keycloak_integration, workflow_utils, workflow_with_tool, similarity_check
@@ -31,6 +34,9 @@ def test_workflow_with_keycloak_tool_direct(
     similarity_check.check_similarity(response, KEYCLOAK_DIRECT_TOOL_RESPONSE)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.keycloak
 @pytest.mark.regression
 def test_workflow_with_keycloak_tool_with_hardcoded_args(
     keycloak_integration, workflow_utils, workflow_with_tool, similarity_check
@@ -49,6 +55,9 @@ def test_workflow_with_keycloak_tool_with_hardcoded_args(
     similarity_check.check_similarity(response, KEYCLOAK_DIRECT_TOOL_RESPONSE)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.keycloak
 @pytest.mark.regression
 def test_workflow_with_keycloak_tool_with_overriding_args(
     keycloak_integration, workflow_utils, workflow_with_tool, similarity_check

codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_test_plan_tools.py

@@ -10,6 +10,9 @@ from codemie_test_harness.tests.test_data.direct_tools.ado_test_plan_tools_test_
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
@@ -38,6 +41,9 @@ def test_workflow_with_ado_test_plan_get_tools_direct(
     similarity_check.check_similarity(response, expected_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",
@@ -64,6 +70,9 @@ def test_workflow_with_ado_test_plan_get_tools_with_hardcoded_args(
     similarity_check.check_similarity(response, expected_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,prompt,expected_response",

codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_wiki_tools.py

@@ -10,6 +10,9 @@ from codemie_test_harness.tests.test_data.direct_tools.ado_wiki_tools_test_data
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-5192")
 @pytest.mark.parametrize(
@@ -39,6 +42,9 @@ def test_workflow_with_ado_wiki_get_tools_direct(
     similarity_check.check_similarity(response, expected_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-5192")
 @pytest.mark.parametrize(
@@ -66,6 +72,9 @@ def test_workflow_with_ado_wiki_get_tools_with_hardcoded_args(
     similarity_check.check_similarity(response, expected_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-5192")
 @pytest.mark.parametrize(

codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_work_item_tools.py

@@ -10,6 +10,9 @@ from codemie_test_harness.tests.test_data.direct_tools.ado_work_item_tools_test_
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-5384")
 @pytest.mark.parametrize(
@@ -39,6 +42,9 @@ def test_workflow_with_ado_work_item_get_tools_direct(
     similarity_check.check_similarity(response, expected_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-5384")
 @pytest.mark.parametrize(
@@ -66,6 +72,9 @@ def test_workflow_with_ado_work_item_get_tools_with_hardcoded_args(
     similarity_check.check_similarity(response, expected_response, 95)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.ado
 @pytest.mark.regression
 @pytest.mark.testcase("EPMCDME-5384")
 @pytest.mark.parametrize(

codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py

@@ -11,6 +11,9 @@ from codemie_test_harness.tests.test_data.direct_tools.cloud_tools_test_data imp
 from codemie_test_harness.tests.utils.base_utils import get_random_name
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.cloud
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,credential_type,credentials,prompt,expected_response",
@@ -48,6 +51,9 @@ def test_workflow_with_cloud_tools_direct(
     similarity_check.check_similarity(response, expected_response)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.cloud
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,credential_type,credentials,prompt,expected_response",
@@ -83,6 +89,9 @@ def test_workflow_with_cloud_tools_with_hardcoded_args(
     similarity_check.check_similarity(response, expected_response)
 
 
+@pytest.mark.workflow
+@pytest.mark.direct_tool
+@pytest.mark.cloud
 @pytest.mark.regression
 @pytest.mark.parametrize(
     "toolkit,tool_name,credential_type,credentials,prompt,expected_response",