codemie-test-harness 0.1.178__py3-none-any.whl → 0.1.180__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (135) hide show
  1. codemie_test_harness/cli/cli.py +2 -0
  2. codemie_test_harness/cli/commands/marks_cmd.py +39 -0
  3. codemie_test_harness/cli/marks_utils.py +338 -0
  4. codemie_test_harness/cli/runner.py +40 -0
  5. codemie_test_harness/tests/assistant/datasource/test_code_datasource.py +3 -3
  6. codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py +3 -3
  7. codemie_test_harness/tests/assistant/datasource/test_file_indexing.py +5 -5
  8. codemie_test_harness/tests/assistant/datasource/test_google_datasource.py +3 -3
  9. codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py +3 -3
  10. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool.py +8 -8
  11. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py +10 -10
  12. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py +8 -8
  13. codemie_test_harness/tests/assistant/test_assistants.py +22 -22
  14. codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py +1 -1
  15. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_test_plan_tools.py +2 -2
  16. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_wiki_tools.py +2 -2
  17. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_work_item_tools.py +2 -2
  18. codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py +1 -1
  19. codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py +2 -2
  20. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +2 -2
  21. codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py +4 -4
  22. codemie_test_harness/tests/assistant/tools/git/test_assistant_with_git_tools.py +7 -7
  23. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +2 -2
  24. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +2 -2
  25. codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py +2 -2
  26. codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py +1 -1
  27. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_development_plugin.py +2 -2
  28. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_plugin_and_mcp_servers.py +2 -2
  29. codemie_test_harness/tests/assistant/tools/{mcp → plugin}/test_single_assistant_dual_time_plugins.py +1 -1
  30. codemie_test_harness/tests/assistant/tools/project_management/test_assistant_pm_tools.py +6 -6
  31. codemie_test_harness/tests/assistant/tools/report_portal/test_assistant_report_portal_tools.py +1 -1
  32. codemie_test_harness/tests/assistant/tools/research/test_assistant_research_tools.py +2 -2
  33. codemie_test_harness/tests/assistant/tools/servicenow/test_servicenow_tools.py +1 -1
  34. codemie_test_harness/tests/assistant/tools/vcs/test_assistant_with_vcs_tools.py +1 -1
  35. codemie_test_harness/tests/conversations/test_conversations_endpoints.py +5 -5
  36. codemie_test_harness/tests/e2e/test_e2e.py +6 -6
  37. codemie_test_harness/tests/integrations/project/test_default_integrations.py +14 -14
  38. codemie_test_harness/tests/integrations/project/test_project_integrations.py +9 -9
  39. codemie_test_harness/tests/integrations/user/test_default_integrations.py +14 -14
  40. codemie_test_harness/tests/integrations/user/test_user_integrations.py +12 -12
  41. codemie_test_harness/tests/llm/assistants/test_lite_llm.py +2 -2
  42. codemie_test_harness/tests/llm/assistants/test_llm.py +3 -3
  43. codemie_test_harness/tests/providers/test_providers_endpoints.py +11 -11
  44. codemie_test_harness/tests/search/test_search_assistant.py +1 -1
  45. codemie_test_harness/tests/search/test_search_datasource.py +5 -5
  46. codemie_test_harness/tests/search/test_search_integration.py +2 -2
  47. codemie_test_harness/tests/search/test_search_workflow.py +1 -1
  48. codemie_test_harness/tests/service/test_assistant_service.py +1 -1
  49. codemie_test_harness/tests/test_data/ado_work_item_tools_test_data.py +3 -3
  50. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +11 -5
  51. codemie_test_harness/tests/ui/assistants/test_create_assistant.py +11 -11
  52. codemie_test_harness/tests/ui/datasource/test_create_datasource.py +19 -16
  53. codemie_test_harness/tests/ui/datasource/test_datasource_page.py +5 -5
  54. codemie_test_harness/tests/ui/datasource/test_edit_datasource.py +15 -11
  55. codemie_test_harness/tests/ui/datasource/test_view_datasource.py +11 -9
  56. codemie_test_harness/tests/ui/integrations/test_create_integration.py +9 -9
  57. codemie_test_harness/tests/ui/pageobject/datasources/create_edit_datasource_page.py +12 -5
  58. codemie_test_harness/tests/ui/pageobject/datasources/datasource_page.py +6 -2
  59. codemie_test_harness/tests/ui/pageobject/datasources/datasource_sidebar.py +2 -2
  60. codemie_test_harness/tests/ui/pageobject/datasources/view_datasource_page.py +3 -1
  61. codemie_test_harness/tests/ui/workflows/test_create_workflow.py +12 -12
  62. codemie_test_harness/tests/ui/workflows/test_edit_workflow.py +15 -15
  63. codemie_test_harness/tests/ui/workflows/test_workflow_details.py +19 -19
  64. codemie_test_harness/tests/ui/workflows/test_workflow_executions_page.py +36 -36
  65. codemie_test_harness/tests/ui/workflows/test_workflow_templates.py +8 -8
  66. codemie_test_harness/tests/ui/workflows/test_workflows.py +8 -8
  67. codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +1 -1
  68. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +2 -2
  69. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +2 -2
  70. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +2 -2
  71. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +1 -1
  72. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +2 -2
  73. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +2 -2
  74. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +8 -8
  75. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +10 -10
  76. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +8 -8
  77. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +5 -5
  78. codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +7 -7
  79. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +3 -3
  80. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +2 -2
  81. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +1 -1
  82. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +2 -2
  83. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +2 -2
  84. codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +1 -1
  85. codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +1 -1
  86. codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +2 -2
  87. codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +1 -1
  88. codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +1 -1
  89. codemie_test_harness/tests/workflow/config_validation/test_config_validation.py +1 -1
  90. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool.py +8 -8
  91. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool_kit.py +8 -8
  92. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_access_management_tool.py +3 -3
  93. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_test_plan_tools.py +3 -3
  94. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_wiki_tools.py +3 -3
  95. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_work_item_tools.py +3 -3
  96. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +3 -3
  97. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_codebase_tools.py +3 -3
  98. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_elastic.py +3 -3
  99. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_sql.py +3 -3
  100. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_file_management_tools.py +3 -3
  101. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py +3 -3
  102. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_open_api_tools.py +3 -3
  103. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_plugin_tools.py +4 -4
  104. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_project_management_tools.py +3 -3
  105. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_report_portal_tools.py +3 -3
  106. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_research_tools.py +3 -3
  107. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_servicenow_tools.py +3 -3
  108. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_vcs_tools.py +5 -3
  109. codemie_test_harness/tests/workflow/test_workflows.py +1 -1
  110. codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +1 -1
  111. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +2 -2
  112. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +2 -2
  113. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +2 -2
  114. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +4 -4
  115. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +2 -2
  116. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +2 -2
  117. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +8 -8
  118. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +10 -10
  119. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +8 -8
  120. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +5 -5
  121. codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +7 -7
  122. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +3 -3
  123. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +4 -4
  124. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +2 -2
  125. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +2 -2
  126. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +2 -2
  127. codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +2 -2
  128. codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +1 -1
  129. codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +2 -2
  130. codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +1 -1
  131. codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +1 -1
  132. {codemie_test_harness-0.1.178.dist-info → codemie_test_harness-0.1.180.dist-info}/METADATA +8 -8
  133. {codemie_test_harness-0.1.178.dist-info → codemie_test_harness-0.1.180.dist-info}/RECORD +135 -133
  134. {codemie_test_harness-0.1.178.dist-info → codemie_test_harness-0.1.180.dist-info}/WHEEL +0 -0
  135. {codemie_test_harness-0.1.178.dist-info → codemie_test_harness-0.1.180.dist-info}/entry_points.txt +0 -0
@@ -7,7 +7,7 @@ from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
7
7
 
8
8
  @pytest.mark.assistant
9
9
  @pytest.mark.report_portal
10
- @pytest.mark.regression
10
+ @pytest.mark.api
11
11
  @pytest.mark.parametrize(
12
12
  "toolkit,tool_name,prompt,expected_response",
13
13
  rp_test_data,
@@ -11,7 +11,7 @@ from codemie_test_harness.tests.utils.base_utils import percent_of_relevant_titl
11
11
 
12
12
  @pytest.mark.assistant
13
13
  @pytest.mark.research
14
- @pytest.mark.regression
14
+ @pytest.mark.api
15
15
  @pytest.mark.parametrize(
16
16
  "tool_name,prompt,expected_percentage",
17
17
  search_tools_test_data,
@@ -39,7 +39,7 @@ def test_assistant_with_search_tools(
39
39
 
40
40
  @pytest.mark.assistant
41
41
  @pytest.mark.research
42
- @pytest.mark.regression
42
+ @pytest.mark.api
43
43
  @pytest.mark.parametrize(
44
44
  "tool_name,prompt,expected_response",
45
45
  interactions_tools_test_data,
@@ -9,7 +9,7 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
9
9
 
10
10
  @pytest.mark.assistant
11
11
  @pytest.mark.servicenow
12
- @pytest.mark.regression
12
+ @pytest.mark.api
13
13
  def test_assistant_with_servicenow_tools(
14
14
  assistant_utils,
15
15
  assistant,
@@ -11,7 +11,7 @@ from codemie_test_harness.tests.utils.constants import vcs_integrations
11
11
 
12
12
  @pytest.mark.assistant
13
13
  @pytest.mark.vcs
14
- @pytest.mark.regression
14
+ @pytest.mark.api
15
15
  @pytest.mark.testcase("EPMCDME-5803, EPMCDME-5804")
16
16
  @pytest.mark.parametrize(
17
17
  "tool_name,prompt,expected_response",
@@ -17,7 +17,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
17
17
 
18
18
  @pytest.mark.assistant
19
19
  @pytest.mark.conversations
20
- @pytest.mark.regression
20
+ @pytest.mark.api
21
21
  def test_list_conversations(assistant, assistant_utils, conversation_utils):
22
22
  assistant = assistant()
23
23
  assistant_utils.ask_assistant(assistant, f"prompt {get_random_name()}")
@@ -45,7 +45,7 @@ def test_list_conversations(assistant, assistant_utils, conversation_utils):
45
45
 
46
46
  @pytest.mark.assistant
47
47
  @pytest.mark.conversations
48
- @pytest.mark.regression
48
+ @pytest.mark.api
49
49
  def test_get_specific_conversation(assistant, assistant_utils, conversation_utils):
50
50
  prompt = f"prompt {get_random_name()}"
51
51
  assistant = assistant()
@@ -69,7 +69,7 @@ def test_get_specific_conversation(assistant, assistant_utils, conversation_util
69
69
 
70
70
  @pytest.mark.assistant
71
71
  @pytest.mark.conversations
72
- @pytest.mark.regression
72
+ @pytest.mark.api
73
73
  def test_get_conversation_by_assistant_id(
74
74
  assistant, assistant_utils, conversation_utils
75
75
  ):
@@ -92,7 +92,7 @@ def test_get_conversation_by_assistant_id(
92
92
 
93
93
  @pytest.mark.assistant
94
94
  @pytest.mark.conversations
95
- @pytest.mark.regression
95
+ @pytest.mark.api
96
96
  def test_delete_conversation(assistant, assistant_utils, conversation_utils):
97
97
  prompt = f"prompt {get_random_name()}"
98
98
  assistant = assistant()
@@ -110,7 +110,7 @@ def test_delete_conversation(assistant, assistant_utils, conversation_utils):
110
110
 
111
111
  @pytest.mark.assistant
112
112
  @pytest.mark.conversations
113
- @pytest.mark.regression
113
+ @pytest.mark.api
114
114
  def test_get_non_existent_conversation(assistant, assistant_utils, conversation_utils):
115
115
  invalid_id = get_random_name()
116
116
  with pytest.raises(Exception) as exc_info:
@@ -35,7 +35,7 @@ def git_integration(integration_utils):
35
35
  @pytest.mark.assistant
36
36
  @pytest.mark.code_kb
37
37
  @pytest.mark.e2e
38
- @pytest.mark.regression
38
+ @pytest.mark.api
39
39
  def test_assistant_with_code_kb(
40
40
  assistant_utils, code_datasource, default_llm, similarity_check
41
41
  ):
@@ -59,7 +59,7 @@ def test_assistant_with_code_kb(
59
59
  @pytest.mark.vcs
60
60
  @pytest.mark.gitlab
61
61
  @pytest.mark.e2e
62
- @pytest.mark.regression
62
+ @pytest.mark.api
63
63
  @pytest.mark.skipif(
64
64
  os.getenv("GIT_ENV") == "github",
65
65
  reason="Test is skipped when GIT_ENV is set to github",
@@ -93,7 +93,7 @@ def test_assistant_with_vcs_gitlab_tool(
93
93
  @pytest.mark.vcs
94
94
  @pytest.mark.github
95
95
  @pytest.mark.e2e
96
- @pytest.mark.regression
96
+ @pytest.mark.api
97
97
  @pytest.mark.skipif(
98
98
  os.getenv("GIT_ENV") == "gitlab",
99
99
  reason="Test is skipped when GIT_ENV is set to gitlab",
@@ -126,7 +126,7 @@ def test_assistant_with_vcs_github_tool(
126
126
  @pytest.mark.assistant
127
127
  @pytest.mark.gitlab
128
128
  @pytest.mark.e2e
129
- @pytest.mark.regression
129
+ @pytest.mark.api
130
130
  def test_assistant_with_list_branches_tool(
131
131
  assistant_utils, code_datasource, default_llm, similarity_check
132
132
  ):
@@ -158,7 +158,7 @@ def test_assistant_with_list_branches_tool(
158
158
  @pytest.mark.jira
159
159
  @pytest.mark.project_management
160
160
  @pytest.mark.e2e
161
- @pytest.mark.regression
161
+ @pytest.mark.api
162
162
  def test_assistant_with_jira_kb(
163
163
  assistant_utils, jira_datasource, default_llm, similarity_check
164
164
  ):
@@ -184,7 +184,7 @@ def test_assistant_with_jira_kb(
184
184
  @pytest.mark.confluence
185
185
  @pytest.mark.project_management
186
186
  @pytest.mark.e2e
187
- @pytest.mark.regression
187
+ @pytest.mark.api
188
188
  def test_assistant_with_confluence_kb(
189
189
  assistant_utils, confluence_datasource, default_llm, similarity_check
190
190
  ):
@@ -53,8 +53,8 @@ pytestmark = pytest.mark.skipif(
53
53
  @pytest.mark.integration
54
54
  @pytest.mark.project_integration
55
55
  @pytest.mark.default_integration
56
- @pytest.mark.regression
57
- @pytest.mark.tescase("EPMCDME-2377")
56
+ @pytest.mark.api
57
+ @pytest.mark.testcase("EPMCDME-2377")
58
58
  @pytest.mark.parametrize(
59
59
  "toolkit, tool_name, credential_type, credentials, prompt, expected_response",
60
60
  cloud_test_data,
@@ -97,8 +97,8 @@ def test_assistant_with_default_integration_cloud(
97
97
  @pytest.mark.integration
98
98
  @pytest.mark.project_integration
99
99
  @pytest.mark.default_integration
100
- @pytest.mark.regression
101
- @pytest.mark.tescase("EPMCDME-2377")
100
+ @pytest.mark.api
101
+ @pytest.mark.testcase("EPMCDME-2377")
102
102
  def test_assistant_with_default_integration_ado(
103
103
  general_integration,
104
104
  integration_utils,
@@ -135,8 +135,8 @@ def test_assistant_with_default_integration_ado(
135
135
  @pytest.mark.integration
136
136
  @pytest.mark.project_integration
137
137
  @pytest.mark.default_integration
138
- @pytest.mark.regression
139
- @pytest.mark.tescase("EPMCDME-2377")
138
+ @pytest.mark.api
139
+ @pytest.mark.testcase("EPMCDME-2377")
140
140
  @pytest.mark.parametrize(
141
141
  "toolkit, tool_name, credentials, prompt, expected_response",
142
142
  sonar_tools_test_data,
@@ -180,8 +180,8 @@ def test_assistant_with_default_integration_codebase(
180
180
  @pytest.mark.integration
181
181
  @pytest.mark.project_integration
182
182
  @pytest.mark.default_integration
183
- @pytest.mark.regression
184
- @pytest.mark.tescase("EPMCDME-2377")
183
+ @pytest.mark.api
184
+ @pytest.mark.testcase("EPMCDME-2377")
185
185
  def test_assistant_with_default_integration_git(
186
186
  general_integration,
187
187
  integration_utils,
@@ -234,8 +234,8 @@ def test_assistant_with_default_integration_git(
234
234
  @pytest.mark.integration
235
235
  @pytest.mark.project_integration
236
236
  @pytest.mark.default_integration
237
- @pytest.mark.regression
238
- @pytest.mark.tescase("EPMCDME-2377")
237
+ @pytest.mark.api
238
+ @pytest.mark.testcase("EPMCDME-2377")
239
239
  def test_assistant_with_default_integration_jira(
240
240
  general_integration,
241
241
  integration_utils,
@@ -273,8 +273,8 @@ def test_assistant_with_default_integration_jira(
273
273
  @pytest.mark.integration
274
274
  @pytest.mark.project_integration
275
275
  @pytest.mark.default_integration
276
- @pytest.mark.regression
277
- @pytest.mark.tescase("EPMCDME-2377")
276
+ @pytest.mark.api
277
+ @pytest.mark.testcase("EPMCDME-2377")
278
278
  def test_assistant_with_default_integration_email(
279
279
  general_integration,
280
280
  integration_utils,
@@ -309,7 +309,7 @@ def test_assistant_with_default_integration_email(
309
309
  @pytest.mark.integration
310
310
  @pytest.mark.project_integration
311
311
  @pytest.mark.default_integration
312
- @pytest.mark.regression
312
+ @pytest.mark.api
313
313
  def test_assistant_with_default_integration_keycloak(
314
314
  general_integration,
315
315
  integration_utils,
@@ -346,7 +346,7 @@ def test_assistant_with_default_integration_keycloak(
346
346
  @pytest.mark.integration
347
347
  @pytest.mark.project_integration
348
348
  @pytest.mark.default_integration
349
- @pytest.mark.regression
349
+ @pytest.mark.api
350
350
  def test_assistant_with_default_integration_report_portal(
351
351
  general_integration,
352
352
  integration_utils,
@@ -22,7 +22,7 @@ from codemie_test_harness.tests.utils.base_utils import (
22
22
 
23
23
  @pytest.mark.integration
24
24
  @pytest.mark.project_integration
25
- @pytest.mark.regression
25
+ @pytest.mark.api
26
26
  @pytest.mark.parametrize(
27
27
  "credential_type, credentials",
28
28
  valid_integrations,
@@ -54,7 +54,7 @@ def test_integration_created(
54
54
 
55
55
  @pytest.mark.integration
56
56
  @pytest.mark.project_integration
57
- @pytest.mark.regression
57
+ @pytest.mark.api
58
58
  @pytest.mark.parametrize(
59
59
  "credential_type, credentials",
60
60
  testable_integrations,
@@ -91,8 +91,8 @@ def test_integration_after_creation(
91
91
 
92
92
  @pytest.mark.integration
93
93
  @pytest.mark.project_integration
94
- @pytest.mark.regression
95
- @pytest.mark.tescase("EPMCDME-2376")
94
+ @pytest.mark.api
95
+ @pytest.mark.testcase("EPMCDME-2376")
96
96
  @pytest.mark.parametrize(
97
97
  "credential_type, credentials",
98
98
  valid_integrations,
@@ -136,7 +136,7 @@ def test_update_integration(
136
136
 
137
137
  @pytest.mark.integration
138
138
  @pytest.mark.project_integration
139
- @pytest.mark.regression
139
+ @pytest.mark.api
140
140
  @pytest.mark.parametrize(
141
141
  "credential_type, credentials, error_message",
142
142
  invalid_integrations,
@@ -177,7 +177,7 @@ def test_integration_with_invalid_credentials(
177
177
 
178
178
  @pytest.mark.integration
179
179
  @pytest.mark.project_integration
180
- @pytest.mark.regression
180
+ @pytest.mark.api
181
181
  @pytest.mark.parametrize(
182
182
  "credential_type, credentials",
183
183
  testable_integrations,
@@ -201,7 +201,7 @@ def test_integration_during_creation(credential_type, credentials, integration_u
201
201
 
202
202
  @pytest.mark.integration
203
203
  @pytest.mark.project_integration
204
- @pytest.mark.regression
204
+ @pytest.mark.api
205
205
  @pytest.mark.parametrize(
206
206
  "credential_type, credentials, error_message",
207
207
  invalid_integrations,
@@ -223,8 +223,8 @@ def test_integration_during_creation_with_invalid_credentials(
223
223
 
224
224
  @pytest.mark.integration
225
225
  @pytest.mark.project_integration
226
- @pytest.mark.regression
227
- @pytest.mark.tescase("EPMCDME-2374")
226
+ @pytest.mark.api
227
+ @pytest.mark.testcase("EPMCDME-2374")
228
228
  @pytest.mark.parametrize(
229
229
  "credential_type, credentials",
230
230
  valid_integrations,
@@ -53,8 +53,8 @@ pytestmark = pytest.mark.skipif(
53
53
  @pytest.mark.integration
54
54
  @pytest.mark.user_integration
55
55
  @pytest.mark.default_integration
56
- @pytest.mark.regression
57
- @pytest.mark.tescase("EPMCDME-2377")
56
+ @pytest.mark.api
57
+ @pytest.mark.testcase("EPMCDME-2377")
58
58
  @pytest.mark.parametrize(
59
59
  "toolkit, tool_name, credential_type, credentials, prompt, expected_response",
60
60
  cloud_test_data,
@@ -97,8 +97,8 @@ def test_assistant_with_default_integration_cloud(
97
97
  @pytest.mark.integration
98
98
  @pytest.mark.user_integration
99
99
  @pytest.mark.default_integration
100
- @pytest.mark.regression
101
- @pytest.mark.tescase("EPMCDME-2377")
100
+ @pytest.mark.api
101
+ @pytest.mark.testcase("EPMCDME-2377")
102
102
  def test_assistant_with_default_integration_ado(
103
103
  general_integration,
104
104
  integration_utils,
@@ -135,8 +135,8 @@ def test_assistant_with_default_integration_ado(
135
135
  @pytest.mark.integration
136
136
  @pytest.mark.user_integration
137
137
  @pytest.mark.default_integration
138
- @pytest.mark.regression
139
- @pytest.mark.tescase("EPMCDME-2377")
138
+ @pytest.mark.api
139
+ @pytest.mark.testcase("EPMCDME-2377")
140
140
  @pytest.mark.parametrize(
141
141
  "toolkit, tool_name, credentials, prompt, expected_response",
142
142
  sonar_tools_test_data,
@@ -180,8 +180,8 @@ def test_assistant_with_default_integration_codebase(
180
180
  @pytest.mark.integration
181
181
  @pytest.mark.user_integration
182
182
  @pytest.mark.default_integration
183
- @pytest.mark.regression
184
- @pytest.mark.tescase("EPMCDME-2377")
183
+ @pytest.mark.api
184
+ @pytest.mark.testcase("EPMCDME-2377")
185
185
  def test_assistant_with_default_integration_git(
186
186
  general_integration,
187
187
  integration_utils,
@@ -234,8 +234,8 @@ def test_assistant_with_default_integration_git(
234
234
  @pytest.mark.integration
235
235
  @pytest.mark.user_integration
236
236
  @pytest.mark.default_integration
237
- @pytest.mark.regression
238
- @pytest.mark.tescase("EPMCDME-2377")
237
+ @pytest.mark.api
238
+ @pytest.mark.testcase("EPMCDME-2377")
239
239
  def test_assistant_with_default_integration_jira(
240
240
  general_integration,
241
241
  integration_utils,
@@ -273,8 +273,8 @@ def test_assistant_with_default_integration_jira(
273
273
  @pytest.mark.integration
274
274
  @pytest.mark.user_integration
275
275
  @pytest.mark.default_integration
276
- @pytest.mark.regression
277
- @pytest.mark.tescase("EPMCDME-2377")
276
+ @pytest.mark.api
277
+ @pytest.mark.testcase("EPMCDME-2377")
278
278
  def test_assistant_with_default_integration_email(
279
279
  general_integration,
280
280
  integration_utils,
@@ -309,7 +309,7 @@ def test_assistant_with_default_integration_email(
309
309
  @pytest.mark.integration
310
310
  @pytest.mark.user_integration
311
311
  @pytest.mark.default_integration
312
- @pytest.mark.regression
312
+ @pytest.mark.api
313
313
  def test_assistant_with_default_integration_keycloak(
314
314
  general_integration,
315
315
  integration_utils,
@@ -346,7 +346,7 @@ def test_assistant_with_default_integration_keycloak(
346
346
  @pytest.mark.integration
347
347
  @pytest.mark.user_integration
348
348
  @pytest.mark.default_integration
349
- @pytest.mark.regression
349
+ @pytest.mark.api
350
350
  def test_assistant_with_default_integration_report_portal(
351
351
  general_integration,
352
352
  integration_utils,
@@ -22,8 +22,8 @@ from codemie_test_harness.tests.utils.base_utils import (
22
22
 
23
23
  @pytest.mark.integration
24
24
  @pytest.mark.user_integration
25
- @pytest.mark.regression
26
- @pytest.mark.tescase("EPMCDME-2486, EPMCDME-2487, EPMCDME-2387, EPMCDME-2388")
25
+ @pytest.mark.api
26
+ @pytest.mark.testcase("EPMCDME-2486, EPMCDME-2487, EPMCDME-2387, EPMCDME-2388")
27
27
  @pytest.mark.parametrize(
28
28
  "credential_type, credentials",
29
29
  valid_integrations,
@@ -55,8 +55,8 @@ def test_integration_created(
55
55
 
56
56
  @pytest.mark.integration
57
57
  @pytest.mark.user_integration
58
- @pytest.mark.regression
59
- @pytest.mark.tescase("EPMCDME-2381, EPMCDME-2380, EPMCDME-3107")
58
+ @pytest.mark.api
59
+ @pytest.mark.testcase("EPMCDME-2381, EPMCDME-2380, EPMCDME-3107")
60
60
  @pytest.mark.parametrize(
61
61
  "credential_type, credentials",
62
62
  testable_integrations,
@@ -93,8 +93,8 @@ def test_integration_after_creation(
93
93
 
94
94
  @pytest.mark.integration
95
95
  @pytest.mark.user_integration
96
- @pytest.mark.regression
97
- @pytest.mark.tescase("EPMCDME-2375")
96
+ @pytest.mark.api
97
+ @pytest.mark.testcase("EPMCDME-2375")
98
98
  @pytest.mark.parametrize(
99
99
  "credential_type, credentials",
100
100
  valid_integrations,
@@ -138,8 +138,8 @@ def test_update_integration(
138
138
 
139
139
  @pytest.mark.integration
140
140
  @pytest.mark.user_integration
141
- @pytest.mark.regression
142
- @pytest.mark.tescase("EPMCDME-3111, EPMCDME-2379, EPMCDME-2384")
141
+ @pytest.mark.api
142
+ @pytest.mark.testcase("EPMCDME-3111, EPMCDME-2379, EPMCDME-2384")
143
143
  @pytest.mark.parametrize(
144
144
  "credential_type, credentials, error_message",
145
145
  invalid_integrations,
@@ -180,7 +180,7 @@ def test_integration_with_invalid_credentials(
180
180
 
181
181
  @pytest.mark.integration
182
182
  @pytest.mark.user_integration
183
- @pytest.mark.regression
183
+ @pytest.mark.api
184
184
  @pytest.mark.parametrize(
185
185
  "credential_type, credentials",
186
186
  testable_integrations,
@@ -204,7 +204,7 @@ def test_integration_during_creation(credential_type, credentials, integration_u
204
204
 
205
205
  @pytest.mark.integration
206
206
  @pytest.mark.user_integration
207
- @pytest.mark.regression
207
+ @pytest.mark.api
208
208
  @pytest.mark.parametrize(
209
209
  "credential_type, credentials, error_message",
210
210
  invalid_integrations,
@@ -226,8 +226,8 @@ def test_integration_during_creation_with_invalid_credentials(
226
226
 
227
227
  @pytest.mark.integration
228
228
  @pytest.mark.user_integration
229
- @pytest.mark.regression
230
- @pytest.mark.tescase("EPMCDME-2373")
229
+ @pytest.mark.api
230
+ @pytest.mark.testcase("EPMCDME-2373")
231
231
  @pytest.mark.parametrize(
232
232
  "credential_type, credentials",
233
233
  valid_integrations,
@@ -70,7 +70,7 @@ def pytest_generate_tests(metafunc):
70
70
 
71
71
  @pytest.mark.assistant
72
72
  @pytest.mark.lite_llm
73
- @pytest.mark.regression
73
+ @pytest.mark.api
74
74
  def test_assistant_with_different_models_in_lite_llm(
75
75
  llm_utils,
76
76
  lite_llm_integration,
@@ -93,7 +93,7 @@ def test_assistant_with_different_models_in_lite_llm(
93
93
 
94
94
  @pytest.mark.assistant
95
95
  @pytest.mark.lite_llm
96
- @pytest.mark.regression
96
+ @pytest.mark.api
97
97
  def test_assistant_with_invalid_lite_llm(
98
98
  invalid_lite_llm_integration,
99
99
  assistant,
@@ -33,7 +33,7 @@ def pytest_generate_tests(metafunc):
33
33
 
34
34
  @pytest.mark.assistant
35
35
  @pytest.mark.llm
36
- @pytest.mark.regression
36
+ @pytest.mark.api
37
37
  @pytest.mark.smoke
38
38
  def test_assistant_with_different_models(
39
39
  llm_utils, assistant_utils, model_type, similarity_check
@@ -54,7 +54,7 @@ def test_assistant_with_different_models(
54
54
  @pytest.mark.assistant
55
55
  @pytest.mark.llm
56
56
  @pytest.mark.top_p
57
- @pytest.mark.regression
57
+ @pytest.mark.api
58
58
  @pytest.mark.smoke
59
59
  def test_assistant_with_different_models_with_top_p_parameter(
60
60
  llm_utils, assistant_utils, model_type, similarity_check
@@ -75,7 +75,7 @@ def test_assistant_with_different_models_with_top_p_parameter(
75
75
  @pytest.mark.assistant
76
76
  @pytest.mark.llm
77
77
  @pytest.mark.temperature
78
- @pytest.mark.regression
78
+ @pytest.mark.api
79
79
  @pytest.mark.smoke
80
80
  def test_assistant_with_different_models_with_temperature_parameter(
81
81
  llm_utils, assistant_utils, model_type, similarity_check
@@ -24,7 +24,7 @@ nonexistent_id = "nonexistent-provider-id-12345"
24
24
 
25
25
 
26
26
  @pytest.mark.provider
27
- @pytest.mark.regression
27
+ @pytest.mark.api
28
28
  def test_get_providers_endpoint(providers_utils):
29
29
  response = providers_utils.send_get_request_to_providers_endpoint()
30
30
 
@@ -47,7 +47,7 @@ def test_get_providers_endpoint(providers_utils):
47
47
 
48
48
 
49
49
  @pytest.mark.provider
50
- @pytest.mark.regression
50
+ @pytest.mark.api
51
51
  def test_post_providers_endpoint(providers_utils):
52
52
  request_json = providers_utils.provider_request_json()
53
53
  # Ensure unique provider name to avoid conflicts
@@ -74,7 +74,7 @@ def test_post_providers_endpoint(providers_utils):
74
74
 
75
75
 
76
76
  @pytest.mark.provider
77
- @pytest.mark.regression
77
+ @pytest.mark.api
78
78
  def test_post_providers_endpoint_validation_error(providers_utils):
79
79
  """Test POST /v1/providers with invalid provided_toolkits node"""
80
80
  request_json = providers_utils.provider_request_json()
@@ -90,7 +90,7 @@ def test_post_providers_endpoint_validation_error(providers_utils):
90
90
 
91
91
 
92
92
  @pytest.mark.provider
93
- @pytest.mark.regression
93
+ @pytest.mark.api
94
94
  def test_post_providers_endpoint_with_existing_name(providers_utils):
95
95
  provider_id = providers_utils.create_provider()
96
96
 
@@ -117,7 +117,7 @@ def test_post_providers_endpoint_with_existing_name(providers_utils):
117
117
 
118
118
 
119
119
  @pytest.mark.provider
120
- @pytest.mark.regression
120
+ @pytest.mark.api
121
121
  def test_get_providers_datasource_schema_endpoint(providers_utils):
122
122
  response = providers_utils.send_get_request_datasource_schemas_endpoint()
123
123
 
@@ -146,7 +146,7 @@ def test_get_providers_datasource_schema_endpoint(providers_utils):
146
146
 
147
147
 
148
148
  @pytest.mark.provider
149
- @pytest.mark.regression
149
+ @pytest.mark.api
150
150
  def test_get_provider_by_id_endpoint(providers_utils):
151
151
  provider_id = providers_utils.create_provider()
152
152
 
@@ -157,7 +157,7 @@ def test_get_provider_by_id_endpoint(providers_utils):
157
157
 
158
158
 
159
159
  @pytest.mark.provider
160
- @pytest.mark.regression
160
+ @pytest.mark.api
161
161
  def test_get_nonexistent_provider(providers_utils):
162
162
  """Test error handling when requesting nonexistent provider."""
163
163
  response = providers_utils.get_provider_by_id(nonexistent_id)
@@ -178,7 +178,7 @@ def test_get_nonexistent_provider(providers_utils):
178
178
 
179
179
 
180
180
  @pytest.mark.provider
181
- @pytest.mark.regression
181
+ @pytest.mark.api
182
182
  def test_put_provider_endpoint(providers_utils):
183
183
  provider_id = providers_utils.create_provider()
184
184
  provider = providers_utils.get_provider_by_id(provider_id).json()
@@ -196,7 +196,7 @@ def test_put_provider_endpoint(providers_utils):
196
196
 
197
197
 
198
198
  @pytest.mark.provider
199
- @pytest.mark.regression
199
+ @pytest.mark.api
200
200
  def test_put_nonexistent_provider(providers_utils):
201
201
  response = providers_utils.update_provider(nonexistent_id, {})
202
202
 
@@ -215,7 +215,7 @@ def test_put_nonexistent_provider(providers_utils):
215
215
 
216
216
 
217
217
  @pytest.mark.provider
218
- @pytest.mark.regression
218
+ @pytest.mark.api
219
219
  def test_delete_provider_endpoint(providers_utils):
220
220
  provider_id = providers_utils.create_provider()
221
221
  delete_response = providers_utils.send_delete_provider_request(provider_id)
@@ -235,7 +235,7 @@ def test_delete_provider_endpoint(providers_utils):
235
235
 
236
236
 
237
237
  @pytest.mark.provider
238
- @pytest.mark.regression
238
+ @pytest.mark.api
239
239
  def test_delete_nonexistent_provider(providers_utils):
240
240
  """Test error handling when requesting nonexistent provider."""
241
241
  response = providers_utils.send_delete_provider_request(nonexistent_id)
@@ -84,7 +84,7 @@ def pytest_generate_tests(metafunc):
84
84
  @pytest.mark.assistant
85
85
  @pytest.mark.search
86
86
  @pytest.mark.testcase("EPMCDME-2429, EPMCDME-4102")
87
- @pytest.mark.regression
87
+ @pytest.mark.api
88
88
  @pytest.mark.smoke
89
89
  def test_search_assistant_by_filters(search_utils, filters, test_assistant_name):
90
90
  response = search_utils.list_assistants(filters)
@@ -152,7 +152,7 @@ def pytest_generate_tests(metafunc):
152
152
  @pytest.mark.datasource
153
153
  @pytest.mark.search
154
154
  @pytest.mark.testcase("EPMCDME-2391")
155
- @pytest.mark.regression
155
+ @pytest.mark.api
156
156
  def test_search_datasource_by_name(
157
157
  search_utils, name_filters, datasource_name_for_search
158
158
  ):
@@ -165,7 +165,7 @@ def test_search_datasource_by_name(
165
165
  @pytest.mark.datasource
166
166
  @pytest.mark.search
167
167
  @pytest.mark.testcase("EPMCDME-2391")
168
- @pytest.mark.regression
168
+ @pytest.mark.api
169
169
  def test_search_datasource_by_type(
170
170
  search_utils, type_test_data, datasource_name_for_search
171
171
  ):
@@ -181,7 +181,7 @@ def test_search_datasource_by_type(
181
181
  @pytest.mark.datasource
182
182
  @pytest.mark.search
183
183
  @pytest.mark.testcase("EPMCDME-2391")
184
- @pytest.mark.regression
184
+ @pytest.mark.api
185
185
  def test_search_datasource_by_owner(
186
186
  search_utils, owner_test_data, datasource_name_for_search
187
187
  ):
@@ -197,7 +197,7 @@ def test_search_datasource_by_owner(
197
197
  @pytest.mark.datasource
198
198
  @pytest.mark.search
199
199
  @pytest.mark.testcase("EPMCDME-2391")
200
- @pytest.mark.regression
200
+ @pytest.mark.api
201
201
  def test_search_datasource_by_status(
202
202
  search_utils, status_test_data, datasource_name_for_search
203
203
  ):
@@ -211,7 +211,7 @@ def test_search_datasource_by_status(
211
211
  @pytest.mark.datasource
212
212
  @pytest.mark.search
213
213
  @pytest.mark.testcase("EPMCDME-2391")
214
- @pytest.mark.regression
214
+ @pytest.mark.api
215
215
  def test_search_datasource_by_project(
216
216
  search_utils, project_test_data, datasource_name_for_search
217
217
  ):
@@ -110,7 +110,7 @@ def pytest_generate_tests(metafunc):
110
110
  @pytest.mark.user_integration
111
111
  @pytest.mark.search
112
112
  @pytest.mark.testcase("EPMCDME-4243")
113
- @pytest.mark.regression
113
+ @pytest.mark.api
114
114
  def test_search_user_integration(search_utils, user_filters, user_integration_name):
115
115
  response = search_utils.list_integrations(
116
116
  setting_type=IntegrationType.USER, filters=user_filters
@@ -124,7 +124,7 @@ def test_search_user_integration(search_utils, user_filters, user_integration_na
124
124
  @pytest.mark.project_integration
125
125
  @pytest.mark.search
126
126
  @pytest.mark.testcase("EPMCDME-3083")
127
- @pytest.mark.regression
127
+ @pytest.mark.api
128
128
  def test_search_project_integration(
129
129
  search_utils, project_filters, project_integration_name
130
130
  ):
@@ -81,7 +81,7 @@ def pytest_generate_tests(metafunc):
81
81
  @pytest.mark.workflow
82
82
  @pytest.mark.search
83
83
  @pytest.mark.testcase("EPMCDME-3948")
84
- @pytest.mark.regression
84
+ @pytest.mark.api
85
85
  @pytest.mark.smoke
86
86
  def test_search_workflows_by_filters(
87
87
  search_utils, test_case_data, workflow_name_for_search