codemie-test-harness 0.1.155__py3-none-any.whl → 0.1.157__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (122) hide show
  1. codemie_test_harness/.env +1 -1
  2. codemie_test_harness/tests/assistant/datasource/test_code_datasource.py +6 -0
  3. codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py +14 -2
  4. codemie_test_harness/tests/assistant/datasource/test_file_indexing.py +10 -0
  5. codemie_test_harness/tests/assistant/datasource/test_google_datasource.py +6 -0
  6. codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py +18 -2
  7. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool.py +10 -0
  8. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py +10 -0
  9. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py +10 -0
  10. codemie_test_harness/tests/assistant/test_assistants.py +24 -2
  11. codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py +2 -0
  12. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_test_plan_tools.py +4 -0
  13. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_wiki_tools.py +4 -0
  14. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_work_item_tools.py +4 -0
  15. codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py +2 -0
  16. codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py +3 -0
  17. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +8 -4
  18. codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py +9 -1
  19. codemie_test_harness/tests/assistant/tools/git/test_assistant_with_git_tools.py +14 -0
  20. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +8 -6
  21. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +8 -6
  22. codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py +6 -0
  23. codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py +6 -4
  24. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_development_plugin.py +4 -1
  25. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_plugin_and_mcp_servers.py +6 -2
  26. codemie_test_harness/tests/assistant/tools/project_management/test_assistant_pm_tools.py +17 -0
  27. codemie_test_harness/tests/assistant/tools/research/test_assistant_research_tools.py +4 -0
  28. codemie_test_harness/tests/assistant/tools/servicenow/test_servicenow_tools.py +2 -0
  29. codemie_test_harness/tests/assistant/tools/vcs/test_assistant_with_vcs_tools.py +3 -1
  30. codemie_test_harness/tests/conversations/test_conversations_endpoints.py +10 -0
  31. codemie_test_harness/tests/e2e/test_e2e.py +24 -8
  32. codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -4
  33. codemie_test_harness/tests/integrations/project/test_project_integrations.py +23 -9
  34. codemie_test_harness/tests/integrations/user/test_default_integrations.py +40 -4
  35. codemie_test_harness/tests/integrations/user/test_user_integrations.py +26 -12
  36. codemie_test_harness/tests/llm/assistants/test_llm.py +8 -0
  37. codemie_test_harness/tests/providers/test_providers_endpoints.py +11 -0
  38. codemie_test_harness/tests/search/test_search_assistant.py +2 -0
  39. codemie_test_harness/tests/search/test_search_datasource.py +10 -0
  40. codemie_test_harness/tests/search/test_search_integration.py +6 -0
  41. codemie_test_harness/tests/search/test_search_workflow.py +2 -0
  42. codemie_test_harness/tests/test_data/cloud_tools_test_data.py +18 -10
  43. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +6 -2
  44. codemie_test_harness/tests/test_data/direct_tools/cloud_tools_test_data.py +18 -10
  45. codemie_test_harness/tests/test_data/direct_tools/codebase_tools_test_data.py +6 -2
  46. codemie_test_harness/tests/test_data/direct_tools/notification_tools_test_data.py +9 -5
  47. codemie_test_harness/tests/test_data/direct_tools/project_management_tools_test_data.py +6 -2
  48. codemie_test_harness/tests/test_data/direct_tools/vcs_tools_test_data.py +6 -2
  49. codemie_test_harness/tests/test_data/integrations_test_data.py +185 -65
  50. codemie_test_harness/tests/test_data/project_management_test_data.py +10 -4
  51. codemie_test_harness/tests/test_data/vcs_tools_test_data.py +8 -2
  52. codemie_test_harness/tests/ui/test_create_workflow.py +13 -0
  53. codemie_test_harness/tests/ui/test_edit_workflow.py +16 -0
  54. codemie_test_harness/tests/ui/test_workflow_details.py +17 -0
  55. codemie_test_harness/tests/ui/test_workflow_executions_page.py +36 -0
  56. codemie_test_harness/tests/ui/test_workflow_templates.py +8 -0
  57. codemie_test_harness/tests/ui/test_workflows.py +8 -0
  58. codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +3 -0
  59. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +6 -0
  60. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +6 -0
  61. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +6 -0
  62. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +3 -0
  63. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +5 -0
  64. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +6 -0
  65. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +20 -0
  66. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +20 -0
  67. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +25 -0
  68. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +15 -0
  69. codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +21 -0
  70. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +9 -3
  71. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +8 -0
  72. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +3 -0
  73. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +6 -2
  74. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +8 -2
  75. codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +3 -0
  76. codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +6 -0
  77. codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +3 -0
  78. codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +3 -0
  79. codemie_test_harness/tests/workflow/config_validation/test_config_validation.py +4 -3
  80. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool.py +17 -2
  81. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool_kit.py +15 -0
  82. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_access_management_tool.py +9 -0
  83. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_test_plan_tools.py +9 -0
  84. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_wiki_tools.py +9 -0
  85. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_work_item_tools.py +9 -0
  86. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +9 -0
  87. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_codebase_tools.py +6 -0
  88. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools.py +9 -0
  89. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_file_management_tools.py +9 -0
  90. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py +3 -0
  91. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_open_api_tools.py +9 -0
  92. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_plugin_tools.py +12 -4
  93. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_project_management_tools.py +9 -0
  94. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_research_tools.py +9 -0
  95. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_servicenow_tools.py +9 -0
  96. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_vcs_tools.py +3 -0
  97. codemie_test_harness/tests/workflow/test_workflows.py +1 -0
  98. codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +3 -0
  99. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +6 -0
  100. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +6 -0
  101. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +6 -0
  102. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +3 -0
  103. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +5 -0
  104. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +6 -0
  105. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +20 -0
  106. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +20 -0
  107. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +25 -0
  108. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +15 -0
  109. codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +21 -0
  110. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +12 -6
  111. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +8 -0
  112. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +6 -3
  113. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +6 -2
  114. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +8 -2
  115. codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +3 -0
  116. codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +6 -0
  117. codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +3 -0
  118. codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +3 -0
  119. {codemie_test_harness-0.1.155.dist-info → codemie_test_harness-0.1.157.dist-info}/METADATA +2 -2
  120. {codemie_test_harness-0.1.155.dist-info → codemie_test_harness-0.1.157.dist-info}/RECORD +122 -122
  121. {codemie_test_harness-0.1.155.dist-info → codemie_test_harness-0.1.157.dist-info}/WHEEL +0 -0
  122. {codemie_test_harness-0.1.155.dist-info → codemie_test_harness-0.1.157.dist-info}/entry_points.txt +0 -0
@@ -13,6 +13,8 @@ from codemie_test_harness.tests.test_data.git_tools_test_data import (
13
13
  from codemie_test_harness.tests.utils.base_utils import get_random_name, to_camel_case
14
14
 
15
15
 
16
+ @pytest.mark.assistant
17
+ @pytest.mark.gitlab
16
18
  @pytest.mark.regression
17
19
  @pytest.mark.parametrize(
18
20
  "toolkit,tool_name,prompt,expected_response",
@@ -42,6 +44,8 @@ def test_assistant_with_list_branch_set_active_branch_tools(
42
44
  similarity_check.check_similarity(response, expected_response)
43
45
 
44
46
 
47
+ @pytest.mark.assistant
48
+ @pytest.mark.gitlab
45
49
  @pytest.mark.regression
46
50
  @pytest.mark.parametrize(
47
51
  "toolkit,tool_name,prompt_template,expected_template",
@@ -86,6 +90,8 @@ def test_assistant_with_create_branch_tool(
86
90
  git_utils.delete_branch(branch_name)
87
91
 
88
92
 
93
+ @pytest.mark.assistant
94
+ @pytest.mark.gitlab
89
95
  @pytest.mark.regression
90
96
  @pytest.mark.parametrize(
91
97
  "toolkit,tool_name,prompt_template,expected_template,expected_content_template",
@@ -132,6 +138,8 @@ def test_assistant_with_create_file_tool(
132
138
  git_utils.delete_file(file_name, "main")
133
139
 
134
140
 
141
+ @pytest.mark.assistant
142
+ @pytest.mark.gitlab
135
143
  @pytest.mark.regression
136
144
  @pytest.mark.parametrize(
137
145
  "toolkit,tool_name,prompt_template,expected_template",
@@ -174,6 +182,8 @@ def test_assistant_with_create_merge_request_tool(
174
182
  git_utils.delete_branch(source_branch)
175
183
 
176
184
 
185
+ @pytest.mark.assistant
186
+ @pytest.mark.gitlab
177
187
  @pytest.mark.regression
178
188
  @pytest.mark.parametrize(
179
189
  "toolkit,tool_name,prompt_template,expected_template,file_content_template",
@@ -235,6 +245,8 @@ def test_assistant_with_delete_file_tool(
235
245
  git_utils.delete_file(file_name, "main")
236
246
 
237
247
 
248
+ @pytest.mark.assistant
249
+ @pytest.mark.gitlab
238
250
  @pytest.mark.regression
239
251
  @pytest.mark.parametrize(
240
252
  "toolkit,tool_name,create_mr_prompt_template,create_mr_expected_template,get_mr_changes_prompt_template,get_mr_changes_expected_template",
@@ -290,6 +302,8 @@ def test_assistant_with_get_merge_request_changes_tool(
290
302
  git_utils.delete_branch(source_branch)
291
303
 
292
304
 
305
+ @pytest.mark.assistant
306
+ @pytest.mark.gitlab
293
307
  @pytest.mark.regression
294
308
  @pytest.mark.parametrize(
295
309
  "toolkit,tool_name,create_prompt_template,create_expected_template,created_content_template,update_prompt_template,update_expected_template,updated_content_template",
@@ -15,16 +15,17 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
15
15
  )
16
16
 
17
17
 
18
+ @pytest.mark.assistant
19
+ @pytest.mark.mcp
20
+ @pytest.mark.regression
21
+ @pytest.mark.skipif(
22
+ os.getenv("ENV") == "local", reason="Skipping this test on local environment"
23
+ )
18
24
  @pytest.mark.parametrize(
19
25
  "command,expected_answer",
20
26
  cli_mcp_server_test_data,
21
27
  ids=[f"{row[0]}" for row in cli_mcp_server_test_data],
22
28
  )
23
- @pytest.mark.regression
24
- @pytest.mark.mcp
25
- @pytest.mark.skipif(
26
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
27
- )
28
29
  def test_cli_mcp_server(
29
30
  assistant_utils, assistant, similarity_check, command, expected_answer
30
31
  ):
@@ -37,8 +38,9 @@ def test_cli_mcp_server(
37
38
  similarity_check.check_similarity(response, expected_answer)
38
39
 
39
40
 
40
- @pytest.mark.regression
41
+ @pytest.mark.assistant
41
42
  @pytest.mark.mcp
43
+ @pytest.mark.regression
42
44
  @pytest.mark.skipif(
43
45
  os.getenv("ENV") == "local", reason="Skipping this test on local environment"
44
46
  )
@@ -12,6 +12,12 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
12
12
  )
13
13
 
14
14
 
15
+ @pytest.mark.assistant
16
+ @pytest.mark.mcp
17
+ @pytest.mark.regression
18
+ @pytest.mark.skipif(
19
+ os.getenv("ENV") == "local", reason="Skipping this test on local environment"
20
+ )
15
21
  @pytest.mark.parametrize(
16
22
  "mcp_server",
17
23
  time_mcp_server_test_data,
@@ -19,11 +25,6 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
19
25
  f"With config: {data.config is not None}" for data in time_mcp_server_test_data
20
26
  ],
21
27
  )
22
- @pytest.mark.regression
23
- @pytest.mark.mcp
24
- @pytest.mark.skipif(
25
- os.getenv("ENV") == "local", reason="Skipping this test on local environment"
26
- )
27
28
  def test_creation_mcp_server_with_form_configuration(
28
29
  assistant_utils, assistant, similarity_check, mcp_server
29
30
  ):
@@ -37,8 +38,9 @@ def test_creation_mcp_server_with_form_configuration(
37
38
  similarity_check.check_similarity(response, time_expected_response)
38
39
 
39
40
 
40
- @pytest.mark.regression
41
+ @pytest.mark.assistant
41
42
  @pytest.mark.mcp
43
+ @pytest.mark.regression
42
44
  @pytest.mark.skipif(
43
45
  os.getenv("ENV") == "local", reason="Skipping this test on local environment"
44
46
  )
@@ -16,6 +16,9 @@ from codemie_test_harness.tests.test_data.notification_tools_test_data import (
16
16
  )
17
17
 
18
18
 
19
+ @pytest.mark.assistant
20
+ @pytest.mark.notification
21
+ @pytest.mark.email
19
22
  @pytest.mark.regression
20
23
  @pytest.mark.skipif(
21
24
  os.getenv("ENV") in ["local", "gcp"],
@@ -59,6 +62,9 @@ def test_assistant_with_email_tool(
59
62
  assert_that(message_title, equal_to(EMAIL_SUBJECT))
60
63
 
61
64
 
65
+ @pytest.mark.assistant
66
+ @pytest.mark.notification
67
+ @pytest.mark.telegram
62
68
  @pytest.mark.regression
63
69
  def test_assistant_with_telegram_tool(
64
70
  assistant, assistant_utils, similarity_check, integration_utils
@@ -8,16 +8,18 @@ from codemie_test_harness.tests.test_data.open_api_tools_test_data import (
8
8
  )
9
9
 
10
10
 
11
+ @pytest.mark.assistant
12
+ @pytest.mark.openapi
11
13
  @pytest.mark.regression
14
+ @pytest.mark.testcase("EPMCDME-6127")
15
+ @pytest.mark.skipif(
16
+ os.getenv("ENV") == "azure", reason="Still have an issue with encoding long strings"
17
+ )
12
18
  @pytest.mark.parametrize(
13
19
  "tool_name,prompt,expected_response",
14
20
  open_api_tools_test_data,
15
21
  ids=[f"{row[0]}" for row in open_api_tools_test_data],
16
22
  )
17
- @pytest.mark.testcase("EPMCDME-6127")
18
- @pytest.mark.skipif(
19
- os.getenv("ENV") == "azure", reason="Still have an issue with encoding long strings"
20
- )
21
23
  def test_create_assistant_with_open_api_tool(
22
24
  assistant_utils,
23
25
  assistant,
@@ -11,8 +11,9 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
11
11
  from codemie_test_harness.tests.utils.constants import TESTS_PATH
12
12
 
13
13
 
14
- @pytest.mark.regression
14
+ @pytest.mark.assistant
15
15
  @pytest.mark.plugin
16
+ @pytest.mark.regression
16
17
  @pytest.mark.parametrize(
17
18
  "prompt,expected_response,tool_name",
18
19
  list_files_plugin_tools_test_data,
@@ -34,6 +35,8 @@ def test_assistant_with_list_files_plugin_tools(
34
35
  similarity_check.check_similarity(response, expected_response)
35
36
 
36
37
 
38
+ @pytest.mark.assistant
39
+ @pytest.mark.plugin
37
40
  @pytest.mark.regression
38
41
  @pytest.mark.plugin
39
42
  def test_assistant_with_modify_files_plugin_tools(
@@ -10,8 +10,10 @@ from codemie_test_harness.tests.test_data.mcp_server_test_data import (
10
10
  from codemie_test_harness.tests.utils.constants import TESTS_PATH
11
11
 
12
12
 
13
- @pytest.mark.regression
13
+ @pytest.mark.assistant
14
14
  @pytest.mark.plugin
15
+ @pytest.mark.mcp
16
+ @pytest.mark.regression
15
17
  @pytest.mark.parametrize(
16
18
  "prompt,expected_response",
17
19
  cli_mcp_server_with_plugin_test_data,
@@ -31,8 +33,10 @@ def test_assistant_with_plugin_and_cli_mcp_server(
31
33
  similarity_check.check_similarity(response, expected_response)
32
34
 
33
35
 
34
- @pytest.mark.regression
36
+ @pytest.mark.assistant
35
37
  @pytest.mark.plugin
38
+ @pytest.mark.mcp
39
+ @pytest.mark.regression
36
40
  @pytest.mark.parametrize(
37
41
  "prompt,expected_response,tool_name",
38
42
  filesystem_mcp_server_with_plugin_test_data,
@@ -20,6 +20,8 @@ from codemie_test_harness.tests.utils.constants import (
20
20
  )
21
21
 
22
22
 
23
+ @pytest.mark.assistant
24
+ @pytest.mark.project_management
23
25
  @pytest.mark.regression
24
26
  @pytest.mark.parametrize(
25
27
  "tool_name, integration_type, prompt, expected_response",
@@ -50,6 +52,9 @@ def test_assistant_with_project_management_tools(
50
52
  similarity_check.check_similarity(response, expected_response)
51
53
 
52
54
 
55
+ @pytest.mark.assistant
56
+ @pytest.mark.project_management
57
+ @pytest.mark.jira
53
58
  @pytest.mark.regression
54
59
  def test_assistant_with_jira_tool_and_integration_id_in_chat(
55
60
  assistant, assistant_utils, integration_utils, similarity_check, jira_integration
@@ -73,6 +78,9 @@ def test_assistant_with_jira_tool_and_integration_id_in_chat(
73
78
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
74
79
 
75
80
 
81
+ @pytest.mark.assistant
82
+ @pytest.mark.project_management
83
+ @pytest.mark.jira
76
84
  @pytest.mark.regression
77
85
  def test_assistant_with_jira_tool_and_credentials_in_chat(
78
86
  assistant, assistant_utils, integration_utils, similarity_check
@@ -93,6 +101,9 @@ def test_assistant_with_jira_tool_and_credentials_in_chat(
93
101
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
94
102
 
95
103
 
104
+ @pytest.mark.assistant
105
+ @pytest.mark.project_management
106
+ @pytest.mark.confluence
96
107
  @pytest.mark.regression
97
108
  def test_assistant_with_confluence_tool_and_integration_id_in_chat(
98
109
  assistant,
@@ -120,6 +131,9 @@ def test_assistant_with_confluence_tool_and_integration_id_in_chat(
120
131
  similarity_check.check_similarity(response, RESPONSE_FOR_CONFLUENCE_TOOL)
121
132
 
122
133
 
134
+ @pytest.mark.assistant
135
+ @pytest.mark.project_management
136
+ @pytest.mark.confluence
123
137
  @pytest.mark.regression
124
138
  def test_assistant_with_confluence_tool_and_credentials_in_chat(
125
139
  assistant, assistant_utils, integration_utils, similarity_check
@@ -137,6 +151,9 @@ def test_assistant_with_confluence_tool_and_credentials_in_chat(
137
151
  similarity_check.check_similarity(response, RESPONSE_FOR_CONFLUENCE_TOOL)
138
152
 
139
153
 
154
+ @pytest.mark.assistant
155
+ @pytest.mark.project_management
156
+ @pytest.mark.confluence
140
157
  @pytest.mark.regression
141
158
  @pytest.mark.skip(reason="Test should be fixed")
142
159
  def test_assistant_with_confluence_tool_and_without_credentials(
@@ -9,6 +9,8 @@ from codemie_test_harness.tests.test_data.research_tools_test_data import (
9
9
  from codemie_test_harness.tests.utils.base_utils import percent_of_relevant_titles
10
10
 
11
11
 
12
+ @pytest.mark.assistant
13
+ @pytest.mark.research
12
14
  @pytest.mark.regression
13
15
  @pytest.mark.parametrize(
14
16
  "tool_name,prompt,expected_percentage",
@@ -35,6 +37,8 @@ def test_assistant_with_search_tools(
35
37
  )
36
38
 
37
39
 
40
+ @pytest.mark.assistant
41
+ @pytest.mark.research
38
42
  @pytest.mark.regression
39
43
  @pytest.mark.parametrize(
40
44
  "tool_name,prompt,expected_response",
@@ -7,6 +7,8 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
7
7
  )
8
8
 
9
9
 
10
+ @pytest.mark.assistant
11
+ @pytest.mark.servicenow
10
12
  @pytest.mark.regression
11
13
  def test_assistant_with_servicenow_tools(
12
14
  assistant_utils,
@@ -9,13 +9,15 @@ from codemie_test_harness.tests.test_data.vcs_tools_test_data import (
9
9
  from codemie_test_harness.tests.utils.constants import vcs_integrations
10
10
 
11
11
 
12
+ @pytest.mark.assistant
13
+ @pytest.mark.vcs
12
14
  @pytest.mark.regression
15
+ @pytest.mark.testcase("EPMCDME-5803, EPMCDME-5804")
13
16
  @pytest.mark.parametrize(
14
17
  "tool_name,prompt,expected_response",
15
18
  vcs_tools_test_data,
16
19
  ids=[f"{row[0]}" for row in vcs_tools_test_data],
17
20
  )
18
- @pytest.mark.testcase("EPMCDME-5803, EPMCDME-5804")
19
21
  def test_create_assistant_with_vcs_tool(
20
22
  request,
21
23
  assistant_utils,
@@ -15,6 +15,8 @@ from hamcrest import (
15
15
  from codemie_test_harness.tests.utils.base_utils import get_random_name
16
16
 
17
17
 
18
+ @pytest.mark.assistant
19
+ @pytest.mark.conversations
18
20
  @pytest.mark.regression
19
21
  def test_list_conversations(assistant, assistant_utils, conversation_utils):
20
22
  assistant = assistant()
@@ -41,6 +43,8 @@ def test_list_conversations(assistant, assistant_utils, conversation_utils):
41
43
  )
42
44
 
43
45
 
46
+ @pytest.mark.assistant
47
+ @pytest.mark.conversations
44
48
  @pytest.mark.regression
45
49
  def test_get_specific_conversation(assistant, assistant_utils, conversation_utils):
46
50
  prompt = f"prompt {get_random_name()}"
@@ -63,6 +67,8 @@ def test_get_specific_conversation(assistant, assistant_utils, conversation_util
63
67
  )
64
68
 
65
69
 
70
+ @pytest.mark.assistant
71
+ @pytest.mark.conversations
66
72
  @pytest.mark.regression
67
73
  def test_get_conversation_by_assistant_id(
68
74
  assistant, assistant_utils, conversation_utils
@@ -84,6 +90,8 @@ def test_get_conversation_by_assistant_id(
84
90
  )
85
91
 
86
92
 
93
+ @pytest.mark.assistant
94
+ @pytest.mark.conversations
87
95
  @pytest.mark.regression
88
96
  def test_delete_conversation(assistant, assistant_utils, conversation_utils):
89
97
  prompt = f"prompt {get_random_name()}"
@@ -100,6 +108,8 @@ def test_delete_conversation(assistant, assistant_utils, conversation_utils):
100
108
  )
101
109
 
102
110
 
111
+ @pytest.mark.assistant
112
+ @pytest.mark.conversations
103
113
  @pytest.mark.regression
104
114
  def test_get_non_existent_conversation(assistant, assistant_utils, conversation_utils):
105
115
  invalid_id = get_random_name()
@@ -32,8 +32,10 @@ def git_integration(integration_utils):
32
32
  integration_utils.delete_integration(integration)
33
33
 
34
34
 
35
- @pytest.mark.regression
35
+ @pytest.mark.assistant
36
36
  @pytest.mark.code_kb
37
+ @pytest.mark.e2e
38
+ @pytest.mark.regression
37
39
  def test_assistant_with_code_kb(
38
40
  assistant_utils, code_datasource, default_llm, similarity_check
39
41
  ):
@@ -53,12 +55,15 @@ def test_assistant_with_code_kb(
53
55
  )
54
56
 
55
57
 
58
+ @pytest.mark.assistant
59
+ @pytest.mark.vcs
60
+ @pytest.mark.gitlab
61
+ @pytest.mark.e2e
62
+ @pytest.mark.regression
56
63
  @pytest.mark.skipif(
57
64
  os.getenv("GIT_ENV") == "github",
58
65
  reason="Test is skipped when GIT_ENV is set to github",
59
66
  )
60
- @pytest.mark.regression
61
- @pytest.mark.gitlab
62
67
  def test_assistant_with_vcs_gitlab_tool(
63
68
  assistant_utils, gitlab_integration, default_llm, similarity_check
64
69
  ):
@@ -84,12 +89,15 @@ def test_assistant_with_vcs_gitlab_tool(
84
89
  )
85
90
 
86
91
 
92
+ @pytest.mark.assistant
93
+ @pytest.mark.vcs
94
+ @pytest.mark.github
95
+ @pytest.mark.e2e
96
+ @pytest.mark.regression
87
97
  @pytest.mark.skipif(
88
98
  os.getenv("GIT_ENV") == "gitlab",
89
99
  reason="Test is skipped when GIT_ENV is set to gitlab",
90
100
  )
91
- @pytest.mark.regression
92
- @pytest.mark.github
93
101
  def test_assistant_with_vcs_github_tool(
94
102
  assistant_utils, github_integration, default_llm, similarity_check
95
103
  ):
@@ -115,8 +123,10 @@ def test_assistant_with_vcs_github_tool(
115
123
  )
116
124
 
117
125
 
126
+ @pytest.mark.assistant
127
+ @pytest.mark.gitlab
128
+ @pytest.mark.e2e
118
129
  @pytest.mark.regression
119
- @pytest.mark.git
120
130
  def test_assistant_with_list_branches_tool(
121
131
  assistant_utils, code_datasource, default_llm, similarity_check
122
132
  ):
@@ -144,8 +154,11 @@ def test_assistant_with_list_branches_tool(
144
154
  )
145
155
 
146
156
 
157
+ @pytest.mark.assistant
158
+ @pytest.mark.jira
159
+ @pytest.mark.project_management
160
+ @pytest.mark.e2e
147
161
  @pytest.mark.regression
148
- @pytest.mark.jira_kb
149
162
  def test_assistant_with_jira_kb(
150
163
  assistant_utils, jira_datasource, default_llm, similarity_check
151
164
  ):
@@ -167,8 +180,11 @@ def test_assistant_with_jira_kb(
167
180
  )
168
181
 
169
182
 
183
+ @pytest.mark.assistant
184
+ @pytest.mark.confluence
185
+ @pytest.mark.project_management
186
+ @pytest.mark.e2e
170
187
  @pytest.mark.regression
171
- @pytest.mark.confluence_kb
172
188
  def test_assistant_with_confluence_kb(
173
189
  assistant_utils, confluence_datasource, default_llm, similarity_check
174
190
  ):
@@ -45,6 +45,13 @@ pytestmark = pytest.mark.skipif(
45
45
  )
46
46
 
47
47
 
48
+ @pytest.mark.assistant
49
+ @pytest.mark.cloud
50
+ @pytest.mark.integration
51
+ @pytest.mark.project_integration
52
+ @pytest.mark.default_integration
53
+ @pytest.mark.regression
54
+ @pytest.mark.tescase("EPMCDME-2377")
48
55
  @pytest.mark.parametrize(
49
56
  "toolkit, tool_name, credential_type, credentials, prompt, expected_response",
50
57
  cloud_test_data,
@@ -55,8 +62,6 @@ pytestmark = pytest.mark.skipif(
55
62
  f"{Toolkit.CLOUD}_{CloudTool.KUBERNETES}",
56
63
  ],
57
64
  )
58
- @pytest.mark.regression
59
- @pytest.mark.tescase("EPMCDME-2377")
60
65
  def test_assistant_with_default_integration_cloud(
61
66
  general_integration,
62
67
  integration_utils,
@@ -90,6 +95,11 @@ def test_assistant_with_default_integration_cloud(
90
95
  similarity_check.check_similarity(response, expected_response)
91
96
 
92
97
 
98
+ @pytest.mark.assistant
99
+ @pytest.mark.ado
100
+ @pytest.mark.integration
101
+ @pytest.mark.project_integration
102
+ @pytest.mark.default_integration
93
103
  @pytest.mark.regression
94
104
  @pytest.mark.tescase("EPMCDME-2377")
95
105
  def test_assistant_with_default_integration_ado(
@@ -124,13 +134,17 @@ def test_assistant_with_default_integration_ado(
124
134
  similarity_check.check_similarity(response, expected_response)
125
135
 
126
136
 
137
+ @pytest.mark.assistant
138
+ @pytest.mark.integration
139
+ @pytest.mark.project_integration
140
+ @pytest.mark.default_integration
141
+ @pytest.mark.regression
142
+ @pytest.mark.tescase("EPMCDME-2377")
127
143
  @pytest.mark.parametrize(
128
144
  "toolkit, tool_name, credentials, prompt, expected_response",
129
145
  sonar_tools_test_data,
130
146
  ids=[f"{row[0]}_{row[1]}" for row in sonar_tools_test_data],
131
147
  )
132
- @pytest.mark.regression
133
- @pytest.mark.tescase("EPMCDME-2377")
134
148
  def test_assistant_with_default_integration_codebase(
135
149
  general_integration,
136
150
  integration_utils,
@@ -165,6 +179,11 @@ def test_assistant_with_default_integration_codebase(
165
179
  similarity_check.check_similarity(response, expected_response)
166
180
 
167
181
 
182
+ @pytest.mark.assistant
183
+ @pytest.mark.gitlab
184
+ @pytest.mark.integration
185
+ @pytest.mark.project_integration
186
+ @pytest.mark.default_integration
168
187
  @pytest.mark.regression
169
188
  @pytest.mark.tescase("EPMCDME-2377")
170
189
  def test_assistant_with_default_integration_git(
@@ -213,6 +232,12 @@ def test_assistant_with_default_integration_git(
213
232
  similarity_check.check_similarity(response, expected_response)
214
233
 
215
234
 
235
+ @pytest.mark.assistant
236
+ @pytest.mark.jira
237
+ @pytest.mark.project_management
238
+ @pytest.mark.integration
239
+ @pytest.mark.project_integration
240
+ @pytest.mark.default_integration
216
241
  @pytest.mark.regression
217
242
  @pytest.mark.tescase("EPMCDME-2377")
218
243
  def test_assistant_with_default_integration_jira(
@@ -246,6 +271,12 @@ def test_assistant_with_default_integration_jira(
246
271
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
247
272
 
248
273
 
274
+ @pytest.mark.assistant
275
+ @pytest.mark.email
276
+ @pytest.mark.notification
277
+ @pytest.mark.integration
278
+ @pytest.mark.project_integration
279
+ @pytest.mark.default_integration
249
280
  @pytest.mark.regression
250
281
  @pytest.mark.tescase("EPMCDME-2377")
251
282
  def test_assistant_with_default_integration_email(
@@ -277,6 +308,12 @@ def test_assistant_with_default_integration_email(
277
308
  similarity_check.check_similarity(response, EMAIL_RESPONSE)
278
309
 
279
310
 
311
+ @pytest.mark.assistant
312
+ @pytest.mark.keycloak
313
+ @pytest.mark.integration
314
+ @pytest.mark.project_integration
315
+ @pytest.mark.default_integration
316
+ @pytest.mark.regression
280
317
  @pytest.mark.regression
281
318
  def test_assistant_with_default_integration_keycloak(
282
319
  general_integration,
@@ -21,6 +21,9 @@ from codemie_test_harness.tests.utils.base_utils import (
21
21
  )
22
22
 
23
23
 
24
+ @pytest.mark.integration
25
+ @pytest.mark.project_integration
26
+ @pytest.mark.regression
24
27
  @pytest.mark.parametrize(
25
28
  "credential_type, credentials",
26
29
  valid_integrations,
@@ -49,7 +52,6 @@ from codemie_test_harness.tests.utils.base_utils import (
49
52
  f"Project integration: {CredentialTypes.KEYCLOAK}",
50
53
  ],
51
54
  )
52
- @pytest.mark.regression
53
55
  def test_integration_created(
54
56
  credential_type, credentials, general_integration, integration_utils
55
57
  ):
@@ -75,6 +77,9 @@ def test_integration_created(
75
77
  )
76
78
 
77
79
 
80
+ @pytest.mark.integration
81
+ @pytest.mark.project_integration
82
+ @pytest.mark.regression
78
83
  @pytest.mark.parametrize(
79
84
  "credential_type, credentials",
80
85
  testable_integrations,
@@ -93,7 +98,6 @@ def test_integration_created(
93
98
  f"Project integration: {CredentialTypes.KUBERNETES}",
94
99
  ],
95
100
  )
96
- @pytest.mark.regression
97
101
  def test_integration_after_creation(
98
102
  credential_type, credentials, general_integration, integration_utils
99
103
  ):
@@ -124,6 +128,10 @@ def test_integration_after_creation(
124
128
  )
125
129
 
126
130
 
131
+ @pytest.mark.integration
132
+ @pytest.mark.project_integration
133
+ @pytest.mark.regression
134
+ @pytest.mark.tescase("EPMCDME-2376")
127
135
  @pytest.mark.parametrize(
128
136
  "credential_type, credentials",
129
137
  valid_integrations,
@@ -152,8 +160,6 @@ def test_integration_after_creation(
152
160
  f"Project integration: {CredentialTypes.KEYCLOAK}",
153
161
  ],
154
162
  )
155
- @pytest.mark.tescase("EPMCDME-2376")
156
- @pytest.mark.regression
157
163
  def test_update_integration(
158
164
  credential_type, credentials, general_integration, integration_utils
159
165
  ):
@@ -191,6 +197,9 @@ def test_update_integration(
191
197
  assert_that(updated_integration.alias, equal_to(updated_alias))
192
198
 
193
199
 
200
+ @pytest.mark.integration
201
+ @pytest.mark.project_integration
202
+ @pytest.mark.regression
194
203
  @pytest.mark.parametrize(
195
204
  "credential_type, credentials, error_message",
196
205
  invalid_integrations,
@@ -207,7 +216,6 @@ def test_update_integration(
207
216
  f"Project integration: {CredentialTypes.KUBERNETES}",
208
217
  ],
209
218
  )
210
- @pytest.mark.regression
211
219
  def test_integration_with_invalid_credentials(
212
220
  credential_type, credentials, error_message, integration_utils
213
221
  ):
@@ -242,6 +250,9 @@ def test_integration_with_invalid_credentials(
242
250
  assert_error_details(e.response, 400, error_message)
243
251
 
244
252
 
253
+ @pytest.mark.integration
254
+ @pytest.mark.project_integration
255
+ @pytest.mark.regression
245
256
  @pytest.mark.parametrize(
246
257
  "credential_type, credentials",
247
258
  testable_integrations,
@@ -260,7 +271,6 @@ def test_integration_with_invalid_credentials(
260
271
  f"Project integration: {CredentialTypes.KUBERNETES}",
261
272
  ],
262
273
  )
263
- @pytest.mark.regression
264
274
  def test_integration_during_creation(credential_type, credentials, integration_utils):
265
275
  integration_test_model = IntegrationTestRequest(
266
276
  credential_type=credential_type,
@@ -278,6 +288,9 @@ def test_integration_during_creation(credential_type, credentials, integration_u
278
288
  )
279
289
 
280
290
 
291
+ @pytest.mark.integration
292
+ @pytest.mark.project_integration
293
+ @pytest.mark.regression
281
294
  @pytest.mark.parametrize(
282
295
  "credential_type, credentials, error_message",
283
296
  invalid_integrations,
@@ -294,7 +307,6 @@ def test_integration_during_creation(credential_type, credentials, integration_u
294
307
  f"Project integration: {CredentialTypes.KUBERNETES}",
295
308
  ],
296
309
  )
297
- @pytest.mark.regression
298
310
  def test_integration_during_creation_with_invalid_credentials(
299
311
  credential_type, credentials, error_message, integration_utils
300
312
  ):
@@ -310,6 +322,10 @@ def test_integration_during_creation_with_invalid_credentials(
310
322
  assert_error_details(e.response, 400, error_message)
311
323
 
312
324
 
325
+ @pytest.mark.integration
326
+ @pytest.mark.project_integration
327
+ @pytest.mark.regression
328
+ @pytest.mark.tescase("EPMCDME-2374")
313
329
  @pytest.mark.parametrize(
314
330
  "credential_type, credentials",
315
331
  valid_integrations,
@@ -338,8 +354,6 @@ def test_integration_during_creation_with_invalid_credentials(
338
354
  f"Project integration: {CredentialTypes.KEYCLOAK}",
339
355
  ],
340
356
  )
341
- @pytest.mark.tescase("EPMCDME-2374")
342
- @pytest.mark.regression
343
357
  def test_delete_integration(
344
358
  credential_type, credentials, general_integration, integration_utils
345
359
  ):