codemie-test-harness 0.1.156__py3-none-any.whl → 0.1.158__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (123) hide show
  1. codemie_test_harness/tests/assistant/datasource/test_code_datasource.py +6 -0
  2. codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py +14 -2
  3. codemie_test_harness/tests/assistant/datasource/test_file_indexing.py +10 -0
  4. codemie_test_harness/tests/assistant/datasource/test_google_datasource.py +6 -0
  5. codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py +18 -2
  6. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool.py +10 -0
  7. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py +10 -0
  8. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py +10 -0
  9. codemie_test_harness/tests/assistant/test_assistants.py +24 -2
  10. codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py +2 -0
  11. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_test_plan_tools.py +4 -0
  12. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_wiki_tools.py +4 -0
  13. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_work_item_tools.py +4 -0
  14. codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py +2 -0
  15. codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py +3 -0
  16. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +9 -1
  17. codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py +9 -1
  18. codemie_test_harness/tests/assistant/tools/git/test_assistant_with_git_tools.py +14 -0
  19. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +8 -6
  20. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +8 -6
  21. codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py +6 -0
  22. codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py +6 -4
  23. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_development_plugin.py +4 -1
  24. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_plugin_and_mcp_servers.py +6 -2
  25. codemie_test_harness/tests/assistant/tools/project_management/test_assistant_pm_tools.py +17 -0
  26. codemie_test_harness/tests/assistant/tools/research/test_assistant_research_tools.py +4 -0
  27. codemie_test_harness/tests/assistant/tools/servicenow/test_servicenow_tools.py +2 -0
  28. codemie_test_harness/tests/assistant/tools/vcs/test_assistant_with_vcs_tools.py +3 -1
  29. codemie_test_harness/tests/conversations/test_conversations_endpoints.py +10 -0
  30. codemie_test_harness/tests/e2e/test_e2e.py +24 -8
  31. codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -4
  32. codemie_test_harness/tests/integrations/project/test_project_integrations.py +23 -9
  33. codemie_test_harness/tests/integrations/user/test_default_integrations.py +40 -4
  34. codemie_test_harness/tests/integrations/user/test_user_integrations.py +26 -12
  35. codemie_test_harness/tests/llm/assistants/test_llm.py +8 -0
  36. codemie_test_harness/tests/providers/test_providers_endpoints.py +11 -0
  37. codemie_test_harness/tests/search/test_search_assistant.py +2 -0
  38. codemie_test_harness/tests/search/test_search_datasource.py +10 -0
  39. codemie_test_harness/tests/search/test_search_integration.py +6 -0
  40. codemie_test_harness/tests/search/test_search_workflow.py +2 -0
  41. codemie_test_harness/tests/test_data/cloud_tools_test_data.py +18 -10
  42. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +6 -2
  43. codemie_test_harness/tests/test_data/data_management_tools_test_data.py +4 -37
  44. codemie_test_harness/tests/test_data/direct_tools/cloud_tools_test_data.py +18 -10
  45. codemie_test_harness/tests/test_data/direct_tools/codebase_tools_test_data.py +6 -2
  46. codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +1 -1
  47. codemie_test_harness/tests/test_data/direct_tools/notification_tools_test_data.py +9 -5
  48. codemie_test_harness/tests/test_data/direct_tools/project_management_tools_test_data.py +6 -2
  49. codemie_test_harness/tests/test_data/direct_tools/vcs_tools_test_data.py +6 -2
  50. codemie_test_harness/tests/test_data/integrations_test_data.py +185 -65
  51. codemie_test_harness/tests/test_data/project_management_test_data.py +10 -4
  52. codemie_test_harness/tests/test_data/vcs_tools_test_data.py +8 -2
  53. codemie_test_harness/tests/ui/test_create_workflow.py +13 -0
  54. codemie_test_harness/tests/ui/test_edit_workflow.py +16 -0
  55. codemie_test_harness/tests/ui/test_workflow_details.py +17 -0
  56. codemie_test_harness/tests/ui/test_workflow_executions_page.py +36 -0
  57. codemie_test_harness/tests/ui/test_workflow_templates.py +8 -0
  58. codemie_test_harness/tests/ui/test_workflows.py +8 -0
  59. codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +3 -0
  60. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +6 -0
  61. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +6 -0
  62. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +6 -0
  63. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +3 -0
  64. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +5 -0
  65. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +10 -5
  66. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +20 -0
  67. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +20 -0
  68. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +25 -0
  69. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +15 -0
  70. codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +21 -0
  71. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +9 -3
  72. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +8 -0
  73. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +3 -0
  74. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +6 -2
  75. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +8 -2
  76. codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +3 -0
  77. codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +6 -0
  78. codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +3 -0
  79. codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +3 -0
  80. codemie_test_harness/tests/workflow/config_validation/test_config_validation.py +4 -3
  81. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool.py +17 -2
  82. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool_kit.py +15 -0
  83. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_access_management_tool.py +9 -0
  84. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_test_plan_tools.py +9 -0
  85. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_wiki_tools.py +9 -0
  86. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_work_item_tools.py +9 -0
  87. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +9 -0
  88. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_codebase_tools.py +6 -0
  89. codemie_test_harness/tests/workflow/direct_tools_calling/{test_workflow_with_data_management_tools.py → test_workflow_with_data_management_tools_sql.py} +10 -1
  90. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_file_management_tools.py +9 -0
  91. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py +3 -0
  92. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_open_api_tools.py +9 -0
  93. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_plugin_tools.py +12 -4
  94. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_project_management_tools.py +9 -0
  95. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_research_tools.py +9 -0
  96. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_servicenow_tools.py +9 -0
  97. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_vcs_tools.py +3 -0
  98. codemie_test_harness/tests/workflow/test_workflows.py +1 -0
  99. codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +3 -0
  100. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +6 -0
  101. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +6 -0
  102. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +6 -0
  103. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +3 -0
  104. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +5 -0
  105. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +10 -5
  106. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +20 -0
  107. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +20 -0
  108. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +25 -0
  109. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +15 -0
  110. codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +21 -0
  111. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +12 -6
  112. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +8 -0
  113. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +6 -3
  114. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +6 -2
  115. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +8 -2
  116. codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +3 -0
  117. codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +6 -0
  118. codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +3 -0
  119. codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +3 -0
  120. {codemie_test_harness-0.1.156.dist-info → codemie_test_harness-0.1.158.dist-info}/METADATA +2 -2
  121. {codemie_test_harness-0.1.156.dist-info → codemie_test_harness-0.1.158.dist-info}/RECORD +123 -123
  122. {codemie_test_harness-0.1.156.dist-info → codemie_test_harness-0.1.158.dist-info}/WHEEL +0 -0
  123. {codemie_test_harness-0.1.156.dist-info → codemie_test_harness-0.1.158.dist-info}/entry_points.txt +0 -0
@@ -8,6 +8,8 @@ from codemie_test_harness.tests.test_data.index_test_data import index_test_data
8
8
  from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
9
9
 
10
10
 
11
+ @pytest.mark.datasource
12
+ @pytest.mark.gitlab
11
13
  @pytest.mark.regression
12
14
  @pytest.mark.parametrize(
13
15
  "embedding_model",
@@ -46,6 +48,8 @@ def test_create_index_application_with_embedding_model(
46
48
  similarity_check.check_similarity(answer, expected_answer)
47
49
 
48
50
 
51
+ @pytest.mark.datasource
52
+ @pytest.mark.gitlab
49
53
  @pytest.mark.regression
50
54
  def test_edit_description_for_code_data_source(
51
55
  client,
@@ -62,6 +66,8 @@ def test_edit_description_for_code_data_source(
62
66
  assert_that(updated_datasource.description, equal_to(updated_description))
63
67
 
64
68
 
69
+ @pytest.mark.datasource
70
+ @pytest.mark.gitlab
65
71
  @pytest.mark.regression
66
72
  def test_create_code_datasource_with_existing_name(gitlab_datasource, datasource_utils):
67
73
  datasource = datasource_utils.get_datasource(gitlab_datasource.id)
@@ -27,15 +27,23 @@ def confluence_cloud_datasource(datasource_utils, confluence_cloud_integration):
27
27
  datasource_utils.delete_datasource(datasource)
28
28
 
29
29
 
30
+ @pytest.mark.datasource
31
+ @pytest.mark.project_management
30
32
  @pytest.mark.regression
31
33
  @pytest.mark.parametrize(
32
34
  "datasource_fixture, prompt, expected_response",
33
35
  [
34
- ("confluence_datasource", CONFLUENCE_TOOL_PROMPT, RESPONSE_FOR_CONFLUENCE_TOOL),
35
- (
36
+ pytest.param(
37
+ "confluence_datasource",
38
+ CONFLUENCE_TOOL_PROMPT,
39
+ RESPONSE_FOR_CONFLUENCE_TOOL,
40
+ marks=pytest.mark.confluence,
41
+ ),
42
+ pytest.param(
36
43
  "confluence_cloud_datasource",
37
44
  CONFLUENCE_CLOUD_TOOL_PROMPT,
38
45
  RESPONSE_FOR_CONFLUENCE_CLOUD_TOOL,
46
+ marks=[pytest.mark.confluence, pytest.mark.confluence_cloud],
39
47
  ),
40
48
  ],
41
49
  ids=["Confluence", "Confluence Cloud"],
@@ -65,6 +73,8 @@ def test_create_datasource_with_confluence_and_confluence_cloud_integration(
65
73
  similarity_check.check_similarity(response, expected_response)
66
74
 
67
75
 
76
+ @pytest.mark.datasource
77
+ @pytest.mark.confluence
68
78
  @pytest.mark.regression
69
79
  def test_edit_description_for_confluence_data_source(
70
80
  client, confluence_datasource, datasource_utils
@@ -79,6 +89,8 @@ def test_edit_description_for_confluence_data_source(
79
89
  assert_that(updated_datasource.description, equal_to(updated_description))
80
90
 
81
91
 
92
+ @pytest.mark.datasource
93
+ @pytest.mark.confluence
82
94
  @pytest.mark.regression
83
95
  def test_create_confluence_datasource_with_existing_name(
84
96
  datasource_utils, confluence_datasource
@@ -58,6 +58,8 @@ def pytest_generate_tests(metafunc):
58
58
  metafunc.parametrize("file_name,expected_response", test_data)
59
59
 
60
60
 
61
+ @pytest.mark.datasource
62
+ @pytest.mark.file
61
63
  @pytest.mark.regression
62
64
  @pytest.mark.smoke
63
65
  def test_create_assistant_with_file_datasource(
@@ -87,6 +89,8 @@ def test_create_assistant_with_file_datasource(
87
89
  )
88
90
 
89
91
 
92
+ @pytest.mark.datasource
93
+ @pytest.mark.file
90
94
  @pytest.mark.regression
91
95
  @pytest.mark.smoke
92
96
  def test_edit_description_for_file_datasource(datasource_utils):
@@ -109,6 +113,8 @@ def test_edit_description_for_file_datasource(datasource_utils):
109
113
  assert_that(updated_datasource.description, equal_to(updated_description))
110
114
 
111
115
 
116
+ @pytest.mark.datasource
117
+ @pytest.mark.file
112
118
  @pytest.mark.regression
113
119
  @pytest.mark.smoke
114
120
  @pytest.mark.parametrize("file_name", large_files_test_data)
@@ -126,6 +132,8 @@ def test_create_file_datasource_with_large_files(datasource_utils, file_name):
126
132
  )
127
133
 
128
134
 
135
+ @pytest.mark.datasource
136
+ @pytest.mark.file
129
137
  @pytest.mark.regression
130
138
  @pytest.mark.smoke
131
139
  def test_create_file_datasource_with_big_number_of_files(datasource_utils):
@@ -142,6 +150,8 @@ def test_create_file_datasource_with_big_number_of_files(datasource_utils):
142
150
  assert_response(e.response, 422, "Too many files. Maximum count is 10")
143
151
 
144
152
 
153
+ @pytest.mark.datasource
154
+ @pytest.mark.file
145
155
  @pytest.mark.regression
146
156
  @pytest.mark.smoke
147
157
  def test_create_file_datasource_with_two_files(
@@ -12,6 +12,8 @@ from codemie_test_harness.tests.test_data.google_datasource_test_data import (
12
12
  from codemie_test_harness.tests.utils.base_utils import get_random_name, assert_response
13
13
 
14
14
 
15
+ @pytest.mark.datasource
16
+ @pytest.mark.google
15
17
  @pytest.mark.smoke
16
18
  @pytest.mark.regression
17
19
  def test_create_datasource_with_google_integration(
@@ -35,6 +37,8 @@ def test_create_datasource_with_google_integration(
35
37
  similarity_check.check_similarity(response_from_llm, RESPONSE_FOR_GOOGLE_DOC)
36
38
 
37
39
 
40
+ @pytest.mark.datasource
41
+ @pytest.mark.google
38
42
  @pytest.mark.regression
39
43
  @pytest.mark.smoke
40
44
  def test_edit_description_for_google_data_source(
@@ -50,6 +54,8 @@ def test_edit_description_for_google_data_source(
50
54
  assert_that(updated_datasource.description, equal_to(updated_description))
51
55
 
52
56
 
57
+ @pytest.mark.datasource
58
+ @pytest.mark.google
53
59
  @pytest.mark.regression
54
60
  @pytest.mark.smoke
55
61
  def test_create_google_doc_datasource_with_existing_name(
@@ -25,12 +25,24 @@ def jira_cloud_datasource(datasource_utils, jira_cloud_integration):
25
25
  datasource_utils.delete_datasource(datasource)
26
26
 
27
27
 
28
+ @pytest.mark.datasource
29
+ @pytest.mark.project_management
28
30
  @pytest.mark.regression
29
31
  @pytest.mark.parametrize(
30
32
  "datasource_fixture, prompt, expected_response",
31
33
  [
32
- ("jira_datasource", JIRA_TOOL_PROMPT, RESPONSE_FOR_JIRA_TOOL),
33
- ("jira_cloud_datasource", JIRA_CLOUD_TOOL_PROMPT, RESPONSE_FOR_JIRA_CLOUD_TOOL),
34
+ pytest.param(
35
+ "jira_datasource",
36
+ JIRA_TOOL_PROMPT,
37
+ RESPONSE_FOR_JIRA_TOOL,
38
+ marks=pytest.mark.jira,
39
+ ),
40
+ pytest.param(
41
+ "jira_cloud_datasource",
42
+ JIRA_CLOUD_TOOL_PROMPT,
43
+ RESPONSE_FOR_JIRA_CLOUD_TOOL,
44
+ marks=[pytest.mark.jira, pytest.mark.jira_cloud],
45
+ ),
34
46
  ],
35
47
  ids=["Jira", "Jira Cloud"],
36
48
  )
@@ -59,6 +71,8 @@ def test_create_datasource_with_jira_and_jira_cloud_integration(
59
71
  similarity_check.check_similarity(response, response)
60
72
 
61
73
 
74
+ @pytest.mark.datasource
75
+ @pytest.mark.jira
62
76
  @pytest.mark.regression
63
77
  def test_edit_description_for_jira_data_source(
64
78
  client, datasource_utils, jira_datasource
@@ -73,6 +87,8 @@ def test_edit_description_for_jira_data_source(
73
87
  assert_that(updated_datasource.description, equal_to(updated_description))
74
88
 
75
89
 
90
+ @pytest.mark.datasource
91
+ @pytest.mark.jira
76
92
  @pytest.mark.regression
77
93
  def test_create_jira_datasource_with_existing_name(datasource_utils, jira_datasource):
78
94
  datasource = datasource_utils.get_datasource(jira_datasource.id)
@@ -15,6 +15,8 @@ from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUti
15
15
  from codemie_test_harness.tests.utils.constants import test_project_name
16
16
 
17
17
 
18
+ @pytest.mark.integration
19
+ @pytest.mark.default_integration
18
20
  @pytest.mark.not_for_parallel_run
19
21
  @pytest.mark.regression
20
22
  @pytest.mark.tescase("EPMCDME-6708")
@@ -74,6 +76,8 @@ def test_assistant_should_use_user_integration_by_default(
74
76
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
75
77
 
76
78
 
79
+ @pytest.mark.integration
80
+ @pytest.mark.default_integration
77
81
  @pytest.mark.not_for_parallel_run
78
82
  @pytest.mark.regression
79
83
  @pytest.mark.tescase("EPMCDME-6708")
@@ -116,6 +120,8 @@ def test_assistant_with_global_and_project_integration(
116
120
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
117
121
 
118
122
 
123
+ @pytest.mark.integration
124
+ @pytest.mark.default_integration
119
125
  @pytest.mark.not_for_parallel_run
120
126
  @pytest.mark.regression
121
127
  @pytest.mark.tescase("EPMCDME-6708")
@@ -149,6 +155,8 @@ def test_assistant_with_project_integration_only(
149
155
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
150
156
 
151
157
 
158
+ @pytest.mark.integration
159
+ @pytest.mark.default_integration
152
160
  @pytest.mark.not_for_parallel_run
153
161
  @pytest.mark.regression
154
162
  def test_assistant_with_global_valid_and_user_invalid_integration(
@@ -187,6 +195,8 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
187
195
  similarity_check.check_similarity(response, RESPONSE_FOR_JIRA_TOOL)
188
196
 
189
197
 
198
+ @pytest.mark.integration
199
+ @pytest.mark.default_integration
190
200
  @pytest.mark.not_for_parallel_run
191
201
  @pytest.mark.regression
192
202
  def test_assistant_with_project_valid_and_user_invalid_integration(
@@ -17,6 +17,8 @@ ado_wiki_prompt = ado_wiki_get_test_data[0][2]
17
17
  ado_wiki_answer = ado_wiki_get_test_data[0][3]
18
18
 
19
19
 
20
+ @pytest.mark.integration
21
+ @pytest.mark.default_integration
20
22
  @pytest.mark.not_for_parallel_run
21
23
  @pytest.mark.regression
22
24
  @pytest.mark.tescase("EPMCDME-6708")
@@ -78,6 +80,8 @@ def test_assistant_should_use_user_integration_by_default(
78
80
  similarity_check.check_similarity(response, ado_wiki_answer)
79
81
 
80
82
 
83
+ @pytest.mark.integration
84
+ @pytest.mark.default_integration
81
85
  @pytest.mark.regression
82
86
  @pytest.mark.not_for_parallel_run
83
87
  @pytest.mark.tescase("EPMCDME-6708")
@@ -121,6 +125,8 @@ def test_assistant_with_global_and_project_integration(
121
125
  similarity_check.check_similarity(response, ado_wiki_answer)
122
126
 
123
127
 
128
+ @pytest.mark.integration
129
+ @pytest.mark.default_integration
124
130
  @pytest.mark.not_for_parallel_run
125
131
  @pytest.mark.regression
126
132
  @pytest.mark.tescase("EPMCDME-6708")
@@ -156,6 +162,8 @@ def test_assistant_with_project_integration_only(
156
162
  similarity_check.check_similarity(response, ado_wiki_answer)
157
163
 
158
164
 
165
+ @pytest.mark.integration
166
+ @pytest.mark.default_integration
159
167
  @pytest.mark.not_for_parallel_run
160
168
  @pytest.mark.regression
161
169
  def test_assistant_with_global_valid_and_user_invalid_integration(
@@ -196,6 +204,8 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
196
204
  similarity_check.check_similarity(response, ado_wiki_answer)
197
205
 
198
206
 
207
+ @pytest.mark.integration
208
+ @pytest.mark.default_integration
199
209
  @pytest.mark.not_for_parallel_run
200
210
  @pytest.mark.regression
201
211
  def test_assistant_with_project_valid_and_user_invalid_integration(
@@ -18,6 +18,8 @@ git_tool_prompt = list_branches_set_active_branch_test_data[0][2]
18
18
  git_tool_answer = list_branches_set_active_branch_test_data[0][3]
19
19
 
20
20
 
21
+ @pytest.mark.integration
22
+ @pytest.mark.default_integration
21
23
  @pytest.mark.not_for_parallel_run
22
24
  @pytest.mark.regression
23
25
  @pytest.mark.tescase("EPMCDME-6708")
@@ -89,6 +91,8 @@ def test_assistant_should_use_user_integration_by_default(
89
91
  similarity_check.check_similarity(response, git_tool_answer)
90
92
 
91
93
 
94
+ @pytest.mark.integration
95
+ @pytest.mark.default_integration
92
96
  @pytest.mark.not_for_parallel_run
93
97
  @pytest.mark.regression
94
98
  @pytest.mark.tescase("EPMCDME-6708")
@@ -146,6 +150,8 @@ def test_assistant_with_global_and_project_integration(
146
150
  similarity_check.check_similarity(response, git_tool_answer)
147
151
 
148
152
 
153
+ @pytest.mark.integration
154
+ @pytest.mark.default_integration
149
155
  @pytest.mark.not_for_parallel_run
150
156
  @pytest.mark.regression
151
157
  @pytest.mark.tescase("EPMCDME-6708")
@@ -190,6 +196,8 @@ def test_assistant_with_project_integration_only(
190
196
  similarity_check.check_similarity(response, git_tool_answer)
191
197
 
192
198
 
199
+ @pytest.mark.integration
200
+ @pytest.mark.default_integration
193
201
  @pytest.mark.not_for_parallel_run
194
202
  @pytest.mark.regression
195
203
  def test_assistant_with_global_valid_and_user_invalid_integration(
@@ -246,6 +254,8 @@ def test_assistant_with_global_valid_and_user_invalid_integration(
246
254
  similarity_check.check_similarity(response, git_tool_answer)
247
255
 
248
256
 
257
+ @pytest.mark.integration
258
+ @pytest.mark.default_integration
249
259
  @pytest.mark.not_for_parallel_run
250
260
  @pytest.mark.regression
251
261
  def test_assistant_with_project_valid_and_user_invalid_integration(
@@ -31,6 +31,7 @@ system_prompt_for_chatting_with_files = """Your task is to show the content of t
31
31
  """
32
32
 
33
33
 
34
+ @pytest.mark.assistant
34
35
  @pytest.mark.regression
35
36
  @pytest.mark.smoke
36
37
  def test_create_assistant(assistant_utils, default_llm):
@@ -41,6 +42,7 @@ def test_create_assistant(assistant_utils, default_llm):
41
42
  )
42
43
 
43
44
 
45
+ @pytest.mark.assistant
44
46
  @pytest.mark.regression
45
47
  @pytest.mark.smoke
46
48
  def test_get_all_assistants(search_utils):
@@ -50,6 +52,7 @@ def test_get_all_assistants(search_utils):
50
52
  assert_that(len(ids), greater_than(0), "No assistants found.")
51
53
 
52
54
 
55
+ @pytest.mark.assistant
53
56
  @pytest.mark.regression
54
57
  @pytest.mark.smoke
55
58
  def test_get_prebuilt_assistants(assistant_utils):
@@ -59,6 +62,7 @@ def test_get_prebuilt_assistants(assistant_utils):
59
62
  assert_that(len(ids), greater_than(0), "No prebuilt assistants found.")
60
63
 
61
64
 
65
+ @pytest.mark.assistant
62
66
  @pytest.mark.regression
63
67
  @pytest.mark.smoke
64
68
  def test_get_users_assistants(assistant_utils, search_utils):
@@ -69,6 +73,7 @@ def test_get_users_assistants(assistant_utils, search_utils):
69
73
  assert_that(TEST_USER in users, f"No assistants created by '{TEST_USER}' found.")
70
74
 
71
75
 
76
+ @pytest.mark.assistant
72
77
  @pytest.mark.regression
73
78
  @pytest.mark.smoke
74
79
  def test_get_assistant_context(assistant_utils):
@@ -77,6 +82,7 @@ def test_get_assistant_context(assistant_utils):
77
82
  assert_that(len(response), greater_than(0), "No assistants context found.")
78
83
 
79
84
 
85
+ @pytest.mark.assistant
80
86
  @pytest.mark.regression
81
87
  @pytest.mark.smoke
82
88
  def test_get_available_tools(assistant_utils):
@@ -85,6 +91,7 @@ def test_get_available_tools(assistant_utils):
85
91
  assert_that(len(response), greater_than(0), "No assistants tools found.")
86
92
 
87
93
 
94
+ @pytest.mark.assistant
88
95
  @pytest.mark.regression
89
96
  @pytest.mark.smoke
90
97
  def test_get_assistant_by_id(assistant_utils):
@@ -100,6 +107,7 @@ def test_get_assistant_by_id(assistant_utils):
100
107
  )
101
108
 
102
109
 
110
+ @pytest.mark.assistant
103
111
  @pytest.mark.regression
104
112
  @pytest.mark.smoke
105
113
  def test_get_assistant_by_slug(assistant_utils):
@@ -115,6 +123,7 @@ def test_get_assistant_by_slug(assistant_utils):
115
123
  )
116
124
 
117
125
 
126
+ @pytest.mark.assistant
118
127
  @pytest.mark.regression
119
128
  @pytest.mark.smoke
120
129
  def test_update_assistant(assistant_utils, default_llm):
@@ -152,6 +161,7 @@ def test_update_assistant(assistant_utils, default_llm):
152
161
  )
153
162
 
154
163
 
164
+ @pytest.mark.assistant
155
165
  @pytest.mark.regression
156
166
  @pytest.mark.smoke
157
167
  def test_delete_assistant(assistant_utils, search_utils):
@@ -169,6 +179,7 @@ def test_delete_assistant(assistant_utils, search_utils):
169
179
  assert_that(len(search_response), equal_to(0), "Assistant was not deleted")
170
180
 
171
181
 
182
+ @pytest.mark.assistant
172
183
  @pytest.mark.regression
173
184
  @pytest.mark.smoke
174
185
  def test_ask_assistant(assistant_utils):
@@ -187,6 +198,7 @@ def test_ask_assistant(assistant_utils):
187
198
  assert_that(response.task_id, is_not(empty()), "No response from assistant")
188
199
 
189
200
 
201
+ @pytest.mark.assistant
190
202
  @pytest.mark.regression
191
203
  @pytest.mark.skip(reason="Not implemented yet")
192
204
  def test_export_assistant(assistant_utils):
@@ -202,6 +214,8 @@ def test_export_assistant(assistant_utils):
202
214
  )
203
215
 
204
216
 
217
+ @pytest.mark.assistant
218
+ @pytest.mark.file
205
219
  @pytest.mark.regression
206
220
  @pytest.mark.smoke
207
221
  @pytest.mark.tescase("EPMCDME-4001, EPMCDME-4002, EPMCDME-2527")
@@ -242,6 +256,7 @@ def test_create_assistant_and_prompt_with_file(
242
256
  similarity_check.check_similarity(response, expected_response)
243
257
 
244
258
 
259
+ @pytest.mark.assistant
245
260
  @pytest.mark.regression
246
261
  @pytest.mark.smoke
247
262
  def test_assistant_has_an_access_to_the_history(
@@ -268,6 +283,7 @@ def test_assistant_has_an_access_to_the_history(
268
283
  similarity_check.check_similarity(response, "2 + 5 equals 7.", 90)
269
284
 
270
285
 
286
+ @pytest.mark.assistant
271
287
  @pytest.mark.regression
272
288
  @pytest.mark.smoke
273
289
  def test_passing_history_to_chat(
@@ -293,12 +309,14 @@ def test_passing_history_to_chat(
293
309
  similarity_check.check_similarity(response, "3 + 5 equals 8.", 90)
294
310
 
295
311
 
312
+ @pytest.mark.assistant
313
+ @pytest.mark.file
314
+ @pytest.mark.regression
315
+ @pytest.mark.smoke
296
316
  @pytest.mark.parametrize(
297
317
  "file_name",
298
318
  files_with_different_types_test_data,
299
319
  )
300
- @pytest.mark.regression
301
- @pytest.mark.smoke
302
320
  def test_upload_file_to_chat(
303
321
  assistant_utils,
304
322
  file_name,
@@ -310,6 +328,7 @@ def test_upload_file_to_chat(
310
328
  raise AssertionError(f"Error on uploading file to chat: {error_details}")
311
329
 
312
330
 
331
+ @pytest.mark.assistant
313
332
  @pytest.mark.regression
314
333
  @pytest.mark.smoke
315
334
  def test_chat_with_pydantic_output_schema(assistant_utils, assistant):
@@ -323,6 +342,7 @@ def test_chat_with_pydantic_output_schema(assistant_utils, assistant):
323
342
  assert_that(response.results[0], equal_to(2))
324
343
 
325
344
 
345
+ @pytest.mark.assistant
326
346
  @pytest.mark.regression
327
347
  @pytest.mark.smoke
328
348
  def test_chat_with_json_output_schema(assistant_utils, assistant):
@@ -333,6 +353,8 @@ def test_chat_with_json_output_schema(assistant_utils, assistant):
333
353
  assert_that(response["results"][0], equal_to(2))
334
354
 
335
355
 
356
+ @pytest.mark.assistant
357
+ @pytest.mark.file
336
358
  @pytest.mark.regression
337
359
  @pytest.mark.smoke
338
360
  def test_create_assistant_and_prompt_with_multiple_files(
@@ -7,6 +7,8 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
7
7
  )
8
8
 
9
9
 
10
+ @pytest.mark.assistant
11
+ @pytest.mark.keycloak
10
12
  @pytest.mark.regression
11
13
  def test_assistant_with_keycloak_tool(
12
14
  assistant_utils,
@@ -16,6 +16,8 @@ from codemie_test_harness.tests.utils.constants import ID_PATTERN
16
16
  from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
17
17
 
18
18
 
19
+ @pytest.mark.assistant
20
+ @pytest.mark.ado
19
21
  @pytest.mark.regression
20
22
  @pytest.mark.parametrize(
21
23
  "toolkit,tool_name,prompt,expected_response",
@@ -46,6 +48,8 @@ def test_assistant_with_ado_test_plan_get_tools(
46
48
  similarity_check.check_similarity(response, expected_response)
47
49
 
48
50
 
51
+ @pytest.mark.assistant
52
+ @pytest.mark.ado
49
53
  @pytest.mark.regression
50
54
  def test_assistant_with_ado_test_plan_tools(
51
55
  assistant_utils,
@@ -13,6 +13,8 @@ from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUti
13
13
  from codemie_test_harness.tests.utils.base_utils import get_random_name
14
14
 
15
15
 
16
+ @pytest.mark.assistant
17
+ @pytest.mark.ado
16
18
  @pytest.mark.regression
17
19
  @pytest.mark.parametrize(
18
20
  "toolkit,tool_name,prompt,expected_response",
@@ -43,6 +45,8 @@ def test_assistant_with_ado_wiki_get_tools(
43
45
  similarity_check.check_similarity(response, expected_response)
44
46
 
45
47
 
48
+ @pytest.mark.assistant
49
+ @pytest.mark.ado
46
50
  @pytest.mark.regression
47
51
  def test_assistant_with_ado_wiki_modify_tools(
48
52
  assistant_utils,
@@ -14,6 +14,8 @@ from codemie_test_harness.tests.utils.constants import WORK_ITEM_ID_PATTERN
14
14
  from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
15
15
 
16
16
 
17
+ @pytest.mark.assistant
18
+ @pytest.mark.ado
17
19
  @pytest.mark.regression
18
20
  @pytest.mark.parametrize(
19
21
  "toolkit,tool_name,prompt,expected_response",
@@ -44,6 +46,8 @@ def test_assistant_with_ado_work_item_get_tools(
44
46
  similarity_check.check_similarity(response, expected_response)
45
47
 
46
48
 
49
+ @pytest.mark.assistant
50
+ @pytest.mark.ado
47
51
  @pytest.mark.regression
48
52
  def test_assistant_with_ado_work_item_modify_tools(
49
53
  assistant_utils,
@@ -4,6 +4,8 @@ from codemie_test_harness.tests.enums.tools import Toolkit, CloudTool
4
4
  from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
5
5
 
6
6
 
7
+ @pytest.mark.assistant
8
+ @pytest.mark.cloud
7
9
  @pytest.mark.regression
8
10
  @pytest.mark.parametrize(
9
11
  "toolkit,tool_name,credential_type,credentials,prompt,expected_response",
@@ -10,6 +10,8 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
10
10
  )
11
11
 
12
12
 
13
+ @pytest.mark.assistant
14
+ @pytest.mark.codebase
13
15
  @pytest.mark.regression
14
16
  @pytest.mark.parametrize(
15
17
  "toolkit,tool_name,prompt,expected_response",
@@ -32,6 +34,7 @@ def test_assistant_with_codebase_tools(
32
34
  similarity_check.check_similarity(response, expected_response)
33
35
 
34
36
 
37
+ @pytest.mark.assistant
35
38
  @pytest.mark.regression
36
39
  @pytest.mark.parametrize(
37
40
  "toolkit, tool_name, credentials, prompt, expected",
@@ -21,10 +21,15 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
21
21
  )
22
22
  from codemie_test_harness.tests.utils.aws_parameters_store import CredentialsUtil
23
23
 
24
+ pytestmark = pytest.mark.skipif(
25
+ os.getenv("ENV") == "local", reason="Skipping this tests on local environment"
26
+ )
27
+
24
28
 
29
+ @pytest.mark.assistant
30
+ @pytest.mark.elastic
25
31
  @pytest.mark.regression
26
32
  @pytest.mark.testcase("EPMCDME-6132")
27
- @pytest.mark.skip(reason="Credentials for Elastic are not available")
28
33
  def test_create_assistant_with_elastic_tool(
29
34
  integration_utils, assistant, assistant_utils, similarity_check
30
35
  ):
@@ -43,7 +48,10 @@ def test_create_assistant_with_elastic_tool(
43
48
  similarity_check.check_similarity(response, RESPONSE_FOR_ELASTIC)
44
49
 
45
50
 
51
+ @pytest.mark.assistant
52
+ @pytest.mark.sql
46
53
  @pytest.mark.regression
54
+ @pytest.mark.testcase("EPMCDME-6132")
47
55
  @pytest.mark.parametrize(
48
56
  "db_dialect",
49
57
  sql_tools_test_data,
@@ -19,7 +19,10 @@ from codemie_test_harness.tests.test_data.file_management_tools_test_data import
19
19
  from codemie_test_harness.tests.utils.base_utils import get_random_name
20
20
 
21
21
 
22
+ @pytest.mark.assistant
23
+ @pytest.mark.file_management
22
24
  @pytest.mark.regression
25
+ @pytest.mark.testcase("EPMCDME-6103")
23
26
  @pytest.mark.parametrize(
24
27
  "tool_name,prompt,expected_response",
25
28
  file_management_tools_test_data,
@@ -30,7 +33,6 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
30
33
  FileManagementTool.RUN_COMMAND_LINE,
31
34
  ],
32
35
  )
33
- @pytest.mark.testcase("EPMCDME-6103")
34
36
  def test_create_assistant_with_file_management_tool(
35
37
  filesystem_integration,
36
38
  assistant,
@@ -49,6 +51,8 @@ def test_create_assistant_with_file_management_tool(
49
51
  similarity_check.check_similarity(response, expected_response)
50
52
 
51
53
 
54
+ @pytest.mark.assistant
55
+ @pytest.mark.file_management
52
56
  @pytest.mark.regression
53
57
  @pytest.mark.testcase("EPMCDME-6103")
54
58
  @pytest.mark.skipif(
@@ -74,6 +78,8 @@ def test_create_assistant_with_file_management_generate_image_tool(
74
78
  )
75
79
 
76
80
 
81
+ @pytest.mark.assistant
82
+ @pytest.mark.file_management
77
83
  @pytest.mark.regression
78
84
  @pytest.mark.testcase("EPMCDME-6103")
79
85
  def test_create_assistant_with_file_management_read_file_tool(
@@ -93,6 +99,8 @@ def test_create_assistant_with_file_management_read_file_tool(
93
99
 
94
100
 
95
101
  @pytest.mark.skip(reason="Tests are flaky, tools work unstable")
102
+ @pytest.mark.assistant
103
+ @pytest.mark.file_management
96
104
  @pytest.mark.regression
97
105
  @pytest.mark.parametrize(
98
106
  "tool_name,expected_response",