codemie-test-harness 0.1.179__py3-none-any.whl → 0.1.181__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (134) hide show
  1. codemie_test_harness/cli/cli.py +2 -0
  2. codemie_test_harness/cli/commands/marks_cmd.py +39 -0
  3. codemie_test_harness/cli/marks_utils.py +338 -0
  4. codemie_test_harness/cli/runner.py +40 -0
  5. codemie_test_harness/tests/assistant/datasource/test_code_datasource.py +20 -7
  6. codemie_test_harness/tests/assistant/datasource/test_confluence_datasource.py +20 -7
  7. codemie_test_harness/tests/assistant/datasource/test_file_indexing.py +20 -9
  8. codemie_test_harness/tests/assistant/datasource/test_google_datasource.py +21 -7
  9. codemie_test_harness/tests/assistant/datasource/test_jira_datasource.py +22 -9
  10. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool.py +29 -13
  11. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_kit.py +29 -13
  12. codemie_test_harness/tests/assistant/default_integrations/test_default_integrations_for_tool_with_datasource.py +29 -13
  13. codemie_test_harness/tests/assistant/test_assistants.py +53 -35
  14. codemie_test_harness/tests/assistant/tools/access_management/test_keycloak_tool.py +7 -2
  15. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_test_plan_tools.py +39 -11
  16. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_wiki_tools.py +27 -8
  17. codemie_test_harness/tests/assistant/tools/ado/test_assistant_for_ado_work_item_tools.py +27 -7
  18. codemie_test_harness/tests/assistant/tools/cloud/test_cloud_tools.py +9 -3
  19. codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py +13 -4
  20. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +32 -11
  21. codemie_test_harness/tests/assistant/tools/filemanagement/test_assistant_with_file_management_tools.py +37 -12
  22. codemie_test_harness/tests/assistant/tools/git/test_assistant_with_git_tools.py +48 -17
  23. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +16 -9
  24. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +14 -8
  25. codemie_test_harness/tests/assistant/tools/notification/test_assistant_notification_tools.py +11 -4
  26. codemie_test_harness/tests/assistant/tools/openapi/test_assistant_with_open_api_tools.py +6 -2
  27. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_development_plugin.py +27 -8
  28. codemie_test_harness/tests/assistant/tools/plugin/test_assistant_with_plugin_and_mcp_servers.py +17 -5
  29. codemie_test_harness/tests/assistant/tools/plugin/test_single_assistant_dual_time_plugins.py +12 -3
  30. codemie_test_harness/tests/assistant/tools/project_management/test_assistant_pm_tools.py +36 -17
  31. codemie_test_harness/tests/assistant/tools/report_portal/test_assistant_report_portal_tools.py +7 -2
  32. codemie_test_harness/tests/assistant/tools/research/test_assistant_research_tools.py +17 -5
  33. codemie_test_harness/tests/assistant/tools/servicenow/test_servicenow_tools.py +6 -2
  34. codemie_test_harness/tests/assistant/tools/vcs/test_assistant_with_vcs_tools.py +7 -2
  35. codemie_test_harness/tests/conversations/test_conversations_endpoints.py +5 -5
  36. codemie_test_harness/tests/e2e/test_e2e.py +6 -6
  37. codemie_test_harness/tests/enums/tools.py +32 -0
  38. codemie_test_harness/tests/integrations/project/test_default_integrations.py +14 -14
  39. codemie_test_harness/tests/integrations/project/test_project_integrations.py +9 -9
  40. codemie_test_harness/tests/integrations/user/test_default_integrations.py +14 -14
  41. codemie_test_harness/tests/integrations/user/test_user_integrations.py +12 -12
  42. codemie_test_harness/tests/llm/assistants/test_lite_llm.py +2 -2
  43. codemie_test_harness/tests/llm/assistants/test_llm.py +3 -3
  44. codemie_test_harness/tests/providers/test_providers_endpoints.py +11 -11
  45. codemie_test_harness/tests/search/test_search_assistant.py +1 -1
  46. codemie_test_harness/tests/search/test_search_datasource.py +5 -5
  47. codemie_test_harness/tests/search/test_search_integration.py +2 -2
  48. codemie_test_harness/tests/search/test_search_workflow.py +1 -1
  49. codemie_test_harness/tests/service/test_assistant_service.py +1 -1
  50. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +11 -5
  51. codemie_test_harness/tests/test_data/file_test_data.py +17 -0
  52. codemie_test_harness/tests/ui/assistants/test_create_assistant.py +11 -11
  53. codemie_test_harness/tests/ui/datasource/test_create_datasource.py +7 -7
  54. codemie_test_harness/tests/ui/datasource/test_datasource_page.py +3 -3
  55. codemie_test_harness/tests/ui/datasource/test_edit_datasource.py +5 -5
  56. codemie_test_harness/tests/ui/datasource/test_view_datasource.py +5 -5
  57. codemie_test_harness/tests/ui/integrations/test_create_integration.py +9 -9
  58. codemie_test_harness/tests/ui/workflows/test_create_workflow.py +12 -12
  59. codemie_test_harness/tests/ui/workflows/test_edit_workflow.py +15 -15
  60. codemie_test_harness/tests/ui/workflows/test_workflow_details.py +19 -19
  61. codemie_test_harness/tests/ui/workflows/test_workflow_executions_page.py +36 -36
  62. codemie_test_harness/tests/ui/workflows/test_workflow_templates.py +8 -8
  63. codemie_test_harness/tests/ui/workflows/test_workflows.py +8 -8
  64. codemie_test_harness/tests/utils/assistant_utils.py +34 -1
  65. codemie_test_harness/tests/utils/base_utils.py +61 -0
  66. codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +1 -1
  67. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +2 -2
  68. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +2 -2
  69. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +2 -2
  70. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +1 -1
  71. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +2 -2
  72. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +2 -2
  73. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +8 -8
  74. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +8 -8
  75. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +8 -8
  76. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +5 -5
  77. codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +7 -7
  78. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +3 -3
  79. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +2 -2
  80. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +1 -1
  81. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +2 -2
  82. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +2 -2
  83. codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +1 -1
  84. codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +1 -1
  85. codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +2 -2
  86. codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +1 -1
  87. codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +1 -1
  88. codemie_test_harness/tests/workflow/config_validation/test_config_validation.py +1 -1
  89. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool.py +8 -8
  90. codemie_test_harness/tests/workflow/direct_tools_calling/default_integrations/test_default_integrations_for_tool_kit.py +8 -8
  91. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_access_management_tool.py +3 -3
  92. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_test_plan_tools.py +3 -3
  93. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_wiki_tools.py +3 -3
  94. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_ado_work_item_tools.py +3 -3
  95. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +3 -3
  96. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_codebase_tools.py +3 -3
  97. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_elastic.py +3 -3
  98. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_data_management_tools_sql.py +3 -3
  99. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_file_management_tools.py +3 -3
  100. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_notification_tools.py +3 -3
  101. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_open_api_tools.py +3 -3
  102. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_plugin_tools.py +4 -4
  103. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_project_management_tools.py +3 -3
  104. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_report_portal_tools.py +3 -3
  105. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_research_tools.py +3 -3
  106. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_servicenow_tools.py +3 -3
  107. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_vcs_tools.py +3 -3
  108. codemie_test_harness/tests/workflow/test_workflows.py +1 -1
  109. codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +1 -1
  110. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +2 -2
  111. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +2 -2
  112. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +2 -2
  113. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +4 -4
  114. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +2 -2
  115. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +2 -2
  116. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +8 -8
  117. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +8 -8
  118. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +8 -8
  119. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +5 -5
  120. codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +7 -7
  121. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +3 -3
  122. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +4 -4
  123. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +2 -2
  124. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +2 -2
  125. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +2 -2
  126. codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +2 -2
  127. codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +1 -1
  128. codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +2 -2
  129. codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +1 -1
  130. codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +1 -1
  131. {codemie_test_harness-0.1.179.dist-info → codemie_test_harness-0.1.181.dist-info}/METADATA +8 -8
  132. {codemie_test_harness-0.1.179.dist-info → codemie_test_harness-0.1.181.dist-info}/RECORD +134 -132
  133. {codemie_test_harness-0.1.179.dist-info → codemie_test_harness-0.1.181.dist-info}/WHEEL +0 -0
  134. {codemie_test_harness-0.1.179.dist-info → codemie_test_harness-0.1.181.dist-info}/entry_points.txt +0 -0
@@ -73,8 +73,8 @@ def executions(test_workflow, workflow_utils):
73
73
  # ==================== PAGE STRUCTURE & VISIBILITY TESTS ====================
74
74
 
75
75
 
76
- @pytest.mark.workflow_ui
77
- @pytest.mark.workflow_execution_ui
76
+ @pytest.mark.workflow
77
+ @pytest.mark.workflow_execution
78
78
  @pytest.mark.ui
79
79
  def test_workflow_executions_page_structure(page, executions, test_workflow):
80
80
  """Test that all main page elements are visible and properly structured."""
@@ -91,8 +91,8 @@ def test_workflow_executions_page_structure(page, executions, test_workflow):
91
91
  executions_page.should_have_workflow_title(test_workflow.name)
92
92
 
93
93
 
94
- @pytest.mark.workflow_ui
95
- @pytest.mark.workflow_execution_ui
94
+ @pytest.mark.workflow
95
+ @pytest.mark.workflow_execution
96
96
  @pytest.mark.ui
97
97
  def test_header_elements_visibility(page, executions):
98
98
  """Test that all header elements are visible and functional."""
@@ -110,8 +110,8 @@ def test_header_elements_visibility(page, executions):
110
110
  # ==================== EXECUTION HISTORY SIDEBAR TESTS ====================
111
111
 
112
112
 
113
- @pytest.mark.workflow_ui
114
- @pytest.mark.workflow_execution_ui
113
+ @pytest.mark.workflow
114
+ @pytest.mark.workflow_execution
115
115
  @pytest.mark.ui
116
116
  def test_execution_history_sidebar_content(page, executions):
117
117
  """Test execution history sidebar content and structure."""
@@ -126,8 +126,8 @@ def test_execution_history_sidebar_content(page, executions):
126
126
  executions_page.should_have_execution_history_items()
127
127
 
128
128
 
129
- @pytest.mark.workflow_ui
130
- @pytest.mark.workflow_execution_ui
129
+ @pytest.mark.workflow
130
+ @pytest.mark.workflow_execution
131
131
  @pytest.mark.ui
132
132
  def test_execution_history_item_interactions(page, executions):
133
133
  """Test interactions with individual execution history items."""
@@ -144,8 +144,8 @@ def test_execution_history_item_interactions(page, executions):
144
144
  first_execution.should_have_all_elements_visible()
145
145
 
146
146
 
147
- @pytest.mark.workflow_ui
148
- @pytest.mark.workflow_execution_ui
147
+ @pytest.mark.workflow
148
+ @pytest.mark.workflow_execution
149
149
  @pytest.mark.ui
150
150
  def test_execution_history_status_display(page, executions):
151
151
  """Test execution status display and styling in sidebar."""
@@ -164,8 +164,8 @@ def test_execution_history_status_display(page, executions):
164
164
  # ==================== MAIN CONTENT STATUS & METADATA TESTS ====================
165
165
 
166
166
 
167
- @pytest.mark.workflow_ui
168
- @pytest.mark.workflow_execution_ui
167
+ @pytest.mark.workflow
168
+ @pytest.mark.workflow_execution
169
169
  @pytest.mark.ui
170
170
  def test_execution_status_and_metadata_display(page, executions):
171
171
  """Test execution status and metadata display in main content."""
@@ -190,8 +190,8 @@ def test_execution_status_and_metadata_display(page, executions):
190
190
  )
191
191
 
192
192
 
193
- @pytest.mark.workflow_ui
194
- @pytest.mark.workflow_execution_ui
193
+ @pytest.mark.workflow
194
+ @pytest.mark.workflow_execution
195
195
  @pytest.mark.ui
196
196
  def test_execution_action_buttons(page, executions):
197
197
  """Test execution action buttons functionality."""
@@ -207,8 +207,8 @@ def test_execution_action_buttons(page, executions):
207
207
  # ==================== PROMPT SECTION TESTS ====================
208
208
 
209
209
 
210
- @pytest.mark.workflow_ui
211
- @pytest.mark.workflow_execution_ui
210
+ @pytest.mark.workflow
211
+ @pytest.mark.workflow_execution
212
212
  @pytest.mark.ui
213
213
  def test_prompt_section_display(page, executions):
214
214
  """Test prompt section display and content."""
@@ -225,8 +225,8 @@ def test_prompt_section_display(page, executions):
225
225
  executions_page.should_have_prompt_text(executions[0].prompt)
226
226
 
227
227
 
228
- @pytest.mark.workflow_ui
229
- @pytest.mark.workflow_execution_ui
228
+ @pytest.mark.workflow
229
+ @pytest.mark.workflow_execution
230
230
  @pytest.mark.ui
231
231
  def test_prompt_action_buttons(page, executions):
232
232
  """Test prompt action buttons (info, copy, download)."""
@@ -246,8 +246,8 @@ def test_prompt_action_buttons(page, executions):
246
246
  # ==================== EXECUTION STATES TESTS ====================
247
247
 
248
248
 
249
- @pytest.mark.workflow_ui
250
- @pytest.mark.workflow_execution_ui
249
+ @pytest.mark.workflow
250
+ @pytest.mark.workflow_execution
251
251
  @pytest.mark.ui
252
252
  def test_execution_states_display(page, executions):
253
253
  """Test execution states section display."""
@@ -262,8 +262,8 @@ def test_execution_states_display(page, executions):
262
262
  executions_page.should_have_execution_states()
263
263
 
264
264
 
265
- @pytest.mark.workflow_ui
266
- @pytest.mark.workflow_execution_ui
265
+ @pytest.mark.workflow
266
+ @pytest.mark.workflow_execution
267
267
  @pytest.mark.ui
268
268
  def test_execution_state_content(page, executions):
269
269
  """Test individual execution state content and functionality."""
@@ -278,8 +278,8 @@ def test_execution_state_content(page, executions):
278
278
  first_state.should_have_main_elements_visible()
279
279
 
280
280
 
281
- @pytest.mark.workflow_ui
282
- @pytest.mark.workflow_execution_ui
281
+ @pytest.mark.workflow
282
+ @pytest.mark.workflow_execution
283
283
  @pytest.mark.ui
284
284
  def test_states_expand_collapse_functionality(page, executions):
285
285
  """Test expand/collapse functionality for states."""
@@ -297,8 +297,8 @@ def test_states_expand_collapse_functionality(page, executions):
297
297
  first_state.should_be_collapsed()
298
298
 
299
299
 
300
- @pytest.mark.workflow_ui
301
- @pytest.mark.workflow_execution_ui
300
+ @pytest.mark.workflow
301
+ @pytest.mark.workflow_execution
302
302
  @pytest.mark.ui
303
303
  def test_execution_state_output_interaction(page, executions):
304
304
  """Test execution state output interaction."""
@@ -316,8 +316,8 @@ def test_execution_state_output_interaction(page, executions):
316
316
  # ==================== CONFIGURATION SIDEBAR TESTS ====================
317
317
 
318
318
 
319
- @pytest.mark.workflow_ui
320
- @pytest.mark.workflow_execution_ui
319
+ @pytest.mark.workflow
320
+ @pytest.mark.workflow_execution
321
321
  @pytest.mark.ui
322
322
  def test_configuration_sidebar_toggle(page, executions):
323
323
  """Test configuration sidebar toggle functionality."""
@@ -336,8 +336,8 @@ def test_configuration_sidebar_toggle(page, executions):
336
336
  executions_page.should_have_configuration_sidebar_closed()
337
337
 
338
338
 
339
- @pytest.mark.workflow_ui
340
- @pytest.mark.workflow_execution_ui
339
+ @pytest.mark.workflow
340
+ @pytest.mark.workflow_execution
341
341
  @pytest.mark.ui
342
342
  def test_configuration_sidebar_content(page, executions, test_workflow):
343
343
  """Test configuration sidebar content when open."""
@@ -355,8 +355,8 @@ def test_configuration_sidebar_content(page, executions, test_workflow):
355
355
  )
356
356
 
357
357
 
358
- @pytest.mark.workflow_ui
359
- @pytest.mark.workflow_execution_ui
358
+ @pytest.mark.workflow
359
+ @pytest.mark.workflow_execution
360
360
  @pytest.mark.ui
361
361
  def test_yaml_configuration_display(page, executions):
362
362
  """Test YAML configuration display in sidebar."""
@@ -375,8 +375,8 @@ def test_yaml_configuration_display(page, executions):
375
375
  # ==================== INTEGRATION & NAVIGATION TESTS ====================
376
376
 
377
377
 
378
- @pytest.mark.workflow_ui
379
- @pytest.mark.workflow_execution_ui
378
+ @pytest.mark.workflow
379
+ @pytest.mark.workflow_execution
380
380
  @pytest.mark.ui
381
381
  def test_execution_history_navigation(page, executions):
382
382
  """Test navigation between different executions in history."""
@@ -395,8 +395,8 @@ def test_execution_history_navigation(page, executions):
395
395
  executions_page.should_have_active_execution_item()
396
396
 
397
397
 
398
- @pytest.mark.workflow_ui
399
- @pytest.mark.workflow_execution_ui
398
+ @pytest.mark.workflow
399
+ @pytest.mark.workflow_execution
400
400
  @pytest.mark.ui
401
401
  def test_page_state_persistence(page, executions):
402
402
  """Test that page state persists during interactions."""
@@ -22,8 +22,8 @@ def predefined_templates(workflow_utils):
22
22
  return workflow_utils.get_prebuilt_workflows()
23
23
 
24
24
 
25
- @pytest.mark.workflow_ui
26
- @pytest.mark.workflow_templates_ui
25
+ @pytest.mark.workflow
26
+ @pytest.mark.workflow_templates
27
27
  @pytest.mark.ui
28
28
  def test_templates_visible(page, predefined_templates):
29
29
  """Verify that workflow templates are presented and tooltips visible for the user."""
@@ -43,8 +43,8 @@ def test_templates_visible(page, predefined_templates):
43
43
  )
44
44
 
45
45
 
46
- @pytest.mark.workflow_ui
47
- @pytest.mark.workflow_templates_ui
46
+ @pytest.mark.workflow
47
+ @pytest.mark.workflow_templates
48
48
  @pytest.mark.ui
49
49
  def test_templates_details(page, predefined_templates):
50
50
  """Verify that workflow template leads to specific template page."""
@@ -68,8 +68,8 @@ def test_templates_details(page, predefined_templates):
68
68
  page.go_back()
69
69
 
70
70
 
71
- @pytest.mark.workflow_ui
72
- @pytest.mark.workflow_templates_ui
71
+ @pytest.mark.workflow
72
+ @pytest.mark.workflow_templates
73
73
  @pytest.mark.ui
74
74
  def test_templates_creation(page, predefined_templates):
75
75
  """Verify that workflow template create button leads to Create Workflow Page."""
@@ -94,8 +94,8 @@ def test_templates_creation(page, predefined_templates):
94
94
  page.go_back()
95
95
 
96
96
 
97
- @pytest.mark.workflow_ui
98
- @pytest.mark.workflow_templates_ui
97
+ @pytest.mark.workflow
98
+ @pytest.mark.workflow_templates
99
99
  @pytest.mark.ui
100
100
  def test_template_create(page, predefined_templates):
101
101
  """Verify that workflow template can be created."""
@@ -6,7 +6,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
6
6
  from codemie_test_harness.tests import TEST_USER
7
7
 
8
8
 
9
- @pytest.mark.workflow_ui
9
+ @pytest.mark.workflow
10
10
  @pytest.mark.ui
11
11
  def test_workflows_page_basic_functionality(page):
12
12
  """Test basic workflows page functionality."""
@@ -17,7 +17,7 @@ def test_workflows_page_basic_functionality(page):
17
17
  workflow_page.should_see_workflow_cards()
18
18
 
19
19
 
20
- @pytest.mark.workflow_ui
20
+ @pytest.mark.workflow
21
21
  @pytest.mark.ui
22
22
  def test_workflow_card_interactions(page):
23
23
  """Test individual workflow card interactions."""
@@ -32,7 +32,7 @@ def test_workflow_card_interactions(page):
32
32
  workflow_card.should_have_menu_button()
33
33
 
34
34
 
35
- @pytest.mark.workflow_ui
35
+ @pytest.mark.workflow
36
36
  @pytest.mark.ui
37
37
  def test_workflow_search_functionality(page):
38
38
  """Test workflow search functionality."""
@@ -44,7 +44,7 @@ def test_workflow_search_functionality(page):
44
44
  workflow_page.should_not_see_new_release_popup()
45
45
 
46
46
 
47
- @pytest.mark.workflow_ui
47
+ @pytest.mark.workflow
48
48
  @pytest.mark.ui
49
49
  def test_workflow_card_by_name(page, workflow_with_virtual_assistant):
50
50
  """Test getting and interacting with a specific workflow card."""
@@ -58,7 +58,7 @@ def test_workflow_card_by_name(page, workflow_with_virtual_assistant):
58
58
  workflow_page.should_see_shared_workflow(workflow.name)
59
59
 
60
60
 
61
- @pytest.mark.workflow_ui
61
+ @pytest.mark.workflow
62
62
  @pytest.mark.ui
63
63
  def test_workflow_pagination(page):
64
64
  """Test workflow pagination functionality."""
@@ -71,7 +71,7 @@ def test_workflow_pagination(page):
71
71
  workflow_page.should_see_workflow_cards(minimum_count=12)
72
72
 
73
73
 
74
- @pytest.mark.workflow_ui
74
+ @pytest.mark.workflow
75
75
  @pytest.mark.ui
76
76
  def test_workflow_filtering(page):
77
77
  """Test workflow filtering functionality."""
@@ -83,7 +83,7 @@ def test_workflow_filtering(page):
83
83
  workflow_page.clear_all_filters()
84
84
 
85
85
 
86
- @pytest.mark.workflow_ui
86
+ @pytest.mark.workflow
87
87
  @pytest.mark.ui
88
88
  def test_workflow_navigation(page):
89
89
  """Test navigation between different workflow sections."""
@@ -94,7 +94,7 @@ def test_workflow_navigation(page):
94
94
  workflow_page.sidebar.navigate_to_templates()
95
95
 
96
96
 
97
- @pytest.mark.workflow_ui
97
+ @pytest.mark.workflow
98
98
  @pytest.mark.ui
99
99
  def test_workflow_card_fluent_interface(page):
100
100
  """Test the fluent interface of WorkflowCard component."""
@@ -125,7 +125,40 @@ class AssistantUtils(BaseUtils):
125
125
  assistant_id=assistant.id, request=chat_request
126
126
  )
127
127
 
128
- return response.generated if minimal_response else response
128
+ if minimal_response:
129
+ return response.generated
130
+ else:
131
+ # Extract triggered tools from response thoughts
132
+ triggered_tools = self._extract_triggered_tools(response)
133
+ return response.generated, triggered_tools
134
+
135
+ def _extract_triggered_tools(self, response):
136
+ """
137
+ Extract triggered tools from response thoughts.
138
+
139
+ Filters out 'Codemie Thoughts' entries and error entries,
140
+ returning a list of tool names in lowercase.
141
+
142
+ Args:
143
+ response: The assistant response containing thoughts
144
+
145
+ Returns:
146
+ list: List of triggered tool names in lowercase
147
+ """
148
+ triggered_tools = []
149
+
150
+ # Check if response has thoughts attribute
151
+ if hasattr(response, "thoughts") and response.thoughts:
152
+ for thought in response.thoughts:
153
+ # Filter out 'Codemie Thoughts' and error entries
154
+ if thought.get("author_name") != "Codemie Thoughts" and not thought.get(
155
+ "error", False
156
+ ):
157
+ author_name = thought.get("author_name", "")
158
+ if author_name:
159
+ triggered_tools.append(author_name.lower())
160
+
161
+ return triggered_tools
129
162
 
130
163
  def send_chat_request(
131
164
  self,
@@ -173,3 +173,64 @@ def assert_error_details(response, status_code, message):
173
173
 
174
174
  def credentials_to_dict(credentials):
175
175
  return {cred.key: cred.value for cred in credentials}
176
+
177
+
178
+ def assert_tool_triggered(tool_name, triggered_tools):
179
+ """
180
+ Assert that the expected tool(s) were triggered during assistant interaction.
181
+
182
+ Args:
183
+ tool_name: Either a single tool enum or a tuple of tool enums that should be triggered
184
+ triggered_tools: List of tools that were actually triggered
185
+
186
+ Raises:
187
+ AssertionError: If any of the expected tools were not found in triggered_tools
188
+ (for tuples, ALL tools must be present)
189
+ """
190
+ # Handle both single tools and tuples of tools
191
+ if isinstance(tool_name, tuple):
192
+ tools_to_check = tool_name
193
+ else:
194
+ tools_to_check = (tool_name,)
195
+
196
+ # Check each expected tool
197
+ found_tools = []
198
+ missing_tools = []
199
+
200
+ for tool in tools_to_check:
201
+ tool_value_lower = tool.value.lower()
202
+ tool_value_with_spaces = tool.value.replace("_", " ").lower()
203
+
204
+ # Check if this specific tool was triggered
205
+ tool_found = False
206
+ for triggered_tool in triggered_tools:
207
+ if (
208
+ triggered_tool.lower() == tool_value_lower
209
+ or triggered_tool.lower() == tool_value_with_spaces
210
+ or tool_value_with_spaces in triggered_tool.lower()
211
+ ):
212
+ found_tools.append(tool.value)
213
+ tool_found = True
214
+ break
215
+
216
+ if not tool_found:
217
+ missing_tools.append(tool.value)
218
+
219
+ # Assert that ALL expected tools were found
220
+ if missing_tools:
221
+ expected_tools = [tool.value for tool in tools_to_check]
222
+
223
+ if len(tools_to_check) == 1:
224
+ assert False, (
225
+ f"Tool validation failed:\n"
226
+ f"Expected tool '{expected_tools[0]}' to be triggered\n"
227
+ f"But it was not found in triggered tools: {triggered_tools}\n"
228
+ )
229
+ else:
230
+ assert False, (
231
+ f"Tool validation failed:\n"
232
+ f"Expected ALL of these tools to be triggered: {expected_tools}\n"
233
+ f"Missing tools: {missing_tools}\n"
234
+ f"Found tools: {found_tools}\n"
235
+ f"Actually triggered: {triggered_tools}\n"
236
+ )
@@ -10,7 +10,7 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
10
10
  @pytest.mark.workflow
11
11
  @pytest.mark.workflow_with_assistant
12
12
  @pytest.mark.keycloak
13
- @pytest.mark.regression
13
+ @pytest.mark.api
14
14
  def test_workflow_with_assistant_with_keycloak_tool(
15
15
  assistant,
16
16
  workflow_with_assistant,
@@ -17,7 +17,7 @@ from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_resp
17
17
  @pytest.mark.workflow
18
18
  @pytest.mark.workflow_with_assistant
19
19
  @pytest.mark.ado
20
- @pytest.mark.regression
20
+ @pytest.mark.api
21
21
  @pytest.mark.parametrize(
22
22
  "toolkit, tool_name, prompt, expected_response",
23
23
  ado_test_plan_get_test_data,
@@ -46,7 +46,7 @@ def test_workflow_with_assistant_with_ado_test_plan_get_tools(
46
46
  @pytest.mark.workflow
47
47
  @pytest.mark.workflow_with_assistant
48
48
  @pytest.mark.ado
49
- @pytest.mark.regression
49
+ @pytest.mark.api
50
50
  def test_workflow_with_assistant_with_ado_test_plan_tools(
51
51
  ado_integration,
52
52
  assistant,
@@ -14,7 +14,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
14
14
  @pytest.mark.workflow
15
15
  @pytest.mark.workflow_with_assistant
16
16
  @pytest.mark.ado
17
- @pytest.mark.regression
17
+ @pytest.mark.api
18
18
  @pytest.mark.parametrize(
19
19
  "toolkit, tool_name, prompt, expected_response",
20
20
  ado_wiki_get_test_data,
@@ -43,7 +43,7 @@ def test_workflow_with_assistant_with_ado_wiki_get_tools(
43
43
  @pytest.mark.workflow
44
44
  @pytest.mark.workflow_with_assistant
45
45
  @pytest.mark.ado
46
- @pytest.mark.regression
46
+ @pytest.mark.api
47
47
  def test_workflow_with_assistant_with_ado_wiki_modify_tools(
48
48
  ado_integration,
49
49
  assistant,
@@ -15,7 +15,7 @@ from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_resp
15
15
  @pytest.mark.workflow
16
16
  @pytest.mark.workflow_with_assistant
17
17
  @pytest.mark.ado
18
- @pytest.mark.regression
18
+ @pytest.mark.api
19
19
  @pytest.mark.parametrize(
20
20
  "toolkit, tool_name, prompt, expected_response",
21
21
  ado_work_item_get_test_data,
@@ -44,7 +44,7 @@ def test_workflow_with_assistant_with_ado_work_item_get_tools(
44
44
  @pytest.mark.workflow
45
45
  @pytest.mark.workflow_with_assistant
46
46
  @pytest.mark.ado
47
- @pytest.mark.regression
47
+ @pytest.mark.api
48
48
  def test_workflow_with_assistant_with_ado_work_item_modify_tools(
49
49
  ado_integration,
50
50
  assistant,
@@ -6,7 +6,7 @@ from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_tes
6
6
  @pytest.mark.workflow
7
7
  @pytest.mark.workflow_with_assistant
8
8
  @pytest.mark.cloud
9
- @pytest.mark.regression
9
+ @pytest.mark.api
10
10
  @pytest.mark.parametrize(
11
11
  "toolkit,tool_name,credential_type,credentials,prompt,expected_response",
12
12
  cloud_test_data,
@@ -11,7 +11,7 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
11
11
  @pytest.mark.workflow
12
12
  @pytest.mark.workflow_with_assistant
13
13
  @pytest.mark.codebase
14
- @pytest.mark.regression
14
+ @pytest.mark.api
15
15
  @pytest.mark.parametrize(
16
16
  "toolkit,tool_name,prompt,expected_response",
17
17
  code_tools_test_data,
@@ -40,7 +40,7 @@ def test_workflow_with_codebase_tools(
40
40
 
41
41
  @pytest.mark.workflow
42
42
  @pytest.mark.workflow_with_assistant
43
- @pytest.mark.regression
43
+ @pytest.mark.api
44
44
  @pytest.mark.parametrize(
45
45
  "toolkit, tool_name, credentials, prompt,expected",
46
46
  sonar_tools_test_data,
@@ -24,7 +24,7 @@ pytestmark = pytest.mark.skipif(
24
24
  @pytest.mark.workflow
25
25
  @pytest.mark.workflow_with_assistant
26
26
  @pytest.mark.elastic
27
- @pytest.mark.regression
27
+ @pytest.mark.api
28
28
  def test_workflow_with_assistant_with_elastic_tools(
29
29
  assistant,
30
30
  workflow_with_assistant,
@@ -51,7 +51,7 @@ def test_workflow_with_assistant_with_elastic_tools(
51
51
  @pytest.mark.workflow
52
52
  @pytest.mark.workflow_with_assistant
53
53
  @pytest.mark.sql
54
- @pytest.mark.regression
54
+ @pytest.mark.api
55
55
  @pytest.mark.parametrize(
56
56
  "db_dialect",
57
57
  sql_tools_test_data,
@@ -20,8 +20,8 @@ from codemie_test_harness.tests.utils.constants import test_project_name
20
20
  @pytest.mark.integration
21
21
  @pytest.mark.default_integration
22
22
  @pytest.mark.not_for_parallel_run
23
- @pytest.mark.regression
24
- @pytest.mark.tescase("EPMCDME-6708")
23
+ @pytest.mark.api
24
+ @pytest.mark.testcase("EPMCDME-6708")
25
25
  @pytest.mark.parametrize(
26
26
  "user_integration, is_global, project_integration",
27
27
  integrations,
@@ -87,8 +87,8 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
87
87
  @pytest.mark.integration
88
88
  @pytest.mark.default_integration
89
89
  @pytest.mark.not_for_parallel_run
90
- @pytest.mark.regression
91
- @pytest.mark.tescase("EPMCDME-6708")
90
+ @pytest.mark.api
91
+ @pytest.mark.testcase("EPMCDME-6708")
92
92
  def test_assistant_in_workflow_with_global_and_project_integration(
93
93
  integration_utils,
94
94
  assistant,
@@ -137,8 +137,8 @@ def test_assistant_in_workflow_with_global_and_project_integration(
137
137
  @pytest.mark.integration
138
138
  @pytest.mark.default_integration
139
139
  @pytest.mark.not_for_parallel_run
140
- @pytest.mark.regression
141
- @pytest.mark.tescase("EPMCDME-6708")
140
+ @pytest.mark.api
141
+ @pytest.mark.testcase("EPMCDME-6708")
142
142
  def test_assistant_in_workflow_with_project_integration_only(
143
143
  integration_utils,
144
144
  assistant,
@@ -178,7 +178,7 @@ def test_assistant_in_workflow_with_project_integration_only(
178
178
  @pytest.mark.integration
179
179
  @pytest.mark.default_integration
180
180
  @pytest.mark.not_for_parallel_run
181
- @pytest.mark.regression
181
+ @pytest.mark.api
182
182
  def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
183
183
  assistant,
184
184
  assistant_utils,
@@ -227,7 +227,7 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
227
227
  @pytest.mark.integration
228
228
  @pytest.mark.default_integration
229
229
  @pytest.mark.not_for_parallel_run
230
- @pytest.mark.regression
230
+ @pytest.mark.api
231
231
  def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(
232
232
  assistant,
233
233
  assistant_utils,
@@ -22,8 +22,8 @@ ado_wiki_answer = ado_wiki_get_test_data[1][3]
22
22
  @pytest.mark.integration
23
23
  @pytest.mark.default_integration
24
24
  @pytest.mark.not_for_parallel_run
25
- @pytest.mark.regression
26
- @pytest.mark.tescase("EPMCDME-6708")
25
+ @pytest.mark.api
26
+ @pytest.mark.testcase("EPMCDME-6708")
27
27
  @pytest.mark.parametrize(
28
28
  "user_integration, is_global, project_integration",
29
29
  integrations,
@@ -91,8 +91,8 @@ def test_assistant_in_workflow_should_use_user_integration_by_default(
91
91
  @pytest.mark.integration
92
92
  @pytest.mark.default_integration
93
93
  @pytest.mark.not_for_parallel_run
94
- @pytest.mark.regression
95
- @pytest.mark.tescase("EPMCDME-6708")
94
+ @pytest.mark.api
95
+ @pytest.mark.testcase("EPMCDME-6708")
96
96
  def test_assistant_in_workflow_with_global_and_project_integration(
97
97
  integration_utils,
98
98
  assistant,
@@ -142,8 +142,8 @@ def test_assistant_in_workflow_with_global_and_project_integration(
142
142
  @pytest.mark.integration
143
143
  @pytest.mark.default_integration
144
144
  @pytest.mark.not_for_parallel_run
145
- @pytest.mark.regression
146
- @pytest.mark.tescase("EPMCDME-6708")
145
+ @pytest.mark.api
146
+ @pytest.mark.testcase("EPMCDME-6708")
147
147
  def test_assistant_in_workflow_with_project_integration_only(
148
148
  integration_utils,
149
149
  assistant,
@@ -185,7 +185,7 @@ def test_assistant_in_workflow_with_project_integration_only(
185
185
  @pytest.mark.integration
186
186
  @pytest.mark.default_integration
187
187
  @pytest.mark.not_for_parallel_run
188
- @pytest.mark.regression
188
+ @pytest.mark.api
189
189
  def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
190
190
  assistant,
191
191
  assistant_utils,
@@ -236,7 +236,7 @@ def test_assistant_in_workflow_with_global_valid_and_user_invalid_integration(
236
236
  @pytest.mark.integration
237
237
  @pytest.mark.default_integration
238
238
  @pytest.mark.not_for_parallel_run
239
- @pytest.mark.regression
239
+ @pytest.mark.api
240
240
  def test_assistant_in_workflow_with_project_valid_and_user_invalid_integration(
241
241
  assistant,
242
242
  assistant_utils,