codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.198__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (81) hide show
  1. codemie_test_harness/cli/cli.py +42 -6
  2. codemie_test_harness/cli/commands/config_cmd.py +1 -1
  3. codemie_test_harness/cli/commands/run_cmd.py +24 -1
  4. codemie_test_harness/cli/constants.py +1 -0
  5. codemie_test_harness/cli/runner.py +17 -3
  6. codemie_test_harness/cli/utils.py +12 -2
  7. codemie_test_harness/pytest.ini +3 -0
  8. codemie_test_harness/tests/assistant/test_assistants.py +17 -1
  9. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +47 -6
  10. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +0 -4
  11. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +0 -4
  12. codemie_test_harness/tests/conftest.py +19 -3
  13. codemie_test_harness/tests/enums/environment.py +3 -3
  14. codemie_test_harness/tests/enums/integrations.py +1 -0
  15. codemie_test_harness/tests/enums/model_types.py +1 -0
  16. codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -15
  17. codemie_test_harness/tests/integrations/project/test_project_integrations.py +42 -0
  18. codemie_test_harness/tests/integrations/user/test_default_integrations.py +41 -15
  19. codemie_test_harness/tests/llm/assistants/test_llm.py +45 -2
  20. codemie_test_harness/tests/test_data/assistant_test_data.py +171 -171
  21. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +2 -0
  22. codemie_test_harness/tests/test_data/data_management_tools_test_data.py +18 -0
  23. codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +18 -1
  24. codemie_test_harness/tests/test_data/direct_tools/report_portal_tools_test_data.py +189 -197
  25. codemie_test_harness/tests/test_data/integrations_test_data.py +163 -2
  26. codemie_test_harness/tests/test_data/llm_test_data.py +1 -0
  27. codemie_test_harness/tests/test_data/open_api_tools_test_data.py +22 -1
  28. codemie_test_harness/tests/test_data/report_portal_tools_test_data.py +89 -112
  29. codemie_test_harness/tests/test_data/research_tools_test_data.py +29 -7
  30. codemie_test_harness/tests/utils/assistant_utils.py +22 -12
  31. codemie_test_harness/tests/utils/credentials_manager.py +66 -8
  32. codemie_test_harness/tests/utils/workflow_utils.py +91 -0
  33. codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +7 -0
  34. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +50 -1
  35. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +39 -1
  36. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +34 -1
  37. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +6 -0
  38. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +11 -0
  39. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +72 -1
  40. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +31 -0
  41. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +31 -0
  42. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +26 -0
  43. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +63 -1
  44. codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +82 -7
  45. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +23 -4
  46. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +12 -0
  47. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +6 -0
  48. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +29 -2
  49. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +14 -1
  50. codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +7 -0
  51. codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +7 -0
  52. codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +14 -1
  53. codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +6 -0
  54. codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +6 -0
  55. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +12 -9
  56. codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +8 -1
  57. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +28 -1
  58. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +24 -1
  59. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +20 -1
  60. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +13 -4
  61. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +16 -1
  62. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +73 -1
  63. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +34 -1
  64. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +34 -1
  65. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +34 -1
  66. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +101 -49
  67. codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +42 -3
  68. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +27 -5
  69. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +13 -0
  70. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +10 -1
  71. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +20 -0
  72. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +14 -1
  73. codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +10 -1
  74. codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +10 -1
  75. codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +9 -0
  76. codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +10 -1
  77. codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +9 -1
  78. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/METADATA +134 -3
  79. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/RECORD +81 -81
  80. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/WHEEL +0 -0
  81. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/entry_points.txt +0 -0
@@ -2,11 +2,12 @@ import os
2
2
 
3
3
  import pytest
4
4
 
5
- from codemie_test_harness.tests.enums.tools import Toolkit
5
+ from codemie_test_harness.tests.enums.tools import Toolkit, CliMcpServer, PluginTool
6
6
  from codemie_test_harness.tests.test_data.mcp_server_test_data import (
7
7
  cli_mcp_server_with_plugin_test_data,
8
8
  filesystem_mcp_server_with_plugin_test_data,
9
9
  )
10
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
10
11
  from codemie_test_harness.tests.utils.constants import TESTS_PATH
11
12
 
12
13
 
@@ -36,6 +37,11 @@ def test_workflow_with_assistant_with_plugin_and_cli_mcp_server(
36
37
  workflow_with_assistant.id, assistant.name, user_input=prompt
37
38
  )
38
39
 
40
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
41
+ workflow_with_assistant
42
+ )
43
+ assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
44
+
39
45
  similarity_check.check_similarity(response, expected_response)
40
46
 
41
47
 
@@ -69,6 +75,13 @@ def test_workflow_with_assistant_with_plugin_and_filesystem_mcp_server(
69
75
  workflow_with_assistant.id, assistant.name, user_input=prompt
70
76
  )
71
77
 
78
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
79
+ workflow_with_assistant
80
+ )
81
+ # Match the pattern from assistant test - conditionally assert tools
82
+ if tool_name != PluginTool.READ_FILE:
83
+ assert_tool_triggered(tool_name, triggered_tools)
84
+
72
85
  similarity_check.check_similarity(response, expected_response)
73
86
  finally:
74
87
  file_to_remove = f"{str(TESTS_PATH / 'sdk_tests')}.properties"
@@ -4,6 +4,7 @@ from codemie_test_harness.tests.enums.tools import Toolkit
4
4
  from codemie_test_harness.tests.test_data.project_management_test_data import (
5
5
  pm_tools_test_data,
6
6
  )
7
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
7
8
  from codemie_test_harness.tests.utils.constants import (
8
9
  project_management_integrations,
9
10
  )
@@ -45,4 +46,10 @@ def test_workflow_with_assistant_with_project_management_tools(
45
46
  response = workflow_utils.execute_workflow(
46
47
  workflow_with_assistant.id, assistant.name
47
48
  )
49
+
50
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
51
+ workflow_with_assistant
52
+ )
53
+ assert_tool_triggered(tool_name, triggered_tools)
54
+
48
55
  similarity_check.check_similarity(response, expected_response)
@@ -3,6 +3,7 @@ import pytest
3
3
  from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
4
4
  rp_test_data,
5
5
  )
6
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
6
7
 
7
8
 
8
9
  @pytest.mark.workflow
@@ -35,4 +36,10 @@ def test_workflow_with_assistant_with_report_portal_tools(
35
36
  response = workflow_utils.execute_workflow(
36
37
  workflow_with_assistant.id, assistant.name
37
38
  )
39
+
40
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
41
+ workflow_with_assistant
42
+ )
43
+ assert_tool_triggered(tool_name, triggered_tools)
44
+
38
45
  similarity_check.check_similarity(response, expected_response)
@@ -6,7 +6,10 @@ from codemie_test_harness.tests.test_data.research_tools_test_data import (
6
6
  search_tools_test_data,
7
7
  interactions_tools_test_data,
8
8
  )
9
- from codemie_test_harness.tests.utils.base_utils import percent_of_relevant_titles
9
+ from codemie_test_harness.tests.utils.base_utils import (
10
+ percent_of_relevant_titles,
11
+ assert_tool_triggered,
12
+ )
10
13
 
11
14
 
12
15
  @pytest.mark.workflow
@@ -32,6 +35,11 @@ def test_workflow_with_search_tools(
32
35
  workflow_with_assistant.id, assistant.name, user_input=prompt
33
36
  )
34
37
 
38
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
39
+ workflow_with_assistant
40
+ )
41
+ assert_tool_triggered(tool_name, triggered_tools)
42
+
35
43
  percent = percent_of_relevant_titles(response)
36
44
 
37
45
  assert_that(
@@ -65,4 +73,9 @@ def test_workflow_with_interaction_tools(
65
73
  workflow_with_assistant.id, assistant.name
66
74
  )
67
75
 
76
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
77
+ workflow_with_assistant
78
+ )
79
+ assert_tool_triggered(tool_name, triggered_tools)
80
+
68
81
  similarity_check.check_similarity(response, expected_response)
@@ -5,6 +5,7 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
5
5
  PROMPT,
6
6
  EXPECTED_RESPONSE,
7
7
  )
8
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
8
9
 
9
10
 
10
11
  @pytest.mark.workflow
@@ -30,4 +31,9 @@ def test_workflow_with_assistant_with_servicenow_tools(
30
31
  workflow_with_assistant.id, assistant.name
31
32
  )
32
33
 
34
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
35
+ workflow_with_assistant
36
+ )
37
+ assert_tool_triggered(ServiceNowTool.SERVICE_NOW, triggered_tools)
38
+
33
39
  similarity_check.check_similarity(response, EXPECTED_RESPONSE, 80)
@@ -6,6 +6,7 @@ from codemie_test_harness.tests.enums.tools import (
6
6
  from codemie_test_harness.tests.test_data.vcs_tools_test_data import (
7
7
  vcs_tools_test_data,
8
8
  )
9
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
9
10
  from codemie_test_harness.tests.utils.constants import vcs_integrations
10
11
 
11
12
 
@@ -36,4 +37,9 @@ def test_create_workflow_with_assistant_vcs_tool(
36
37
  workflow_with_assistant.id, assistant.name
37
38
  )
38
39
 
40
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
41
+ workflow_with_assistant
42
+ )
43
+ assert_tool_triggered(tool_name, triggered_tools)
44
+
39
45
  similarity_check.check_similarity(response, expected_response)
@@ -26,7 +26,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
26
26
  ],
27
27
  )
28
28
  def test_workflow_with_cloud_tools_direct(
29
- cloud_integration,
29
+ integration,
30
30
  workflow_utils,
31
31
  similarity_check,
32
32
  workflow_with_tool,
@@ -39,10 +39,10 @@ def test_workflow_with_cloud_tools_direct(
39
39
  ):
40
40
  tool_and_state_name = get_random_name()
41
41
 
42
- integration = cloud_integration(credential_type, credentials)
42
+ cloud_integration = integration(credential_type, credentials)
43
43
 
44
44
  _workflow = workflow_with_tool(
45
- tool_and_state_name, tool_name, integration=integration
45
+ tool_and_state_name, tool_name, integration=cloud_integration
46
46
  )
47
47
  response = workflow_utils.execute_workflow(
48
48
  _workflow.id, tool_and_state_name, user_input=json.dumps(prompt)
@@ -66,7 +66,7 @@ def test_workflow_with_cloud_tools_direct(
66
66
  ],
67
67
  )
68
68
  def test_workflow_with_cloud_tools_with_hardcoded_args(
69
- cloud_integration,
69
+ integration,
70
70
  workflow_utils,
71
71
  similarity_check,
72
72
  workflow_with_tool,
@@ -79,10 +79,10 @@ def test_workflow_with_cloud_tools_with_hardcoded_args(
79
79
  ):
80
80
  tool_and_state_name = get_random_name()
81
81
 
82
- integration = cloud_integration(credential_type, credentials)
82
+ cloud_integration = integration(credential_type, credentials)
83
83
 
84
84
  _workflow = workflow_with_tool(
85
- tool_and_state_name, tool_name, integration=integration, tool_args=prompt
85
+ tool_and_state_name, tool_name, integration=cloud_integration, tool_args=prompt
86
86
  )
87
87
  response = workflow_utils.execute_workflow(_workflow.id, tool_and_state_name)
88
88
 
@@ -104,7 +104,7 @@ def test_workflow_with_cloud_tools_with_hardcoded_args(
104
104
  ],
105
105
  )
106
106
  def test_workflow_with_cloud_tools_with_overriding_args(
107
- cloud_integration,
107
+ integration,
108
108
  workflow_utils,
109
109
  similarity_check,
110
110
  workflow_with_tool,
@@ -117,13 +117,16 @@ def test_workflow_with_cloud_tools_with_overriding_args(
117
117
  ):
118
118
  tool_and_state_name = get_random_name()
119
119
 
120
- integration = cloud_integration(credential_type, credentials)
120
+ cloud_integration = integration(credential_type, credentials)
121
121
 
122
122
  args_copy = copy.deepcopy(prompt)
123
123
  args_copy = {key: random.randint(1, 10) for key in args_copy}
124
124
 
125
125
  _workflow = workflow_with_tool(
126
- tool_and_state_name, tool_name, integration=integration, tool_args=args_copy
126
+ tool_and_state_name,
127
+ tool_name,
128
+ integration=cloud_integration,
129
+ tool_args=args_copy,
127
130
  )
128
131
  response = workflow_utils.execute_workflow(
129
132
  _workflow.id, tool_and_state_name, user_input=json.dumps(prompt)
@@ -5,7 +5,10 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
5
5
  KEYCLOAK_TOOL_PROMPT,
6
6
  KEYCLOAK_TOOL_RESPONSE,
7
7
  )
8
- from codemie_test_harness.tests.utils.base_utils import get_random_name
8
+ from codemie_test_harness.tests.utils.base_utils import (
9
+ get_random_name,
10
+ assert_tool_triggered,
11
+ )
9
12
 
10
13
 
11
14
  @pytest.mark.workflow
@@ -29,4 +32,8 @@ def test_workflow_with_virtual_assistant_with_keycloak_tool(
29
32
  response = workflow_utils.execute_workflow(
30
33
  test_workflow.id, assistant_and_state_name
31
34
  )
35
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
36
+ test_workflow
37
+ )
38
+ assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
32
39
  similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
@@ -9,7 +9,10 @@ from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
9
9
  ADO_TEST_PLAN_DELETE_SUITE,
10
10
  ADO_TEST_PLAN_DELETE_PLAN,
11
11
  )
12
- from codemie_test_harness.tests.utils.base_utils import get_random_name
12
+ from codemie_test_harness.tests.utils.base_utils import (
13
+ get_random_name,
14
+ assert_tool_triggered,
15
+ )
13
16
  from codemie_test_harness.tests.utils.constants import ID_PATTERN
14
17
  from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
15
18
 
@@ -44,6 +47,10 @@ def test_workflow_with_ado_test_plan_get_tools(
44
47
  response = workflow_utils.execute_workflow(
45
48
  test_workflow.id, assistant_and_state_name, prompt
46
49
  )
50
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
51
+ test_workflow
52
+ )
53
+ assert_tool_triggered(tool_name, triggered_tools)
47
54
  similarity_check.check_similarity(response, expected_response)
48
55
 
49
56
 
@@ -74,6 +81,10 @@ def test_workflow_with_ado_test_plan_tools(
74
81
  create_response = workflow_utils.execute_workflow(
75
82
  create_test_suite_workflow.id, assistant_and_state_name
76
83
  )
84
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
85
+ create_test_suite_workflow
86
+ )
87
+ assert_tool_triggered(AzureDevOpsTestPlanTool.CREATE_TEST_PLAN, triggered_tools)
77
88
 
78
89
  # Extract the test plan ID from the response
79
90
  test_plan_id = extract_id_from_ado_response(create_response, ID_PATTERN)
@@ -98,6 +109,10 @@ def test_workflow_with_ado_test_plan_tools(
98
109
  create_suite_response = workflow_utils.execute_workflow(
99
110
  create_test_suite_workflow.id, assistant_and_state_name, create_suite_prompt
100
111
  )
112
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
113
+ create_test_suite_workflow
114
+ )
115
+ assert_tool_triggered(AzureDevOpsTestPlanTool.CREATE_TEST_SUITE, triggered_tools)
101
116
 
102
117
  # Extract the test suite ID from the response
103
118
  test_suite_id = extract_id_from_ado_response(create_suite_response, ID_PATTERN)
@@ -119,6 +134,10 @@ def test_workflow_with_ado_test_plan_tools(
119
134
  add_case_response = workflow_utils.execute_workflow(
120
135
  add_test_case_workflow.id, assistant_and_state_name, add_case_prompt
121
136
  )
137
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
138
+ add_test_case_workflow
139
+ )
140
+ assert_tool_triggered(AzureDevOpsTestPlanTool.ADD_TEST_CASE, triggered_tools)
122
141
  add_case_expected = ADO_TEST_PLAN_ADD_TEST_CASE["expected_llm_answer"].format(
123
142
  test_suite_id, test_plan_id
124
143
  )
@@ -137,6 +156,10 @@ def test_workflow_with_ado_test_plan_tools(
137
156
  delete_suite_response = workflow_utils.execute_workflow(
138
157
  delete_test_suite_workflow.id, assistant_and_state_name, delete_suite_prompt
139
158
  )
159
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
160
+ delete_test_suite_workflow
161
+ )
162
+ assert_tool_triggered(AzureDevOpsTestPlanTool.DELETE_TEST_SUITE, triggered_tools)
140
163
  delete_suite_expected = ADO_TEST_PLAN_DELETE_SUITE["expected_llm_answer"].format(
141
164
  test_suite_id, test_plan_id
142
165
  )
@@ -155,6 +178,10 @@ def test_workflow_with_ado_test_plan_tools(
155
178
  delete_plan_response = workflow_utils.execute_workflow(
156
179
  delete_test_plan_workflow.id, assistant_and_state_name, delete_plan_prompt
157
180
  )
181
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
182
+ delete_test_plan_workflow
183
+ )
184
+ assert_tool_triggered(AzureDevOpsTestPlanTool.DELETE_TEST_PLAN, triggered_tools)
158
185
  delete_plan_expected = ADO_TEST_PLAN_DELETE_PLAN["expected_llm_answer"].format(
159
186
  test_plan_id
160
187
  )
@@ -8,7 +8,10 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
8
8
  ADO_WIKI_MODIFY_PAGE,
9
9
  ADO_WIKI_DELETE_PAGE,
10
10
  )
11
- from codemie_test_harness.tests.utils.base_utils import get_random_name
11
+ from codemie_test_harness.tests.utils.base_utils import (
12
+ get_random_name,
13
+ assert_tool_triggered,
14
+ )
12
15
 
13
16
 
14
17
  @pytest.mark.workflow
@@ -41,6 +44,10 @@ def test_workflow_with_ado_wiki_get_tools(
41
44
  response = workflow_utils.execute_workflow(
42
45
  test_workflow.id, assistant_and_state_name, prompt
43
46
  )
47
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
48
+ test_workflow
49
+ )
50
+ assert_tool_triggered(tool_name, triggered_tools)
44
51
  similarity_check.check_similarity(response, expected_response)
45
52
 
46
53
 
@@ -68,6 +75,10 @@ def test_workflow_with_ado_wiki_modify_tools(
68
75
  create_response = workflow_utils.execute_workflow(
69
76
  create_page_workflow.id, assistant_and_state_name, create_prompt
70
77
  )
78
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
79
+ create_page_workflow
80
+ )
81
+ assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, triggered_tools)
71
82
  similarity_check.check_similarity(create_response, create_expected)
72
83
 
73
84
  # 2. Rename the page
@@ -86,6 +97,10 @@ def test_workflow_with_ado_wiki_modify_tools(
86
97
  rename_response = workflow_utils.execute_workflow(
87
98
  rename_page_workflow.id, assistant_and_state_name, rename_prompt
88
99
  )
100
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
101
+ rename_page_workflow
102
+ )
103
+ assert_tool_triggered(AzureDevOpsWikiTool.RENAME_WIKI_PAGE, triggered_tools)
89
104
  similarity_check.check_similarity(rename_response, rename_expected)
90
105
 
91
106
  # 3. Modify the page
@@ -104,6 +119,10 @@ def test_workflow_with_ado_wiki_modify_tools(
104
119
  modify_response = workflow_utils.execute_workflow(
105
120
  modify_page_workflow.id, assistant_and_state_name, modify_prompt
106
121
  )
122
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
123
+ modify_page_workflow
124
+ )
125
+ assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, triggered_tools)
107
126
  similarity_check.check_similarity(modify_response, modify_expected)
108
127
 
109
128
  # 4. Delete the page
@@ -122,4 +141,8 @@ def test_workflow_with_ado_wiki_modify_tools(
122
141
  delete_response = workflow_utils.execute_workflow(
123
142
  delete_page_workflow.id, assistant_and_state_name, delete_prompt
124
143
  )
144
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
145
+ delete_page_workflow
146
+ )
147
+ assert_tool_triggered(AzureDevOpsWikiTool.DELETE_WIKI_PAGE_BY_PATH, triggered_tools)
125
148
  similarity_check.check_similarity(delete_response, delete_expected)
@@ -7,7 +7,10 @@ from codemie_test_harness.tests.test_data.ado_work_item_tools_test_data import (
7
7
  ADO_WORK_ITEM_UPDATE,
8
8
  ADO_WORK_ITEM_LINK,
9
9
  )
10
- from codemie_test_harness.tests.utils.base_utils import get_random_name
10
+ from codemie_test_harness.tests.utils.base_utils import (
11
+ get_random_name,
12
+ assert_tool_triggered,
13
+ )
11
14
  from codemie_test_harness.tests.utils.constants import WORK_ITEM_ID_PATTERN
12
15
  from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
13
16
 
@@ -42,6 +45,10 @@ def test_workflow_with_ado_work_item_get_tools(
42
45
  response = workflow_utils.execute_workflow(
43
46
  test_workflow.id, assistant_and_state_name, prompt
44
47
  )
48
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
49
+ test_workflow
50
+ )
51
+ assert_tool_triggered(tool_name, triggered_tools)
45
52
  similarity_check.check_similarity(response, expected_response)
46
53
 
47
54
 
@@ -73,6 +80,10 @@ def test_workflow_with_ado_work_item_modify_tools(
73
80
  create_response = workflow_utils.execute_workflow(
74
81
  create_item_workflow.id, assistant_and_state_name, create_prompt
75
82
  )
83
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
84
+ create_item_workflow
85
+ )
86
+ assert_tool_triggered(AzureDevOpsWorkItemTool.CREATE_WORK_ITEM, triggered_tools)
76
87
  work_item_id = extract_id_from_ado_response(create_response, WORK_ITEM_ID_PATTERN)
77
88
  similarity_check.check_similarity(create_response, create_expected)
78
89
 
@@ -94,6 +105,10 @@ def test_workflow_with_ado_work_item_modify_tools(
94
105
  update_response = workflow_utils.execute_workflow(
95
106
  update_item_workflow.id, assistant_and_state_name, update_prompt
96
107
  )
108
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
109
+ update_item_workflow
110
+ )
111
+ assert_tool_triggered(AzureDevOpsWorkItemTool.UPDATE_WORK_ITEM, triggered_tools)
97
112
  similarity_check.check_similarity(update_response, update_expected)
98
113
 
99
114
  # 3. Link the item
@@ -111,4 +126,8 @@ def test_workflow_with_ado_work_item_modify_tools(
111
126
  link_response = workflow_utils.execute_workflow(
112
127
  link_item_workflow.id, assistant_and_state_name, link_prompt
113
128
  )
129
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
130
+ link_item_workflow
131
+ )
132
+ assert_tool_triggered(AzureDevOpsWorkItemTool.LINK_WORK_ITEMS, triggered_tools)
114
133
  similarity_check.check_similarity(link_response, link_expected)
@@ -1,7 +1,10 @@
1
1
  import pytest
2
2
 
3
3
  from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
4
- from codemie_test_harness.tests.utils.base_utils import get_random_name
4
+ from codemie_test_harness.tests.utils.base_utils import (
5
+ get_random_name,
6
+ assert_tool_triggered,
7
+ )
5
8
 
6
9
 
7
10
  @pytest.mark.workflow
@@ -17,7 +20,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
17
20
  )
18
21
  def test_workflow_with_cloud_tools(
19
22
  workflow_with_virtual_assistant,
20
- cloud_integration,
23
+ integration,
21
24
  workflow_utils,
22
25
  similarity_check,
23
26
  toolkit,
@@ -29,15 +32,21 @@ def test_workflow_with_cloud_tools(
29
32
  ):
30
33
  assistant_and_state_name = get_random_name()
31
34
 
32
- integration = cloud_integration(credential_type, credentials)
35
+ cloud_integration = integration(credential_type, credentials)
33
36
 
34
37
  test_workflow = workflow_with_virtual_assistant(
35
38
  assistant_and_state_name,
36
39
  tool_name,
37
- integration=integration,
40
+ integration=cloud_integration,
38
41
  task=prompt,
39
42
  )
40
43
  response = workflow_utils.execute_workflow(
41
44
  test_workflow.id, assistant_and_state_name
42
45
  )
46
+
47
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
48
+ test_workflow
49
+ )
50
+ assert_tool_triggered(tool_name, triggered_tools)
51
+
43
52
  similarity_check.check_similarity(response, expected_response)
@@ -7,7 +7,10 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
7
7
  sonar_tools_test_data,
8
8
  )
9
9
  from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
10
- from codemie_test_harness.tests.utils.base_utils import get_random_name
10
+ from codemie_test_harness.tests.utils.base_utils import (
11
+ get_random_name,
12
+ assert_tool_triggered,
13
+ )
11
14
 
12
15
 
13
16
  @pytest.fixture(scope="module")
@@ -60,6 +63,12 @@ def test_workflow_with_codebase_tools(
60
63
  response = workflow_utils.execute_workflow(
61
64
  test_workflow.id, assistant_and_state_name
62
65
  )
66
+
67
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
68
+ test_workflow
69
+ )
70
+ assert_tool_triggered(tool_name, triggered_tools)
71
+
63
72
  similarity_check.check_similarity(response, expected_response)
64
73
 
65
74
 
@@ -98,4 +107,10 @@ def test_workflow_with_sonar_tools(
98
107
  response = workflow_utils.execute_workflow(
99
108
  test_workflow.id, assistant_and_state_name
100
109
  )
110
+
111
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
112
+ test_workflow
113
+ )
114
+ assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
115
+
101
116
  similarity_check.check_similarity(response, expected_response)
@@ -1,6 +1,7 @@
1
1
  import pytest
2
2
  from codemie_sdk.models.integration import CredentialTypes
3
3
 
4
+ from codemie_test_harness.tests.enums.integrations import DataBaseDialect
4
5
  from codemie_test_harness.tests.enums.tools import DataManagementTool
5
6
  from codemie_test_harness.tests.test_data.data_management_tools_test_data import (
6
7
  ELASTIC_TOOL_TASK,
@@ -11,10 +12,15 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
11
12
  SQL_TOOL_INSERT_TABLE_TASK,
12
13
  SQL_TOOL_QUERY_TABLE_TASK,
13
14
  RESPONSE_FOR_SQL,
15
+ INFLUXDB_QUERY_MEASUREMENT_TASK,
16
+ RESPONSE_FOR_INFLUXDB,
17
+ )
18
+ from codemie_test_harness.tests.utils.base_utils import (
19
+ get_random_name,
20
+ assert_tool_triggered,
14
21
  )
15
22
  from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
16
23
  from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
17
- from codemie_test_harness.tests.utils.base_utils import get_random_name
18
24
 
19
25
  pytestmark = pytest.mark.skipif(
20
26
  EnvironmentResolver.is_localhost(),
@@ -49,6 +55,10 @@ def test_workflow_with_elastic_tools(
49
55
  response = workflow_utils.execute_workflow(
50
56
  test_workflow.id, assistant_and_state_name
51
57
  )
58
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
59
+ test_workflow
60
+ )
61
+ assert_tool_triggered(DataManagementTool.ELASTIC, triggered_tools)
52
62
  similarity_check.check_similarity(response, RESPONSE_FOR_ELASTIC)
53
63
 
54
64
 
@@ -81,17 +91,79 @@ def test_workflow_with_sql_tools(
81
91
  integration=integration,
82
92
  task=f"DB dialect is {db_dialect.value}. Run SQL tool and execute SQL queries to perform user requests.",
83
93
  )
94
+
95
+ # Step 1: Create table
84
96
  workflow_utils.execute_workflow(
85
97
  test_workflow.id, assistant_and_state_name, SQL_TOOL_CREATE_TABLE_TASK
86
98
  )
99
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
100
+ test_workflow
101
+ )
102
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
103
+
104
+ # Step 2: Insert data
87
105
  workflow_utils.execute_workflow(
88
106
  test_workflow.id, assistant_and_state_name, SQL_TOOL_INSERT_TABLE_TASK
89
107
  )
108
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
109
+ test_workflow
110
+ )
111
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
112
+
113
+ # Step 3: Query data
90
114
  response = workflow_utils.execute_workflow(
91
115
  test_workflow.id, assistant_and_state_name, SQL_TOOL_QUERY_TABLE_TASK
92
116
  )
117
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
118
+ test_workflow
119
+ )
120
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
121
+
122
+ # Step 4: Delete table
93
123
  workflow_utils.execute_workflow(
94
124
  test_workflow.id, assistant_and_state_name, SQL_TOOL_DELETE_TABLE_TASK
95
125
  )
126
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
127
+ test_workflow
128
+ )
129
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
96
130
 
97
131
  similarity_check.check_similarity(response, RESPONSE_FOR_SQL)
132
+
133
+
134
+ @pytest.mark.workflow
135
+ @pytest.mark.virtual_workflow
136
+ @pytest.mark.influx
137
+ @pytest.mark.api
138
+ @pytest.mark.testcase("EPMCDME-6431")
139
+ @pytest.mark.skipif(
140
+ not EnvironmentResolver.is_sandbox(),
141
+ reason="InfluxDB is only available in sandbox environments",
142
+ )
143
+ def test_workflow_with_influxdb_tools(
144
+ workflow_with_virtual_assistant,
145
+ workflow_utils,
146
+ integration_utils,
147
+ similarity_check,
148
+ ):
149
+ """Test workflow execution with InfluxDB tools."""
150
+
151
+ assistant_and_state_name = get_random_name()
152
+ credential_values = CredentialsManager.sql_credentials(DataBaseDialect.INFLUX)
153
+ integration = integration_utils.create_integration(
154
+ CredentialTypes.SQL, credential_values
155
+ )
156
+
157
+ test_workflow = workflow_with_virtual_assistant(
158
+ assistant_and_state_name,
159
+ DataManagementTool.SQL,
160
+ integration=integration,
161
+ task=f"DB dialect is {DataBaseDialect.INFLUX.value}. Run SQL tool and execute InfluxQL queries to perform user requests.",
162
+ )
163
+
164
+ # Query the measurement
165
+ response = workflow_utils.execute_workflow(
166
+ test_workflow.id, assistant_and_state_name, INFLUXDB_QUERY_MEASUREMENT_TASK
167
+ )
168
+
169
+ similarity_check.check_similarity(response, RESPONSE_FOR_INFLUXDB)