codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.198__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of codemie-test-harness might be problematic. Click here for more details.

Files changed (81) hide show
  1. codemie_test_harness/cli/cli.py +42 -6
  2. codemie_test_harness/cli/commands/config_cmd.py +1 -1
  3. codemie_test_harness/cli/commands/run_cmd.py +24 -1
  4. codemie_test_harness/cli/constants.py +1 -0
  5. codemie_test_harness/cli/runner.py +17 -3
  6. codemie_test_harness/cli/utils.py +12 -2
  7. codemie_test_harness/pytest.ini +3 -0
  8. codemie_test_harness/tests/assistant/test_assistants.py +17 -1
  9. codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +47 -6
  10. codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +0 -4
  11. codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +0 -4
  12. codemie_test_harness/tests/conftest.py +19 -3
  13. codemie_test_harness/tests/enums/environment.py +3 -3
  14. codemie_test_harness/tests/enums/integrations.py +1 -0
  15. codemie_test_harness/tests/enums/model_types.py +1 -0
  16. codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -15
  17. codemie_test_harness/tests/integrations/project/test_project_integrations.py +42 -0
  18. codemie_test_harness/tests/integrations/user/test_default_integrations.py +41 -15
  19. codemie_test_harness/tests/llm/assistants/test_llm.py +45 -2
  20. codemie_test_harness/tests/test_data/assistant_test_data.py +171 -171
  21. codemie_test_harness/tests/test_data/codebase_tools_test_data.py +2 -0
  22. codemie_test_harness/tests/test_data/data_management_tools_test_data.py +18 -0
  23. codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +18 -1
  24. codemie_test_harness/tests/test_data/direct_tools/report_portal_tools_test_data.py +189 -197
  25. codemie_test_harness/tests/test_data/integrations_test_data.py +163 -2
  26. codemie_test_harness/tests/test_data/llm_test_data.py +1 -0
  27. codemie_test_harness/tests/test_data/open_api_tools_test_data.py +22 -1
  28. codemie_test_harness/tests/test_data/report_portal_tools_test_data.py +89 -112
  29. codemie_test_harness/tests/test_data/research_tools_test_data.py +29 -7
  30. codemie_test_harness/tests/utils/assistant_utils.py +22 -12
  31. codemie_test_harness/tests/utils/credentials_manager.py +66 -8
  32. codemie_test_harness/tests/utils/workflow_utils.py +91 -0
  33. codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +7 -0
  34. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +50 -1
  35. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +39 -1
  36. codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +34 -1
  37. codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +6 -0
  38. codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +11 -0
  39. codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +72 -1
  40. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +31 -0
  41. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +31 -0
  42. codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +26 -0
  43. codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +63 -1
  44. codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +82 -7
  45. codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +23 -4
  46. codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +12 -0
  47. codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +6 -0
  48. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +29 -2
  49. codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +14 -1
  50. codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +7 -0
  51. codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +7 -0
  52. codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +14 -1
  53. codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +6 -0
  54. codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +6 -0
  55. codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +12 -9
  56. codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +8 -1
  57. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +28 -1
  58. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +24 -1
  59. codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +20 -1
  60. codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +13 -4
  61. codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +16 -1
  62. codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +73 -1
  63. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +34 -1
  64. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +34 -1
  65. codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +34 -1
  66. codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +101 -49
  67. codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +42 -3
  68. codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +27 -5
  69. codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +13 -0
  70. codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +10 -1
  71. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +20 -0
  72. codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +14 -1
  73. codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +10 -1
  74. codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +10 -1
  75. codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +9 -0
  76. codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +10 -1
  77. codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +9 -1
  78. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/METADATA +134 -3
  79. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/RECORD +81 -81
  80. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/WHEEL +0 -0
  81. {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/entry_points.txt +0 -0
@@ -9,9 +9,10 @@ import os
9
9
  import re
10
10
  from typing import Optional, List, Dict, Any
11
11
 
12
+ from codemie_sdk.models.integration import CredentialValues
13
+
12
14
  from codemie_test_harness.tests.utils.aws_parameters_store import AwsParameterStore
13
15
  from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
14
- from codemie_sdk.models.integration import CredentialValues
15
16
 
16
17
  # Constants
17
18
  INTEGRATIONS_PARAMETER_PATH = "/codemie/autotests/integrations/"
@@ -27,7 +28,7 @@ class CredentialsManager:
27
28
  3. Default values (final fallback)
28
29
  """
29
30
 
30
- AUTO_GENERATED = "AUTO_GENERATED"
31
+ AUTO_GENERATED = "AutoGenerated"
31
32
 
32
33
  # Configuration mapping: env_var -> AWS parameter details
33
34
  CREDENTIAL_MAPPINGS = {
@@ -541,6 +542,37 @@ class CredentialsManager:
541
542
  "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
542
543
  "aws_path": "sandbox.mssql.password",
543
544
  },
545
+ # Sandbox InfluxDB (only available in sandbox)
546
+ "SANDBOX_INFLUX_DIALECT": {
547
+ "env": "INFLUX_DIALECT",
548
+ "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
549
+ "aws_path": "sandbox.influxdb.dialect",
550
+ },
551
+ "SANDBOX_INFLUX_URL": {
552
+ "env": "INFLUX_URL",
553
+ "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
554
+ "aws_path": "sandbox.influxdb.url",
555
+ },
556
+ "SANDBOX_INFLUX_PORT": {
557
+ "env": "INFLUX_PORT",
558
+ "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
559
+ "aws_path": "sandbox.influxdb.port",
560
+ },
561
+ "SANDBOX_INFLUX_TOKEN": {
562
+ "env": "INFLUX_TOKEN",
563
+ "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
564
+ "aws_path": "sandbox.influxdb.token",
565
+ },
566
+ "SANDBOX_INFLUX_ORG_NAME": {
567
+ "env": "INFLUX_ORG_NAME",
568
+ "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
569
+ "aws_path": "sandbox.influxdb.org_name",
570
+ },
571
+ "SANDBOX_INFLUX_BUCKET": {
572
+ "env": "INFLUX_BUCKET",
573
+ "aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
574
+ "aws_path": "sandbox.influxdb.bucket",
575
+ },
544
576
  }
545
577
 
546
578
  # Cache for AWS Parameter Store data to avoid repeated calls
@@ -1126,6 +1158,28 @@ class CredentialsManager:
1126
1158
  CredentialValues(key="password", value=password),
1127
1159
  ]
1128
1160
 
1161
+ @staticmethod
1162
+ def influx_credentials() -> List[CredentialValues]:
1163
+ """Create INFLUX credentials using configuration-driven approach.
1164
+
1165
+ INFLUX DB is only available in sandbox environment.
1166
+ """
1167
+ dialect = CredentialsManager.get_parameter("SANDBOX_INFLUX_DIALECT")
1168
+ url = CredentialsManager.get_parameter("SANDBOX_INFLUX_URL")
1169
+ port = CredentialsManager.get_parameter("SANDBOX_INFLUX_PORT")
1170
+ token = CredentialsManager.get_parameter("SANDBOX_INFLUX_TOKEN")
1171
+ org = CredentialsManager.get_parameter("SANDBOX_INFLUX_ORG_NAME")
1172
+ bucket = CredentialsManager.get_parameter("SANDBOX_INFLUX_BUCKET")
1173
+
1174
+ return [
1175
+ CredentialValues(key="dialect", value=dialect),
1176
+ CredentialValues(key="url", value=url),
1177
+ CredentialValues(key="port", value=port),
1178
+ CredentialValues(key="token", value=token),
1179
+ CredentialValues(key="org", value=org),
1180
+ CredentialValues(key="bucket", value=bucket),
1181
+ ]
1182
+
1129
1183
  @staticmethod
1130
1184
  def sql_credentials(db_dialect: str) -> list[CredentialValues] | None:
1131
1185
  """Create SQL credentials using environment-aware configuration-driven approach.
@@ -1136,12 +1190,16 @@ class CredentialsManager:
1136
1190
  Returns:
1137
1191
  List of credential values for the specified database dialect
1138
1192
  """
1139
- if db_dialect.lower() == "mysql":
1140
- return CredentialsManager.mysql_credentials()
1141
- elif db_dialect.lower() in ["postgres", "postgresql"]:
1142
- return CredentialsManager.postgres_credentials()
1143
- elif db_dialect.lower() in ["mssql", "sqlserver"]:
1144
- return CredentialsManager.mssql_credentials()
1193
+ sql_credentials_map = {
1194
+ "mysql": CredentialsManager.mysql_credentials(),
1195
+ "postgres": CredentialsManager.postgres_credentials(),
1196
+ "postgresql": CredentialsManager.postgres_credentials(),
1197
+ "mssql": CredentialsManager.mssql_credentials(),
1198
+ "sqlserver": CredentialsManager.mssql_credentials(),
1199
+ "influxdb": CredentialsManager.influx_credentials(),
1200
+ }
1201
+
1202
+ return sql_credentials_map.get(db_dialect.lower())
1145
1203
 
1146
1204
  @staticmethod
1147
1205
  def open_api_credentials(token: str) -> List[CredentialValues]:
@@ -178,3 +178,94 @@ class WorkflowUtils(BaseUtils):
178
178
 
179
179
  def get_first_execution(self, workflow):
180
180
  return self.get_executions(workflow)[0]
181
+
182
+ def extract_triggered_tools_from_execution(self, workflow, execution=None):
183
+ """
184
+ Extract triggered tools from workflow execution thoughts.
185
+
186
+ Args:
187
+ workflow: Workflow object
188
+ execution: Execution object
189
+
190
+ Returns:
191
+ list: List of triggered tool names in lowercase
192
+ """
193
+
194
+ if execution:
195
+ execution_id = execution.execution_id
196
+ else:
197
+ executions = self.get_executions(workflow)
198
+ assert len(executions) > 0, "No workflow executions found"
199
+ first_execution = executions[0]
200
+ execution_id = first_execution.execution_id
201
+
202
+ triggered_tools = []
203
+
204
+ # Get execution states to find thought IDs
205
+ states_service = self.client.workflows.executions(workflow.id).states(
206
+ execution_id
207
+ )
208
+ states = states_service.list()
209
+
210
+ if not states:
211
+ return triggered_tools
212
+
213
+ # Collect all thought IDs from states
214
+ thought_ids = []
215
+ for state in states:
216
+ if hasattr(state, "thoughts") and state.thoughts:
217
+ for thought in state.thoughts:
218
+ thought_ids.append(thought.id)
219
+
220
+ if not thought_ids:
221
+ return triggered_tools
222
+
223
+ # Get detailed thoughts information
224
+ executions_service = self.client.workflows.executions(workflow.id)
225
+ thoughts_data = executions_service.get_thoughts(execution_id, thought_ids)
226
+
227
+ # Convert thoughts objects to dictionaries for processing
228
+ thoughts_dict_data = [thought.model_dump() for thought in thoughts_data]
229
+
230
+ # Extract triggered tools from thoughts
231
+ triggered_tools = self._extract_tools_from_thoughts(thoughts_dict_data)
232
+
233
+ return triggered_tools
234
+
235
+ @staticmethod
236
+ def _extract_tools_from_thoughts(thoughts_data):
237
+ """
238
+ Extract tool names from thoughts data recursively.
239
+
240
+ Args:
241
+ thoughts_data: List of thought objects with nested children
242
+
243
+ Returns:
244
+ list: List of triggered tool names in lowercase
245
+ """
246
+ triggered_tools = []
247
+
248
+ def process_thought(_thought):
249
+ # Check if this thought represents a tool call
250
+ author_name = _thought.get("author_name", "")
251
+ author_type = _thought.get("author_type", "")
252
+
253
+ # Filter out 'Codemie Thoughts' and non-tool entries
254
+ if (
255
+ author_type == "Tool"
256
+ and author_name
257
+ and author_name != "Codemie Thoughts"
258
+ ):
259
+ triggered_tools.append(author_name.lower())
260
+
261
+ # Recursively process children
262
+ children = _thought.get("children", [])
263
+ for child in children:
264
+ process_thought(child)
265
+
266
+ # Process all top-level thoughts
267
+ if isinstance(thoughts_data, list):
268
+ for thought in thoughts_data:
269
+ process_thought(thought)
270
+
271
+ return triggered_tools
@@ -5,6 +5,7 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
5
5
  KEYCLOAK_TOOL_PROMPT,
6
6
  KEYCLOAK_TOOL_RESPONSE,
7
7
  )
8
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
8
9
 
9
10
 
10
11
  @pytest.mark.workflow
@@ -28,4 +29,10 @@ def test_workflow_with_assistant_with_keycloak_tool(
28
29
  response = workflow_utils.execute_workflow(
29
30
  workflow_with_assistant.id, assistant.name
30
31
  )
32
+
33
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
34
+ workflow_with_assistant
35
+ )
36
+ assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
37
+
31
38
  similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
@@ -9,7 +9,10 @@ from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
9
9
  ADO_TEST_PLAN_DELETE_SUITE,
10
10
  ADO_TEST_PLAN_DELETE_PLAN,
11
11
  )
12
- from codemie_test_harness.tests.utils.base_utils import get_random_name
12
+ from codemie_test_harness.tests.utils.base_utils import (
13
+ get_random_name,
14
+ assert_tool_triggered,
15
+ )
13
16
  from codemie_test_harness.tests.utils.constants import ID_PATTERN
14
17
  from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
15
18
 
@@ -40,6 +43,11 @@ def test_workflow_with_assistant_with_ado_test_plan_get_tools(
40
43
  workflow_with_assistant.id, assistant.name
41
44
  )
42
45
 
46
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
47
+ workflow_with_assistant
48
+ )
49
+ assert_tool_triggered(tool_name, triggered_tools)
50
+
43
51
  similarity_check.check_similarity(response, expected_response)
44
52
 
45
53
 
@@ -76,6 +84,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
76
84
  workflow_with_assistant.id, assistant.name, create_prompt
77
85
  )
78
86
 
87
+ # Assert create test plan tool was triggered
88
+ create_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
89
+ workflow_with_assistant
90
+ )
91
+ assert_tool_triggered(
92
+ AzureDevOpsTestPlanTool.CREATE_TEST_PLAN, create_triggered_tools
93
+ )
94
+
79
95
  # Extract the test plan ID from the response
80
96
  test_plan_id = extract_id_from_ado_response(create_response, ID_PATTERN)
81
97
  create_expected = ADO_TEST_PLAN_CREATE_TEST_PLAN["expected_llm_answer"].format(
@@ -94,6 +110,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
94
110
  workflow_with_assistant.id, assistant.name, create_suite_prompt
95
111
  )
96
112
 
113
+ # Assert create test suite tool was triggered
114
+ create_suite_triggered_tools = (
115
+ workflow_utils.extract_triggered_tools_from_execution(workflow_with_assistant)
116
+ )
117
+ assert_tool_triggered(
118
+ AzureDevOpsTestPlanTool.CREATE_TEST_SUITE, create_suite_triggered_tools
119
+ )
120
+
97
121
  # Extract the test suite ID from the response
98
122
  test_suite_id = extract_id_from_ado_response(create_suite_response, ID_PATTERN)
99
123
  create_suite_expected = ADO_TEST_PLAN_CREATE_SUITE["expected_llm_answer"].format(
@@ -108,6 +132,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
108
132
  add_case_response = workflow_utils.execute_workflow(
109
133
  workflow_with_assistant.id, assistant.name, add_case_prompt
110
134
  )
135
+
136
+ # Assert add test case tool was triggered
137
+ add_case_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
138
+ workflow_with_assistant
139
+ )
140
+ assert_tool_triggered(
141
+ AzureDevOpsTestPlanTool.ADD_TEST_CASE, add_case_triggered_tools
142
+ )
111
143
  add_case_expected = ADO_TEST_PLAN_ADD_TEST_CASE["expected_llm_answer"].format(
112
144
  test_suite_id, test_plan_id
113
145
  )
@@ -120,6 +152,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
120
152
  delete_suite_response = workflow_utils.execute_workflow(
121
153
  workflow_with_assistant.id, assistant.name, delete_suite_prompt
122
154
  )
155
+
156
+ # Assert delete test suite tool was triggered
157
+ delete_suite_triggered_tools = (
158
+ workflow_utils.extract_triggered_tools_from_execution(workflow_with_assistant)
159
+ )
160
+ assert_tool_triggered(
161
+ AzureDevOpsTestPlanTool.DELETE_TEST_SUITE, delete_suite_triggered_tools
162
+ )
123
163
  delete_suite_expected = ADO_TEST_PLAN_DELETE_SUITE["expected_llm_answer"].format(
124
164
  test_suite_id, test_plan_id
125
165
  )
@@ -132,6 +172,15 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
132
172
  delete_plan_response = workflow_utils.execute_workflow(
133
173
  workflow_with_assistant.id, assistant.name, delete_plan_prompt
134
174
  )
175
+
176
+ # Assert delete test plan tool was triggered
177
+ delete_plan_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
178
+ workflow_with_assistant
179
+ )
180
+ assert_tool_triggered(
181
+ AzureDevOpsTestPlanTool.DELETE_TEST_PLAN, delete_plan_triggered_tools
182
+ )
183
+
135
184
  delete_plan_expected = ADO_TEST_PLAN_DELETE_PLAN["expected_llm_answer"].format(
136
185
  test_plan_id
137
186
  )
@@ -8,7 +8,10 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
8
8
  ADO_WIKI_MODIFY_PAGE,
9
9
  ADO_WIKI_DELETE_PAGE,
10
10
  )
11
- from codemie_test_harness.tests.utils.base_utils import get_random_name
11
+ from codemie_test_harness.tests.utils.base_utils import (
12
+ get_random_name,
13
+ assert_tool_triggered,
14
+ )
12
15
 
13
16
 
14
17
  @pytest.mark.workflow
@@ -37,6 +40,11 @@ def test_workflow_with_assistant_with_ado_wiki_get_tools(
37
40
  workflow_with_assistant.id, assistant.name
38
41
  )
39
42
 
43
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
44
+ workflow_with_assistant
45
+ )
46
+ assert_tool_triggered(tool_name, triggered_tools)
47
+
40
48
  similarity_check.check_similarity(response, expected_response)
41
49
 
42
50
 
@@ -69,6 +77,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
69
77
  create_response = workflow_utils.execute_workflow(
70
78
  workflow_with_assistant.id, assistant.name, create_prompt
71
79
  )
80
+
81
+ # Assert that modify wiki page tool was triggered
82
+ create_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
83
+ workflow_with_assistant
84
+ )
85
+ assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, create_triggered_tools)
86
+
72
87
  similarity_check.check_similarity(create_response, create_expected)
73
88
 
74
89
  # 2. Rename the page
@@ -81,6 +96,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
81
96
  rename_response = workflow_utils.execute_workflow(
82
97
  workflow_with_assistant.id, assistant.name, rename_prompt
83
98
  )
99
+
100
+ # Assert that rename wiki page tool was triggered
101
+ rename_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
102
+ workflow_with_assistant
103
+ )
104
+ assert_tool_triggered(AzureDevOpsWikiTool.RENAME_WIKI_PAGE, rename_triggered_tools)
105
+
84
106
  similarity_check.check_similarity(rename_response, rename_expected)
85
107
 
86
108
  # 3. Modify the page
@@ -93,6 +115,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
93
115
  modify_response = workflow_utils.execute_workflow(
94
116
  workflow_with_assistant.id, assistant.name, modify_prompt
95
117
  )
118
+
119
+ # Assert that modify wiki page tool was triggered
120
+ modify_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
121
+ workflow_with_assistant
122
+ )
123
+ assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, modify_triggered_tools)
124
+
96
125
  similarity_check.check_similarity(modify_response, modify_expected)
97
126
 
98
127
  # 4. Delete the page
@@ -105,4 +134,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
105
134
  delete_response = workflow_utils.execute_workflow(
106
135
  workflow_with_assistant.id, assistant.name, delete_prompt
107
136
  )
137
+
138
+ # Assert that delete wiki page tool was triggered
139
+ delete_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
140
+ workflow_with_assistant
141
+ )
142
+ assert_tool_triggered(
143
+ AzureDevOpsWikiTool.DELETE_WIKI_PAGE_BY_PATH, delete_triggered_tools
144
+ )
145
+
108
146
  similarity_check.check_similarity(delete_response, delete_expected)
@@ -7,7 +7,10 @@ from codemie_test_harness.tests.test_data.ado_work_item_tools_test_data import (
7
7
  ADO_WORK_ITEM_UPDATE,
8
8
  ADO_WORK_ITEM_LINK,
9
9
  )
10
- from codemie_test_harness.tests.utils.base_utils import get_random_name
10
+ from codemie_test_harness.tests.utils.base_utils import (
11
+ get_random_name,
12
+ assert_tool_triggered,
13
+ )
11
14
  from codemie_test_harness.tests.utils.constants import WORK_ITEM_ID_PATTERN
12
15
  from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
13
16
 
@@ -38,6 +41,11 @@ def test_workflow_with_assistant_with_ado_work_item_get_tools(
38
41
  workflow_with_assistant.id, assistant.name
39
42
  )
40
43
 
44
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
45
+ workflow_with_assistant
46
+ )
47
+ assert_tool_triggered(tool_name, triggered_tools)
48
+
41
49
  similarity_check.check_similarity(response, expected_response)
42
50
 
43
51
 
@@ -74,6 +82,15 @@ def test_workflow_with_assistant_with_ado_work_item_modify_tools(
74
82
  create_response = workflow_utils.execute_workflow(
75
83
  workflow_with_assistant.id, assistant.name, create_prompt
76
84
  )
85
+
86
+ # Assert that create work item tool was triggered
87
+ create_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
88
+ workflow_with_assistant
89
+ )
90
+ assert_tool_triggered(
91
+ AzureDevOpsWorkItemTool.CREATE_WORK_ITEM, create_triggered_tools
92
+ )
93
+
77
94
  work_item_id = extract_id_from_ado_response(create_response, WORK_ITEM_ID_PATTERN)
78
95
  similarity_check.check_similarity(create_response, create_expected)
79
96
 
@@ -89,6 +106,15 @@ def test_workflow_with_assistant_with_ado_work_item_modify_tools(
89
106
  update_response = workflow_utils.execute_workflow(
90
107
  workflow_with_assistant.id, assistant.name, update_prompt
91
108
  )
109
+
110
+ # Assert that update work item tool was triggered
111
+ update_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
112
+ workflow_with_assistant
113
+ )
114
+ assert_tool_triggered(
115
+ AzureDevOpsWorkItemTool.UPDATE_WORK_ITEM, update_triggered_tools
116
+ )
117
+
92
118
  similarity_check.check_similarity(update_response, update_expected)
93
119
 
94
120
  # 3. Link the item
@@ -100,4 +126,11 @@ def test_workflow_with_assistant_with_ado_work_item_modify_tools(
100
126
  link_response = workflow_utils.execute_workflow(
101
127
  workflow_with_assistant.id, assistant.name, link_prompt
102
128
  )
129
+
130
+ # Assert that link work items tool was triggered
131
+ link_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
132
+ workflow_with_assistant
133
+ )
134
+ assert_tool_triggered(AzureDevOpsWorkItemTool.LINK_WORK_ITEMS, link_triggered_tools)
135
+
103
136
  similarity_check.check_similarity(link_response, link_expected)
@@ -1,6 +1,7 @@
1
1
  import pytest
2
2
 
3
3
  from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
4
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
4
5
 
5
6
 
6
7
  @pytest.mark.workflow
@@ -32,4 +33,9 @@ def test_workflow_with_cloud_tools(
32
33
  workflow_with_assistant.id, assistant.name
33
34
  )
34
35
 
36
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
37
+ workflow_with_assistant
38
+ )
39
+ assert_tool_triggered(tool_name, triggered_tools)
40
+
35
41
  similarity_check.check_similarity(response, expected_response)
@@ -6,6 +6,7 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
6
6
  code_tools_test_data,
7
7
  sonar_tools_test_data,
8
8
  )
9
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
9
10
 
10
11
 
11
12
  @pytest.mark.workflow
@@ -35,6 +36,11 @@ def test_workflow_with_codebase_tools(
35
36
  workflow_with_assistant.id, assistant.name
36
37
  )
37
38
 
39
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
40
+ workflow_with_assistant
41
+ )
42
+ assert_tool_triggered(tool_name, triggered_tools)
43
+
38
44
  similarity_check.check_similarity(response, expected_response)
39
45
 
40
46
 
@@ -72,4 +78,9 @@ def test_workflow_with_sonar_tools(
72
78
  workflow_with_assistant.id, assistant.name
73
79
  )
74
80
 
81
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
82
+ workflow_with_assistant
83
+ )
84
+ assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
85
+
75
86
  similarity_check.check_similarity(response, expected)
@@ -1,7 +1,9 @@
1
1
  import pytest
2
2
  from codemie_sdk.models.integration import CredentialTypes
3
3
 
4
+ from codemie_test_harness.tests.enums.integrations import DataBaseDialect
4
5
  from codemie_test_harness.tests.enums.tools import DataManagementTool, Toolkit
6
+ from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
5
7
  from codemie_test_harness.tests.test_data.data_management_tools_test_data import (
6
8
  ELASTIC_TOOL_TASK,
7
9
  RESPONSE_FOR_ELASTIC,
@@ -11,6 +13,8 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
11
13
  SQL_TOOL_INSERT_TABLE_TASK,
12
14
  SQL_TOOL_QUERY_TABLE_TASK,
13
15
  RESPONSE_FOR_SQL,
16
+ INFLUXDB_QUERY_MEASUREMENT_TASK,
17
+ RESPONSE_FOR_INFLUXDB,
14
18
  )
15
19
  from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
16
20
  from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
@@ -45,6 +49,12 @@ def test_workflow_with_assistant_with_elastic_tools(
45
49
  response = workflow_utils.execute_workflow(
46
50
  workflow_with_assistant.id, assistant.name
47
51
  )
52
+
53
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
54
+ workflow_with_assistant
55
+ )
56
+ assert_tool_triggered(DataManagementTool.ELASTIC, triggered_tools)
57
+
48
58
  similarity_check.check_similarity(response, RESPONSE_FOR_ELASTIC)
49
59
 
50
60
 
@@ -71,21 +81,82 @@ def test_workflow_with_assistant_with_sql_tools(
71
81
  )
72
82
 
73
83
  assistant = assistant(
74
- Toolkit.DATA_MANAGEMENT, DataManagementTool.SQL, settings=settings
84
+ Toolkit.DATA_MANAGEMENT,
85
+ DataManagementTool.SQL,
86
+ settings=settings,
87
+ system_prompt="Always run tools for user prompt",
75
88
  )
76
89
  workflow_with_assistant = workflow_with_assistant(assistant, "Run")
77
90
 
91
+ # Step 1: Create table
78
92
  workflow_utils.execute_workflow(
79
93
  workflow_with_assistant.id, assistant.name, SQL_TOOL_CREATE_TABLE_TASK
80
94
  )
95
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
96
+ workflow_with_assistant
97
+ )
98
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
99
+
100
+ # Step 2: Insert data
81
101
  workflow_utils.execute_workflow(
82
102
  workflow_with_assistant.id, assistant.name, SQL_TOOL_INSERT_TABLE_TASK
83
103
  )
104
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
105
+ workflow_with_assistant
106
+ )
107
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
108
+
109
+ # Step 3: Query data
84
110
  response = workflow_utils.execute_workflow(
85
111
  workflow_with_assistant.id, assistant.name, SQL_TOOL_QUERY_TABLE_TASK
86
112
  )
113
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
114
+ workflow_with_assistant
115
+ )
116
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
117
+
118
+ # Step 4: Delete table
87
119
  workflow_utils.execute_workflow(
88
120
  workflow_with_assistant.id, assistant.name, SQL_TOOL_DELETE_TABLE_TASK
89
121
  )
122
+ triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
123
+ workflow_with_assistant
124
+ )
125
+ assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
90
126
 
91
127
  similarity_check.check_similarity(response, RESPONSE_FOR_SQL)
128
+
129
+
130
+ @pytest.mark.workflow
131
+ @pytest.mark.workflow_with_assistant
132
+ @pytest.mark.influx
133
+ @pytest.mark.api
134
+ @pytest.mark.skipif(
135
+ not EnvironmentResolver.is_sandbox(),
136
+ reason="InfluxDB is only available in sandbox environments",
137
+ )
138
+ def test_workflow_with_assistant_with_influxdb_tools(
139
+ assistant,
140
+ workflow_with_assistant,
141
+ workflow_utils,
142
+ integration_utils,
143
+ similarity_check,
144
+ ):
145
+ """Test workflow execution with InfluxDB tools via assistant."""
146
+
147
+ credential_values = CredentialsManager.sql_credentials(DataBaseDialect.INFLUX)
148
+ settings = integration_utils.create_integration(
149
+ CredentialTypes.SQL, credential_values
150
+ )
151
+
152
+ assistant = assistant(
153
+ Toolkit.DATA_MANAGEMENT, DataManagementTool.SQL, settings=settings
154
+ )
155
+ workflow_with_assistant = workflow_with_assistant(assistant, "Run")
156
+
157
+ # Query the measurement
158
+ response = workflow_utils.execute_workflow(
159
+ workflow_with_assistant.id, assistant.name, INFLUXDB_QUERY_MEASUREMENT_TASK
160
+ )
161
+
162
+ similarity_check.check_similarity(response, RESPONSE_FOR_INFLUXDB)