codemie-test-harness 0.1.183__py3-none-any.whl → 0.1.197__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of codemie-test-harness might be problematic. Click here for more details.
- codemie_test_harness/cli/cli.py +21 -6
- codemie_test_harness/cli/commands/run_cmd.py +15 -1
- codemie_test_harness/cli/constants.py +1 -21
- codemie_test_harness/cli/runner.py +13 -3
- codemie_test_harness/cli/utils.py +12 -2
- codemie_test_harness/tests/assistant/test_assistants.py +17 -1
- codemie_test_harness/tests/assistant/tools/codebase/test_codebase_tools.py +1 -1
- codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +47 -6
- codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +0 -4
- codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +0 -4
- codemie_test_harness/tests/conftest.py +2 -2
- codemie_test_harness/tests/enums/environment.py +3 -3
- codemie_test_harness/tests/enums/integrations.py +1 -0
- codemie_test_harness/tests/enums/model_types.py +1 -0
- codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -15
- codemie_test_harness/tests/integrations/project/test_project_integrations.py +42 -0
- codemie_test_harness/tests/integrations/user/test_default_integrations.py +0 -7
- codemie_test_harness/tests/llm/assistants/test_llm.py +56 -2
- codemie_test_harness/tests/test_data/assistant_test_data.py +448 -448
- codemie_test_harness/tests/test_data/codebase_tools_test_data.py +2 -0
- codemie_test_harness/tests/test_data/data_management_tools_test_data.py +18 -0
- codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +18 -1
- codemie_test_harness/tests/test_data/direct_tools/report_portal_tools_test_data.py +189 -197
- codemie_test_harness/tests/test_data/integrations_test_data.py +163 -2
- codemie_test_harness/tests/test_data/llm_test_data.py +1 -0
- codemie_test_harness/tests/test_data/open_api_tools_test_data.py +22 -1
- codemie_test_harness/tests/test_data/report_portal_tools_test_data.py +89 -112
- codemie_test_harness/tests/test_data/research_tools_test_data.py +29 -7
- codemie_test_harness/tests/utils/assistant_utils.py +22 -12
- codemie_test_harness/tests/utils/credentials_manager.py +66 -8
- codemie_test_harness/tests/utils/workflow_utils.py +91 -0
- codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +50 -1
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +39 -1
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +34 -1
- codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +11 -0
- codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +72 -1
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +31 -0
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +31 -0
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +26 -0
- codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +63 -1
- codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +82 -7
- codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +23 -4
- codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +12 -0
- codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +29 -2
- codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +14 -1
- codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +14 -1
- codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +6 -0
- codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +12 -9
- codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +8 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +28 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +24 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +20 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +13 -4
- codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +16 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +73 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +101 -49
- codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +42 -3
- codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +27 -5
- codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +13 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +20 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +14 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +9 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +9 -1
- {codemie_test_harness-0.1.183.dist-info → codemie_test_harness-0.1.197.dist-info}/METADATA +38 -3
- {codemie_test_harness-0.1.183.dist-info → codemie_test_harness-0.1.197.dist-info}/RECORD +80 -80
- {codemie_test_harness-0.1.183.dist-info → codemie_test_harness-0.1.197.dist-info}/WHEEL +0 -0
- {codemie_test_harness-0.1.183.dist-info → codemie_test_harness-0.1.197.dist-info}/entry_points.txt +0 -0
|
@@ -9,9 +9,10 @@ import os
|
|
|
9
9
|
import re
|
|
10
10
|
from typing import Optional, List, Dict, Any
|
|
11
11
|
|
|
12
|
+
from codemie_sdk.models.integration import CredentialValues
|
|
13
|
+
|
|
12
14
|
from codemie_test_harness.tests.utils.aws_parameters_store import AwsParameterStore
|
|
13
15
|
from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
|
|
14
|
-
from codemie_sdk.models.integration import CredentialValues
|
|
15
16
|
|
|
16
17
|
# Constants
|
|
17
18
|
INTEGRATIONS_PARAMETER_PATH = "/codemie/autotests/integrations/"
|
|
@@ -27,7 +28,7 @@ class CredentialsManager:
|
|
|
27
28
|
3. Default values (final fallback)
|
|
28
29
|
"""
|
|
29
30
|
|
|
30
|
-
AUTO_GENERATED = "
|
|
31
|
+
AUTO_GENERATED = "AutoGenerated"
|
|
31
32
|
|
|
32
33
|
# Configuration mapping: env_var -> AWS parameter details
|
|
33
34
|
CREDENTIAL_MAPPINGS = {
|
|
@@ -541,6 +542,37 @@ class CredentialsManager:
|
|
|
541
542
|
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
542
543
|
"aws_path": "sandbox.mssql.password",
|
|
543
544
|
},
|
|
545
|
+
# Sandbox InfluxDB (only available in sandbox)
|
|
546
|
+
"SANDBOX_INFLUX_DIALECT": {
|
|
547
|
+
"env": "INFLUX_DIALECT",
|
|
548
|
+
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
549
|
+
"aws_path": "sandbox.influxdb.dialect",
|
|
550
|
+
},
|
|
551
|
+
"SANDBOX_INFLUX_URL": {
|
|
552
|
+
"env": "INFLUX_URL",
|
|
553
|
+
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
554
|
+
"aws_path": "sandbox.influxdb.url",
|
|
555
|
+
},
|
|
556
|
+
"SANDBOX_INFLUX_PORT": {
|
|
557
|
+
"env": "INFLUX_PORT",
|
|
558
|
+
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
559
|
+
"aws_path": "sandbox.influxdb.port",
|
|
560
|
+
},
|
|
561
|
+
"SANDBOX_INFLUX_TOKEN": {
|
|
562
|
+
"env": "INFLUX_TOKEN",
|
|
563
|
+
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
564
|
+
"aws_path": "sandbox.influxdb.token",
|
|
565
|
+
},
|
|
566
|
+
"SANDBOX_INFLUX_ORG_NAME": {
|
|
567
|
+
"env": "INFLUX_ORG_NAME",
|
|
568
|
+
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
569
|
+
"aws_path": "sandbox.influxdb.org_name",
|
|
570
|
+
},
|
|
571
|
+
"SANDBOX_INFLUX_BUCKET": {
|
|
572
|
+
"env": "INFLUX_BUCKET",
|
|
573
|
+
"aws_param": f"{INTEGRATIONS_PARAMETER_PATH}sql",
|
|
574
|
+
"aws_path": "sandbox.influxdb.bucket",
|
|
575
|
+
},
|
|
544
576
|
}
|
|
545
577
|
|
|
546
578
|
# Cache for AWS Parameter Store data to avoid repeated calls
|
|
@@ -1126,6 +1158,28 @@ class CredentialsManager:
|
|
|
1126
1158
|
CredentialValues(key="password", value=password),
|
|
1127
1159
|
]
|
|
1128
1160
|
|
|
1161
|
+
@staticmethod
|
|
1162
|
+
def influx_credentials() -> List[CredentialValues]:
|
|
1163
|
+
"""Create INFLUX credentials using configuration-driven approach.
|
|
1164
|
+
|
|
1165
|
+
INFLUX DB is only available in sandbox environment.
|
|
1166
|
+
"""
|
|
1167
|
+
dialect = CredentialsManager.get_parameter("SANDBOX_INFLUX_DIALECT")
|
|
1168
|
+
url = CredentialsManager.get_parameter("SANDBOX_INFLUX_URL")
|
|
1169
|
+
port = CredentialsManager.get_parameter("SANDBOX_INFLUX_PORT")
|
|
1170
|
+
token = CredentialsManager.get_parameter("SANDBOX_INFLUX_TOKEN")
|
|
1171
|
+
org = CredentialsManager.get_parameter("SANDBOX_INFLUX_ORG_NAME")
|
|
1172
|
+
bucket = CredentialsManager.get_parameter("SANDBOX_INFLUX_BUCKET")
|
|
1173
|
+
|
|
1174
|
+
return [
|
|
1175
|
+
CredentialValues(key="dialect", value=dialect),
|
|
1176
|
+
CredentialValues(key="url", value=url),
|
|
1177
|
+
CredentialValues(key="port", value=port),
|
|
1178
|
+
CredentialValues(key="token", value=token),
|
|
1179
|
+
CredentialValues(key="org", value=org),
|
|
1180
|
+
CredentialValues(key="bucket", value=bucket),
|
|
1181
|
+
]
|
|
1182
|
+
|
|
1129
1183
|
@staticmethod
|
|
1130
1184
|
def sql_credentials(db_dialect: str) -> list[CredentialValues] | None:
|
|
1131
1185
|
"""Create SQL credentials using environment-aware configuration-driven approach.
|
|
@@ -1136,12 +1190,16 @@ class CredentialsManager:
|
|
|
1136
1190
|
Returns:
|
|
1137
1191
|
List of credential values for the specified database dialect
|
|
1138
1192
|
"""
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1193
|
+
sql_credentials_map = {
|
|
1194
|
+
"mysql": CredentialsManager.mysql_credentials(),
|
|
1195
|
+
"postgres": CredentialsManager.postgres_credentials(),
|
|
1196
|
+
"postgresql": CredentialsManager.postgres_credentials(),
|
|
1197
|
+
"mssql": CredentialsManager.mssql_credentials(),
|
|
1198
|
+
"sqlserver": CredentialsManager.mssql_credentials(),
|
|
1199
|
+
"influxdb": CredentialsManager.influx_credentials(),
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
return sql_credentials_map.get(db_dialect.lower())
|
|
1145
1203
|
|
|
1146
1204
|
@staticmethod
|
|
1147
1205
|
def open_api_credentials(token: str) -> List[CredentialValues]:
|
|
@@ -178,3 +178,94 @@ class WorkflowUtils(BaseUtils):
|
|
|
178
178
|
|
|
179
179
|
def get_first_execution(self, workflow):
|
|
180
180
|
return self.get_executions(workflow)[0]
|
|
181
|
+
|
|
182
|
+
def extract_triggered_tools_from_execution(self, workflow, execution=None):
|
|
183
|
+
"""
|
|
184
|
+
Extract triggered tools from workflow execution thoughts.
|
|
185
|
+
|
|
186
|
+
Args:
|
|
187
|
+
workflow: Workflow object
|
|
188
|
+
execution: Execution object
|
|
189
|
+
|
|
190
|
+
Returns:
|
|
191
|
+
list: List of triggered tool names in lowercase
|
|
192
|
+
"""
|
|
193
|
+
|
|
194
|
+
if execution:
|
|
195
|
+
execution_id = execution.execution_id
|
|
196
|
+
else:
|
|
197
|
+
executions = self.get_executions(workflow)
|
|
198
|
+
assert len(executions) > 0, "No workflow executions found"
|
|
199
|
+
first_execution = executions[0]
|
|
200
|
+
execution_id = first_execution.execution_id
|
|
201
|
+
|
|
202
|
+
triggered_tools = []
|
|
203
|
+
|
|
204
|
+
# Get execution states to find thought IDs
|
|
205
|
+
states_service = self.client.workflows.executions(workflow.id).states(
|
|
206
|
+
execution_id
|
|
207
|
+
)
|
|
208
|
+
states = states_service.list()
|
|
209
|
+
|
|
210
|
+
if not states:
|
|
211
|
+
return triggered_tools
|
|
212
|
+
|
|
213
|
+
# Collect all thought IDs from states
|
|
214
|
+
thought_ids = []
|
|
215
|
+
for state in states:
|
|
216
|
+
if hasattr(state, "thoughts") and state.thoughts:
|
|
217
|
+
for thought in state.thoughts:
|
|
218
|
+
thought_ids.append(thought.id)
|
|
219
|
+
|
|
220
|
+
if not thought_ids:
|
|
221
|
+
return triggered_tools
|
|
222
|
+
|
|
223
|
+
# Get detailed thoughts information
|
|
224
|
+
executions_service = self.client.workflows.executions(workflow.id)
|
|
225
|
+
thoughts_data = executions_service.get_thoughts(execution_id, thought_ids)
|
|
226
|
+
|
|
227
|
+
# Convert thoughts objects to dictionaries for processing
|
|
228
|
+
thoughts_dict_data = [thought.model_dump() for thought in thoughts_data]
|
|
229
|
+
|
|
230
|
+
# Extract triggered tools from thoughts
|
|
231
|
+
triggered_tools = self._extract_tools_from_thoughts(thoughts_dict_data)
|
|
232
|
+
|
|
233
|
+
return triggered_tools
|
|
234
|
+
|
|
235
|
+
@staticmethod
|
|
236
|
+
def _extract_tools_from_thoughts(thoughts_data):
|
|
237
|
+
"""
|
|
238
|
+
Extract tool names from thoughts data recursively.
|
|
239
|
+
|
|
240
|
+
Args:
|
|
241
|
+
thoughts_data: List of thought objects with nested children
|
|
242
|
+
|
|
243
|
+
Returns:
|
|
244
|
+
list: List of triggered tool names in lowercase
|
|
245
|
+
"""
|
|
246
|
+
triggered_tools = []
|
|
247
|
+
|
|
248
|
+
def process_thought(_thought):
|
|
249
|
+
# Check if this thought represents a tool call
|
|
250
|
+
author_name = _thought.get("author_name", "")
|
|
251
|
+
author_type = _thought.get("author_type", "")
|
|
252
|
+
|
|
253
|
+
# Filter out 'Codemie Thoughts' and non-tool entries
|
|
254
|
+
if (
|
|
255
|
+
author_type == "Tool"
|
|
256
|
+
and author_name
|
|
257
|
+
and author_name != "Codemie Thoughts"
|
|
258
|
+
):
|
|
259
|
+
triggered_tools.append(author_name.lower())
|
|
260
|
+
|
|
261
|
+
# Recursively process children
|
|
262
|
+
children = _thought.get("children", [])
|
|
263
|
+
for child in children:
|
|
264
|
+
process_thought(child)
|
|
265
|
+
|
|
266
|
+
# Process all top-level thoughts
|
|
267
|
+
if isinstance(thoughts_data, list):
|
|
268
|
+
for thought in thoughts_data:
|
|
269
|
+
process_thought(thought)
|
|
270
|
+
|
|
271
|
+
return triggered_tools
|
|
@@ -5,6 +5,7 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
|
|
|
5
5
|
KEYCLOAK_TOOL_PROMPT,
|
|
6
6
|
KEYCLOAK_TOOL_RESPONSE,
|
|
7
7
|
)
|
|
8
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
@pytest.mark.workflow
|
|
@@ -28,4 +29,10 @@ def test_workflow_with_assistant_with_keycloak_tool(
|
|
|
28
29
|
response = workflow_utils.execute_workflow(
|
|
29
30
|
workflow_with_assistant.id, assistant.name
|
|
30
31
|
)
|
|
32
|
+
|
|
33
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
34
|
+
workflow_with_assistant
|
|
35
|
+
)
|
|
36
|
+
assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
|
|
37
|
+
|
|
31
38
|
similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
|
|
@@ -9,7 +9,10 @@ from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
|
|
|
9
9
|
ADO_TEST_PLAN_DELETE_SUITE,
|
|
10
10
|
ADO_TEST_PLAN_DELETE_PLAN,
|
|
11
11
|
)
|
|
12
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
12
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
13
|
+
get_random_name,
|
|
14
|
+
assert_tool_triggered,
|
|
15
|
+
)
|
|
13
16
|
from codemie_test_harness.tests.utils.constants import ID_PATTERN
|
|
14
17
|
from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
|
|
15
18
|
|
|
@@ -40,6 +43,11 @@ def test_workflow_with_assistant_with_ado_test_plan_get_tools(
|
|
|
40
43
|
workflow_with_assistant.id, assistant.name
|
|
41
44
|
)
|
|
42
45
|
|
|
46
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
47
|
+
workflow_with_assistant
|
|
48
|
+
)
|
|
49
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
50
|
+
|
|
43
51
|
similarity_check.check_similarity(response, expected_response)
|
|
44
52
|
|
|
45
53
|
|
|
@@ -76,6 +84,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
|
|
|
76
84
|
workflow_with_assistant.id, assistant.name, create_prompt
|
|
77
85
|
)
|
|
78
86
|
|
|
87
|
+
# Assert create test plan tool was triggered
|
|
88
|
+
create_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
89
|
+
workflow_with_assistant
|
|
90
|
+
)
|
|
91
|
+
assert_tool_triggered(
|
|
92
|
+
AzureDevOpsTestPlanTool.CREATE_TEST_PLAN, create_triggered_tools
|
|
93
|
+
)
|
|
94
|
+
|
|
79
95
|
# Extract the test plan ID from the response
|
|
80
96
|
test_plan_id = extract_id_from_ado_response(create_response, ID_PATTERN)
|
|
81
97
|
create_expected = ADO_TEST_PLAN_CREATE_TEST_PLAN["expected_llm_answer"].format(
|
|
@@ -94,6 +110,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
|
|
|
94
110
|
workflow_with_assistant.id, assistant.name, create_suite_prompt
|
|
95
111
|
)
|
|
96
112
|
|
|
113
|
+
# Assert create test suite tool was triggered
|
|
114
|
+
create_suite_triggered_tools = (
|
|
115
|
+
workflow_utils.extract_triggered_tools_from_execution(workflow_with_assistant)
|
|
116
|
+
)
|
|
117
|
+
assert_tool_triggered(
|
|
118
|
+
AzureDevOpsTestPlanTool.CREATE_TEST_SUITE, create_suite_triggered_tools
|
|
119
|
+
)
|
|
120
|
+
|
|
97
121
|
# Extract the test suite ID from the response
|
|
98
122
|
test_suite_id = extract_id_from_ado_response(create_suite_response, ID_PATTERN)
|
|
99
123
|
create_suite_expected = ADO_TEST_PLAN_CREATE_SUITE["expected_llm_answer"].format(
|
|
@@ -108,6 +132,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
|
|
|
108
132
|
add_case_response = workflow_utils.execute_workflow(
|
|
109
133
|
workflow_with_assistant.id, assistant.name, add_case_prompt
|
|
110
134
|
)
|
|
135
|
+
|
|
136
|
+
# Assert add test case tool was triggered
|
|
137
|
+
add_case_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
138
|
+
workflow_with_assistant
|
|
139
|
+
)
|
|
140
|
+
assert_tool_triggered(
|
|
141
|
+
AzureDevOpsTestPlanTool.ADD_TEST_CASE, add_case_triggered_tools
|
|
142
|
+
)
|
|
111
143
|
add_case_expected = ADO_TEST_PLAN_ADD_TEST_CASE["expected_llm_answer"].format(
|
|
112
144
|
test_suite_id, test_plan_id
|
|
113
145
|
)
|
|
@@ -120,6 +152,14 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
|
|
|
120
152
|
delete_suite_response = workflow_utils.execute_workflow(
|
|
121
153
|
workflow_with_assistant.id, assistant.name, delete_suite_prompt
|
|
122
154
|
)
|
|
155
|
+
|
|
156
|
+
# Assert delete test suite tool was triggered
|
|
157
|
+
delete_suite_triggered_tools = (
|
|
158
|
+
workflow_utils.extract_triggered_tools_from_execution(workflow_with_assistant)
|
|
159
|
+
)
|
|
160
|
+
assert_tool_triggered(
|
|
161
|
+
AzureDevOpsTestPlanTool.DELETE_TEST_SUITE, delete_suite_triggered_tools
|
|
162
|
+
)
|
|
123
163
|
delete_suite_expected = ADO_TEST_PLAN_DELETE_SUITE["expected_llm_answer"].format(
|
|
124
164
|
test_suite_id, test_plan_id
|
|
125
165
|
)
|
|
@@ -132,6 +172,15 @@ def test_workflow_with_assistant_with_ado_test_plan_tools(
|
|
|
132
172
|
delete_plan_response = workflow_utils.execute_workflow(
|
|
133
173
|
workflow_with_assistant.id, assistant.name, delete_plan_prompt
|
|
134
174
|
)
|
|
175
|
+
|
|
176
|
+
# Assert delete test plan tool was triggered
|
|
177
|
+
delete_plan_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
178
|
+
workflow_with_assistant
|
|
179
|
+
)
|
|
180
|
+
assert_tool_triggered(
|
|
181
|
+
AzureDevOpsTestPlanTool.DELETE_TEST_PLAN, delete_plan_triggered_tools
|
|
182
|
+
)
|
|
183
|
+
|
|
135
184
|
delete_plan_expected = ADO_TEST_PLAN_DELETE_PLAN["expected_llm_answer"].format(
|
|
136
185
|
test_plan_id
|
|
137
186
|
)
|
|
@@ -8,7 +8,10 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
|
|
|
8
8
|
ADO_WIKI_MODIFY_PAGE,
|
|
9
9
|
ADO_WIKI_DELETE_PAGE,
|
|
10
10
|
)
|
|
11
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
11
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
12
|
+
get_random_name,
|
|
13
|
+
assert_tool_triggered,
|
|
14
|
+
)
|
|
12
15
|
|
|
13
16
|
|
|
14
17
|
@pytest.mark.workflow
|
|
@@ -37,6 +40,11 @@ def test_workflow_with_assistant_with_ado_wiki_get_tools(
|
|
|
37
40
|
workflow_with_assistant.id, assistant.name
|
|
38
41
|
)
|
|
39
42
|
|
|
43
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
44
|
+
workflow_with_assistant
|
|
45
|
+
)
|
|
46
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
47
|
+
|
|
40
48
|
similarity_check.check_similarity(response, expected_response)
|
|
41
49
|
|
|
42
50
|
|
|
@@ -69,6 +77,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
|
|
|
69
77
|
create_response = workflow_utils.execute_workflow(
|
|
70
78
|
workflow_with_assistant.id, assistant.name, create_prompt
|
|
71
79
|
)
|
|
80
|
+
|
|
81
|
+
# Assert that modify wiki page tool was triggered
|
|
82
|
+
create_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
83
|
+
workflow_with_assistant
|
|
84
|
+
)
|
|
85
|
+
assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, create_triggered_tools)
|
|
86
|
+
|
|
72
87
|
similarity_check.check_similarity(create_response, create_expected)
|
|
73
88
|
|
|
74
89
|
# 2. Rename the page
|
|
@@ -81,6 +96,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
|
|
|
81
96
|
rename_response = workflow_utils.execute_workflow(
|
|
82
97
|
workflow_with_assistant.id, assistant.name, rename_prompt
|
|
83
98
|
)
|
|
99
|
+
|
|
100
|
+
# Assert that rename wiki page tool was triggered
|
|
101
|
+
rename_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
102
|
+
workflow_with_assistant
|
|
103
|
+
)
|
|
104
|
+
assert_tool_triggered(AzureDevOpsWikiTool.RENAME_WIKI_PAGE, rename_triggered_tools)
|
|
105
|
+
|
|
84
106
|
similarity_check.check_similarity(rename_response, rename_expected)
|
|
85
107
|
|
|
86
108
|
# 3. Modify the page
|
|
@@ -93,6 +115,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
|
|
|
93
115
|
modify_response = workflow_utils.execute_workflow(
|
|
94
116
|
workflow_with_assistant.id, assistant.name, modify_prompt
|
|
95
117
|
)
|
|
118
|
+
|
|
119
|
+
# Assert that modify wiki page tool was triggered
|
|
120
|
+
modify_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
121
|
+
workflow_with_assistant
|
|
122
|
+
)
|
|
123
|
+
assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, modify_triggered_tools)
|
|
124
|
+
|
|
96
125
|
similarity_check.check_similarity(modify_response, modify_expected)
|
|
97
126
|
|
|
98
127
|
# 4. Delete the page
|
|
@@ -105,4 +134,13 @@ def test_workflow_with_assistant_with_ado_wiki_modify_tools(
|
|
|
105
134
|
delete_response = workflow_utils.execute_workflow(
|
|
106
135
|
workflow_with_assistant.id, assistant.name, delete_prompt
|
|
107
136
|
)
|
|
137
|
+
|
|
138
|
+
# Assert that delete wiki page tool was triggered
|
|
139
|
+
delete_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
140
|
+
workflow_with_assistant
|
|
141
|
+
)
|
|
142
|
+
assert_tool_triggered(
|
|
143
|
+
AzureDevOpsWikiTool.DELETE_WIKI_PAGE_BY_PATH, delete_triggered_tools
|
|
144
|
+
)
|
|
145
|
+
|
|
108
146
|
similarity_check.check_similarity(delete_response, delete_expected)
|
|
@@ -7,7 +7,10 @@ from codemie_test_harness.tests.test_data.ado_work_item_tools_test_data import (
|
|
|
7
7
|
ADO_WORK_ITEM_UPDATE,
|
|
8
8
|
ADO_WORK_ITEM_LINK,
|
|
9
9
|
)
|
|
10
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
10
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
11
|
+
get_random_name,
|
|
12
|
+
assert_tool_triggered,
|
|
13
|
+
)
|
|
11
14
|
from codemie_test_harness.tests.utils.constants import WORK_ITEM_ID_PATTERN
|
|
12
15
|
from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
|
|
13
16
|
|
|
@@ -38,6 +41,11 @@ def test_workflow_with_assistant_with_ado_work_item_get_tools(
|
|
|
38
41
|
workflow_with_assistant.id, assistant.name
|
|
39
42
|
)
|
|
40
43
|
|
|
44
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
45
|
+
workflow_with_assistant
|
|
46
|
+
)
|
|
47
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
48
|
+
|
|
41
49
|
similarity_check.check_similarity(response, expected_response)
|
|
42
50
|
|
|
43
51
|
|
|
@@ -74,6 +82,15 @@ def test_workflow_with_assistant_with_ado_work_item_modify_tools(
|
|
|
74
82
|
create_response = workflow_utils.execute_workflow(
|
|
75
83
|
workflow_with_assistant.id, assistant.name, create_prompt
|
|
76
84
|
)
|
|
85
|
+
|
|
86
|
+
# Assert that create work item tool was triggered
|
|
87
|
+
create_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
88
|
+
workflow_with_assistant
|
|
89
|
+
)
|
|
90
|
+
assert_tool_triggered(
|
|
91
|
+
AzureDevOpsWorkItemTool.CREATE_WORK_ITEM, create_triggered_tools
|
|
92
|
+
)
|
|
93
|
+
|
|
77
94
|
work_item_id = extract_id_from_ado_response(create_response, WORK_ITEM_ID_PATTERN)
|
|
78
95
|
similarity_check.check_similarity(create_response, create_expected)
|
|
79
96
|
|
|
@@ -89,6 +106,15 @@ def test_workflow_with_assistant_with_ado_work_item_modify_tools(
|
|
|
89
106
|
update_response = workflow_utils.execute_workflow(
|
|
90
107
|
workflow_with_assistant.id, assistant.name, update_prompt
|
|
91
108
|
)
|
|
109
|
+
|
|
110
|
+
# Assert that update work item tool was triggered
|
|
111
|
+
update_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
112
|
+
workflow_with_assistant
|
|
113
|
+
)
|
|
114
|
+
assert_tool_triggered(
|
|
115
|
+
AzureDevOpsWorkItemTool.UPDATE_WORK_ITEM, update_triggered_tools
|
|
116
|
+
)
|
|
117
|
+
|
|
92
118
|
similarity_check.check_similarity(update_response, update_expected)
|
|
93
119
|
|
|
94
120
|
# 3. Link the item
|
|
@@ -100,4 +126,11 @@ def test_workflow_with_assistant_with_ado_work_item_modify_tools(
|
|
|
100
126
|
link_response = workflow_utils.execute_workflow(
|
|
101
127
|
workflow_with_assistant.id, assistant.name, link_prompt
|
|
102
128
|
)
|
|
129
|
+
|
|
130
|
+
# Assert that link work items tool was triggered
|
|
131
|
+
link_triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
132
|
+
workflow_with_assistant
|
|
133
|
+
)
|
|
134
|
+
assert_tool_triggered(AzureDevOpsWorkItemTool.LINK_WORK_ITEMS, link_triggered_tools)
|
|
135
|
+
|
|
103
136
|
similarity_check.check_similarity(link_response, link_expected)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
|
|
3
3
|
from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
|
|
4
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
@pytest.mark.workflow
|
|
@@ -32,4 +33,9 @@ def test_workflow_with_cloud_tools(
|
|
|
32
33
|
workflow_with_assistant.id, assistant.name
|
|
33
34
|
)
|
|
34
35
|
|
|
36
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
37
|
+
workflow_with_assistant
|
|
38
|
+
)
|
|
39
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
40
|
+
|
|
35
41
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -6,6 +6,7 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
|
|
|
6
6
|
code_tools_test_data,
|
|
7
7
|
sonar_tools_test_data,
|
|
8
8
|
)
|
|
9
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
9
10
|
|
|
10
11
|
|
|
11
12
|
@pytest.mark.workflow
|
|
@@ -35,6 +36,11 @@ def test_workflow_with_codebase_tools(
|
|
|
35
36
|
workflow_with_assistant.id, assistant.name
|
|
36
37
|
)
|
|
37
38
|
|
|
39
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
40
|
+
workflow_with_assistant
|
|
41
|
+
)
|
|
42
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
43
|
+
|
|
38
44
|
similarity_check.check_similarity(response, expected_response)
|
|
39
45
|
|
|
40
46
|
|
|
@@ -72,4 +78,9 @@ def test_workflow_with_sonar_tools(
|
|
|
72
78
|
workflow_with_assistant.id, assistant.name
|
|
73
79
|
)
|
|
74
80
|
|
|
81
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
82
|
+
workflow_with_assistant
|
|
83
|
+
)
|
|
84
|
+
assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
|
|
85
|
+
|
|
75
86
|
similarity_check.check_similarity(response, expected)
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
from codemie_sdk.models.integration import CredentialTypes
|
|
3
3
|
|
|
4
|
+
from codemie_test_harness.tests.enums.integrations import DataBaseDialect
|
|
4
5
|
from codemie_test_harness.tests.enums.tools import DataManagementTool, Toolkit
|
|
6
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
5
7
|
from codemie_test_harness.tests.test_data.data_management_tools_test_data import (
|
|
6
8
|
ELASTIC_TOOL_TASK,
|
|
7
9
|
RESPONSE_FOR_ELASTIC,
|
|
@@ -11,6 +13,8 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
|
|
|
11
13
|
SQL_TOOL_INSERT_TABLE_TASK,
|
|
12
14
|
SQL_TOOL_QUERY_TABLE_TASK,
|
|
13
15
|
RESPONSE_FOR_SQL,
|
|
16
|
+
INFLUXDB_QUERY_MEASUREMENT_TASK,
|
|
17
|
+
RESPONSE_FOR_INFLUXDB,
|
|
14
18
|
)
|
|
15
19
|
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
16
20
|
from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
|
|
@@ -45,6 +49,12 @@ def test_workflow_with_assistant_with_elastic_tools(
|
|
|
45
49
|
response = workflow_utils.execute_workflow(
|
|
46
50
|
workflow_with_assistant.id, assistant.name
|
|
47
51
|
)
|
|
52
|
+
|
|
53
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
54
|
+
workflow_with_assistant
|
|
55
|
+
)
|
|
56
|
+
assert_tool_triggered(DataManagementTool.ELASTIC, triggered_tools)
|
|
57
|
+
|
|
48
58
|
similarity_check.check_similarity(response, RESPONSE_FOR_ELASTIC)
|
|
49
59
|
|
|
50
60
|
|
|
@@ -71,21 +81,82 @@ def test_workflow_with_assistant_with_sql_tools(
|
|
|
71
81
|
)
|
|
72
82
|
|
|
73
83
|
assistant = assistant(
|
|
74
|
-
Toolkit.DATA_MANAGEMENT,
|
|
84
|
+
Toolkit.DATA_MANAGEMENT,
|
|
85
|
+
DataManagementTool.SQL,
|
|
86
|
+
settings=settings,
|
|
87
|
+
system_prompt="Always run tools for user prompt",
|
|
75
88
|
)
|
|
76
89
|
workflow_with_assistant = workflow_with_assistant(assistant, "Run")
|
|
77
90
|
|
|
91
|
+
# Step 1: Create table
|
|
78
92
|
workflow_utils.execute_workflow(
|
|
79
93
|
workflow_with_assistant.id, assistant.name, SQL_TOOL_CREATE_TABLE_TASK
|
|
80
94
|
)
|
|
95
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
96
|
+
workflow_with_assistant
|
|
97
|
+
)
|
|
98
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
99
|
+
|
|
100
|
+
# Step 2: Insert data
|
|
81
101
|
workflow_utils.execute_workflow(
|
|
82
102
|
workflow_with_assistant.id, assistant.name, SQL_TOOL_INSERT_TABLE_TASK
|
|
83
103
|
)
|
|
104
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
105
|
+
workflow_with_assistant
|
|
106
|
+
)
|
|
107
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
108
|
+
|
|
109
|
+
# Step 3: Query data
|
|
84
110
|
response = workflow_utils.execute_workflow(
|
|
85
111
|
workflow_with_assistant.id, assistant.name, SQL_TOOL_QUERY_TABLE_TASK
|
|
86
112
|
)
|
|
113
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
114
|
+
workflow_with_assistant
|
|
115
|
+
)
|
|
116
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
117
|
+
|
|
118
|
+
# Step 4: Delete table
|
|
87
119
|
workflow_utils.execute_workflow(
|
|
88
120
|
workflow_with_assistant.id, assistant.name, SQL_TOOL_DELETE_TABLE_TASK
|
|
89
121
|
)
|
|
122
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
123
|
+
workflow_with_assistant
|
|
124
|
+
)
|
|
125
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
90
126
|
|
|
91
127
|
similarity_check.check_similarity(response, RESPONSE_FOR_SQL)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@pytest.mark.workflow
|
|
131
|
+
@pytest.mark.workflow_with_assistant
|
|
132
|
+
@pytest.mark.influx
|
|
133
|
+
@pytest.mark.api
|
|
134
|
+
@pytest.mark.skipif(
|
|
135
|
+
not EnvironmentResolver.is_sandbox(),
|
|
136
|
+
reason="InfluxDB is only available in sandbox environments",
|
|
137
|
+
)
|
|
138
|
+
def test_workflow_with_assistant_with_influxdb_tools(
|
|
139
|
+
assistant,
|
|
140
|
+
workflow_with_assistant,
|
|
141
|
+
workflow_utils,
|
|
142
|
+
integration_utils,
|
|
143
|
+
similarity_check,
|
|
144
|
+
):
|
|
145
|
+
"""Test workflow execution with InfluxDB tools via assistant."""
|
|
146
|
+
|
|
147
|
+
credential_values = CredentialsManager.sql_credentials(DataBaseDialect.INFLUX)
|
|
148
|
+
settings = integration_utils.create_integration(
|
|
149
|
+
CredentialTypes.SQL, credential_values
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
assistant = assistant(
|
|
153
|
+
Toolkit.DATA_MANAGEMENT, DataManagementTool.SQL, settings=settings
|
|
154
|
+
)
|
|
155
|
+
workflow_with_assistant = workflow_with_assistant(assistant, "Run")
|
|
156
|
+
|
|
157
|
+
# Query the measurement
|
|
158
|
+
response = workflow_utils.execute_workflow(
|
|
159
|
+
workflow_with_assistant.id, assistant.name, INFLUXDB_QUERY_MEASUREMENT_TASK
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
similarity_check.check_similarity(response, RESPONSE_FOR_INFLUXDB)
|