codemie-test-harness 0.1.184__py3-none-any.whl → 0.1.198__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of codemie-test-harness might be problematic. Click here for more details.
- codemie_test_harness/cli/cli.py +42 -6
- codemie_test_harness/cli/commands/config_cmd.py +1 -1
- codemie_test_harness/cli/commands/run_cmd.py +24 -1
- codemie_test_harness/cli/constants.py +1 -0
- codemie_test_harness/cli/runner.py +17 -3
- codemie_test_harness/cli/utils.py +12 -2
- codemie_test_harness/pytest.ini +3 -0
- codemie_test_harness/tests/assistant/test_assistants.py +17 -1
- codemie_test_harness/tests/assistant/tools/datamanagement/test_assistant_with_data_management_tools.py +47 -6
- codemie_test_harness/tests/assistant/tools/mcp/test_cli_mcp_server.py +0 -4
- codemie_test_harness/tests/assistant/tools/mcp/test_mcp_servers.py +0 -4
- codemie_test_harness/tests/conftest.py +19 -3
- codemie_test_harness/tests/enums/environment.py +3 -3
- codemie_test_harness/tests/enums/integrations.py +1 -0
- codemie_test_harness/tests/enums/model_types.py +1 -0
- codemie_test_harness/tests/integrations/project/test_default_integrations.py +41 -15
- codemie_test_harness/tests/integrations/project/test_project_integrations.py +42 -0
- codemie_test_harness/tests/integrations/user/test_default_integrations.py +41 -15
- codemie_test_harness/tests/llm/assistants/test_llm.py +45 -2
- codemie_test_harness/tests/test_data/assistant_test_data.py +171 -171
- codemie_test_harness/tests/test_data/codebase_tools_test_data.py +2 -0
- codemie_test_harness/tests/test_data/data_management_tools_test_data.py +18 -0
- codemie_test_harness/tests/test_data/direct_tools/data_management_tools_test_data.py +18 -1
- codemie_test_harness/tests/test_data/direct_tools/report_portal_tools_test_data.py +189 -197
- codemie_test_harness/tests/test_data/integrations_test_data.py +163 -2
- codemie_test_harness/tests/test_data/llm_test_data.py +1 -0
- codemie_test_harness/tests/test_data/open_api_tools_test_data.py +22 -1
- codemie_test_harness/tests/test_data/report_portal_tools_test_data.py +89 -112
- codemie_test_harness/tests/test_data/research_tools_test_data.py +29 -7
- codemie_test_harness/tests/utils/assistant_utils.py +22 -12
- codemie_test_harness/tests/utils/credentials_manager.py +66 -8
- codemie_test_harness/tests/utils/workflow_utils.py +91 -0
- codemie_test_harness/tests/workflow/assistant_tools/access_management/test_workflow_with_assistant_with_keycloak_tool.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_test_plan_tools.py +50 -1
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_wiki_tools.py +39 -1
- codemie_test_harness/tests/workflow/assistant_tools/ado/test_workflow_with_assistant_with_ado_work_item_tools.py +34 -1
- codemie_test_harness/tests/workflow/assistant_tools/cloud/test_workflow_with_assistant_cloud_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/codebase/test_worfklow_with_assistant_codebase_tools.py +11 -0
- codemie_test_harness/tests/workflow/assistant_tools/data_management/test_workflow_with_assistant_with_data_management_tools.py +72 -1
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool.py +31 -0
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +31 -0
- codemie_test_harness/tests/workflow/assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +26 -0
- codemie_test_harness/tests/workflow/assistant_tools/file_management/test_workflow_with_assistant_with_file_management_tools.py +63 -1
- codemie_test_harness/tests/workflow/assistant_tools/git/test_workflow_with_assistant_git_tools.py +82 -7
- codemie_test_harness/tests/workflow/assistant_tools/mcp/test_workflow_with_assistant_with_mcp_server.py +23 -4
- codemie_test_harness/tests/workflow/assistant_tools/notification/test_workflow_with_assistant_notification_tools.py +12 -0
- codemie_test_harness/tests/workflow/assistant_tools/open_api/test_workflow_with_assistant_with_open_api_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_development_plugin.py +29 -2
- codemie_test_harness/tests/workflow/assistant_tools/plugin/test_workflow_with_assistant_with_plugin_and_mcp_servers.py +14 -1
- codemie_test_harness/tests/workflow/assistant_tools/project_management/test_workflow_with_assistant_pm_tools.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/report_portal/test_workflow_with_assistant_with_report_portal_tools.py +7 -0
- codemie_test_harness/tests/workflow/assistant_tools/research/test_workflow_with_assistant_research_tools.py +14 -1
- codemie_test_harness/tests/workflow/assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +6 -0
- codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py +6 -0
- codemie_test_harness/tests/workflow/direct_tools_calling/test_workflow_with_cloud_tools.py +12 -9
- codemie_test_harness/tests/workflow/virtual_assistant_tools/access_management/test_workflow_with_keycloak_tool.py +8 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_test_plan_tools.py +28 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py +24 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_work_item_tools.py +20 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py +13 -4
- codemie_test_harness/tests/workflow/virtual_assistant_tools/codebase/test_workflow_with_codebase_tools.py +16 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/data_management/test_workflow_with_data_management_tools.py +73 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_kit.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/default_integrations/test_default_integrations_for_tool_with_datasource.py +34 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/file_management/test_workflow_with_file_management_tools.py +101 -49
- codemie_test_harness/tests/workflow/virtual_assistant_tools/git/test_workflow_with_git_tools.py +42 -3
- codemie_test_harness/tests/workflow/virtual_assistant_tools/mcp/test_workflow_with_mcp_server.py +27 -5
- codemie_test_harness/tests/workflow/virtual_assistant_tools/notification/test_workflow_with_notification_tools.py +13 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/open_api/test_workflow_with_open_api_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_development_plugin.py +20 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/plugin/test_workflow_with_plugin_and_mcp_servers.py +14 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/project_management/test_workflow_with_project_management_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/report_portal/test_workflow_with_report_portal_tool.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/research/test_workflow_with_research_tools.py +9 -0
- codemie_test_harness/tests/workflow/virtual_assistant_tools/servicenow/test_workflow_with_servicenow_tools.py +10 -1
- codemie_test_harness/tests/workflow/virtual_assistant_tools/vcs/test_workflow_with_vcs_tools.py +9 -1
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/METADATA +134 -3
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/RECORD +81 -81
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/WHEEL +0 -0
- {codemie_test_harness-0.1.184.dist-info → codemie_test_harness-0.1.198.dist-info}/entry_points.txt +0 -0
|
@@ -2,11 +2,12 @@ import os
|
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
|
|
5
|
-
from codemie_test_harness.tests.enums.tools import Toolkit
|
|
5
|
+
from codemie_test_harness.tests.enums.tools import Toolkit, CliMcpServer, PluginTool
|
|
6
6
|
from codemie_test_harness.tests.test_data.mcp_server_test_data import (
|
|
7
7
|
cli_mcp_server_with_plugin_test_data,
|
|
8
8
|
filesystem_mcp_server_with_plugin_test_data,
|
|
9
9
|
)
|
|
10
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
10
11
|
from codemie_test_harness.tests.utils.constants import TESTS_PATH
|
|
11
12
|
|
|
12
13
|
|
|
@@ -36,6 +37,11 @@ def test_workflow_with_assistant_with_plugin_and_cli_mcp_server(
|
|
|
36
37
|
workflow_with_assistant.id, assistant.name, user_input=prompt
|
|
37
38
|
)
|
|
38
39
|
|
|
40
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
41
|
+
workflow_with_assistant
|
|
42
|
+
)
|
|
43
|
+
assert_tool_triggered(CliMcpServer.RUN_COMMAND, triggered_tools)
|
|
44
|
+
|
|
39
45
|
similarity_check.check_similarity(response, expected_response)
|
|
40
46
|
|
|
41
47
|
|
|
@@ -69,6 +75,13 @@ def test_workflow_with_assistant_with_plugin_and_filesystem_mcp_server(
|
|
|
69
75
|
workflow_with_assistant.id, assistant.name, user_input=prompt
|
|
70
76
|
)
|
|
71
77
|
|
|
78
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
79
|
+
workflow_with_assistant
|
|
80
|
+
)
|
|
81
|
+
# Match the pattern from assistant test - conditionally assert tools
|
|
82
|
+
if tool_name != PluginTool.READ_FILE:
|
|
83
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
84
|
+
|
|
72
85
|
similarity_check.check_similarity(response, expected_response)
|
|
73
86
|
finally:
|
|
74
87
|
file_to_remove = f"{str(TESTS_PATH / 'sdk_tests')}.properties"
|
|
@@ -4,6 +4,7 @@ from codemie_test_harness.tests.enums.tools import Toolkit
|
|
|
4
4
|
from codemie_test_harness.tests.test_data.project_management_test_data import (
|
|
5
5
|
pm_tools_test_data,
|
|
6
6
|
)
|
|
7
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
7
8
|
from codemie_test_harness.tests.utils.constants import (
|
|
8
9
|
project_management_integrations,
|
|
9
10
|
)
|
|
@@ -45,4 +46,10 @@ def test_workflow_with_assistant_with_project_management_tools(
|
|
|
45
46
|
response = workflow_utils.execute_workflow(
|
|
46
47
|
workflow_with_assistant.id, assistant.name
|
|
47
48
|
)
|
|
49
|
+
|
|
50
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
51
|
+
workflow_with_assistant
|
|
52
|
+
)
|
|
53
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
54
|
+
|
|
48
55
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -3,6 +3,7 @@ import pytest
|
|
|
3
3
|
from codemie_test_harness.tests.test_data.report_portal_tools_test_data import (
|
|
4
4
|
rp_test_data,
|
|
5
5
|
)
|
|
6
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
6
7
|
|
|
7
8
|
|
|
8
9
|
@pytest.mark.workflow
|
|
@@ -35,4 +36,10 @@ def test_workflow_with_assistant_with_report_portal_tools(
|
|
|
35
36
|
response = workflow_utils.execute_workflow(
|
|
36
37
|
workflow_with_assistant.id, assistant.name
|
|
37
38
|
)
|
|
39
|
+
|
|
40
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
41
|
+
workflow_with_assistant
|
|
42
|
+
)
|
|
43
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
44
|
+
|
|
38
45
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -6,7 +6,10 @@ from codemie_test_harness.tests.test_data.research_tools_test_data import (
|
|
|
6
6
|
search_tools_test_data,
|
|
7
7
|
interactions_tools_test_data,
|
|
8
8
|
)
|
|
9
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
9
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
10
|
+
percent_of_relevant_titles,
|
|
11
|
+
assert_tool_triggered,
|
|
12
|
+
)
|
|
10
13
|
|
|
11
14
|
|
|
12
15
|
@pytest.mark.workflow
|
|
@@ -32,6 +35,11 @@ def test_workflow_with_search_tools(
|
|
|
32
35
|
workflow_with_assistant.id, assistant.name, user_input=prompt
|
|
33
36
|
)
|
|
34
37
|
|
|
38
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
39
|
+
workflow_with_assistant
|
|
40
|
+
)
|
|
41
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
42
|
+
|
|
35
43
|
percent = percent_of_relevant_titles(response)
|
|
36
44
|
|
|
37
45
|
assert_that(
|
|
@@ -65,4 +73,9 @@ def test_workflow_with_interaction_tools(
|
|
|
65
73
|
workflow_with_assistant.id, assistant.name
|
|
66
74
|
)
|
|
67
75
|
|
|
76
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
77
|
+
workflow_with_assistant
|
|
78
|
+
)
|
|
79
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
80
|
+
|
|
68
81
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -5,6 +5,7 @@ from codemie_test_harness.tests.test_data.servicenow_tools_test_data import (
|
|
|
5
5
|
PROMPT,
|
|
6
6
|
EXPECTED_RESPONSE,
|
|
7
7
|
)
|
|
8
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
@pytest.mark.workflow
|
|
@@ -30,4 +31,9 @@ def test_workflow_with_assistant_with_servicenow_tools(
|
|
|
30
31
|
workflow_with_assistant.id, assistant.name
|
|
31
32
|
)
|
|
32
33
|
|
|
34
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
35
|
+
workflow_with_assistant
|
|
36
|
+
)
|
|
37
|
+
assert_tool_triggered(ServiceNowTool.SERVICE_NOW, triggered_tools)
|
|
38
|
+
|
|
33
39
|
similarity_check.check_similarity(response, EXPECTED_RESPONSE, 80)
|
codemie_test_harness/tests/workflow/assistant_tools/vcs/workflow_with_assistant_vcs_tools.py
CHANGED
|
@@ -6,6 +6,7 @@ from codemie_test_harness.tests.enums.tools import (
|
|
|
6
6
|
from codemie_test_harness.tests.test_data.vcs_tools_test_data import (
|
|
7
7
|
vcs_tools_test_data,
|
|
8
8
|
)
|
|
9
|
+
from codemie_test_harness.tests.utils.base_utils import assert_tool_triggered
|
|
9
10
|
from codemie_test_harness.tests.utils.constants import vcs_integrations
|
|
10
11
|
|
|
11
12
|
|
|
@@ -36,4 +37,9 @@ def test_create_workflow_with_assistant_vcs_tool(
|
|
|
36
37
|
workflow_with_assistant.id, assistant.name
|
|
37
38
|
)
|
|
38
39
|
|
|
40
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
41
|
+
workflow_with_assistant
|
|
42
|
+
)
|
|
43
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
44
|
+
|
|
39
45
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -26,7 +26,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
|
|
|
26
26
|
],
|
|
27
27
|
)
|
|
28
28
|
def test_workflow_with_cloud_tools_direct(
|
|
29
|
-
|
|
29
|
+
integration,
|
|
30
30
|
workflow_utils,
|
|
31
31
|
similarity_check,
|
|
32
32
|
workflow_with_tool,
|
|
@@ -39,10 +39,10 @@ def test_workflow_with_cloud_tools_direct(
|
|
|
39
39
|
):
|
|
40
40
|
tool_and_state_name = get_random_name()
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
cloud_integration = integration(credential_type, credentials)
|
|
43
43
|
|
|
44
44
|
_workflow = workflow_with_tool(
|
|
45
|
-
tool_and_state_name, tool_name, integration=
|
|
45
|
+
tool_and_state_name, tool_name, integration=cloud_integration
|
|
46
46
|
)
|
|
47
47
|
response = workflow_utils.execute_workflow(
|
|
48
48
|
_workflow.id, tool_and_state_name, user_input=json.dumps(prompt)
|
|
@@ -66,7 +66,7 @@ def test_workflow_with_cloud_tools_direct(
|
|
|
66
66
|
],
|
|
67
67
|
)
|
|
68
68
|
def test_workflow_with_cloud_tools_with_hardcoded_args(
|
|
69
|
-
|
|
69
|
+
integration,
|
|
70
70
|
workflow_utils,
|
|
71
71
|
similarity_check,
|
|
72
72
|
workflow_with_tool,
|
|
@@ -79,10 +79,10 @@ def test_workflow_with_cloud_tools_with_hardcoded_args(
|
|
|
79
79
|
):
|
|
80
80
|
tool_and_state_name = get_random_name()
|
|
81
81
|
|
|
82
|
-
|
|
82
|
+
cloud_integration = integration(credential_type, credentials)
|
|
83
83
|
|
|
84
84
|
_workflow = workflow_with_tool(
|
|
85
|
-
tool_and_state_name, tool_name, integration=
|
|
85
|
+
tool_and_state_name, tool_name, integration=cloud_integration, tool_args=prompt
|
|
86
86
|
)
|
|
87
87
|
response = workflow_utils.execute_workflow(_workflow.id, tool_and_state_name)
|
|
88
88
|
|
|
@@ -104,7 +104,7 @@ def test_workflow_with_cloud_tools_with_hardcoded_args(
|
|
|
104
104
|
],
|
|
105
105
|
)
|
|
106
106
|
def test_workflow_with_cloud_tools_with_overriding_args(
|
|
107
|
-
|
|
107
|
+
integration,
|
|
108
108
|
workflow_utils,
|
|
109
109
|
similarity_check,
|
|
110
110
|
workflow_with_tool,
|
|
@@ -117,13 +117,16 @@ def test_workflow_with_cloud_tools_with_overriding_args(
|
|
|
117
117
|
):
|
|
118
118
|
tool_and_state_name = get_random_name()
|
|
119
119
|
|
|
120
|
-
|
|
120
|
+
cloud_integration = integration(credential_type, credentials)
|
|
121
121
|
|
|
122
122
|
args_copy = copy.deepcopy(prompt)
|
|
123
123
|
args_copy = {key: random.randint(1, 10) for key in args_copy}
|
|
124
124
|
|
|
125
125
|
_workflow = workflow_with_tool(
|
|
126
|
-
tool_and_state_name,
|
|
126
|
+
tool_and_state_name,
|
|
127
|
+
tool_name,
|
|
128
|
+
integration=cloud_integration,
|
|
129
|
+
tool_args=args_copy,
|
|
127
130
|
)
|
|
128
131
|
response = workflow_utils.execute_workflow(
|
|
129
132
|
_workflow.id, tool_and_state_name, user_input=json.dumps(prompt)
|
|
@@ -5,7 +5,10 @@ from codemie_test_harness.tests.test_data.keycloak_tool_test_data import (
|
|
|
5
5
|
KEYCLOAK_TOOL_PROMPT,
|
|
6
6
|
KEYCLOAK_TOOL_RESPONSE,
|
|
7
7
|
)
|
|
8
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
8
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
9
|
+
get_random_name,
|
|
10
|
+
assert_tool_triggered,
|
|
11
|
+
)
|
|
9
12
|
|
|
10
13
|
|
|
11
14
|
@pytest.mark.workflow
|
|
@@ -29,4 +32,8 @@ def test_workflow_with_virtual_assistant_with_keycloak_tool(
|
|
|
29
32
|
response = workflow_utils.execute_workflow(
|
|
30
33
|
test_workflow.id, assistant_and_state_name
|
|
31
34
|
)
|
|
35
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
36
|
+
test_workflow
|
|
37
|
+
)
|
|
38
|
+
assert_tool_triggered(AccessManagementTool.KEYCLOAK, triggered_tools)
|
|
32
39
|
similarity_check.check_similarity(response, KEYCLOAK_TOOL_RESPONSE)
|
|
@@ -9,7 +9,10 @@ from codemie_test_harness.tests.test_data.ado_test_plan_tools_test_data import (
|
|
|
9
9
|
ADO_TEST_PLAN_DELETE_SUITE,
|
|
10
10
|
ADO_TEST_PLAN_DELETE_PLAN,
|
|
11
11
|
)
|
|
12
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
12
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
13
|
+
get_random_name,
|
|
14
|
+
assert_tool_triggered,
|
|
15
|
+
)
|
|
13
16
|
from codemie_test_harness.tests.utils.constants import ID_PATTERN
|
|
14
17
|
from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
|
|
15
18
|
|
|
@@ -44,6 +47,10 @@ def test_workflow_with_ado_test_plan_get_tools(
|
|
|
44
47
|
response = workflow_utils.execute_workflow(
|
|
45
48
|
test_workflow.id, assistant_and_state_name, prompt
|
|
46
49
|
)
|
|
50
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
51
|
+
test_workflow
|
|
52
|
+
)
|
|
53
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
47
54
|
similarity_check.check_similarity(response, expected_response)
|
|
48
55
|
|
|
49
56
|
|
|
@@ -74,6 +81,10 @@ def test_workflow_with_ado_test_plan_tools(
|
|
|
74
81
|
create_response = workflow_utils.execute_workflow(
|
|
75
82
|
create_test_suite_workflow.id, assistant_and_state_name
|
|
76
83
|
)
|
|
84
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
85
|
+
create_test_suite_workflow
|
|
86
|
+
)
|
|
87
|
+
assert_tool_triggered(AzureDevOpsTestPlanTool.CREATE_TEST_PLAN, triggered_tools)
|
|
77
88
|
|
|
78
89
|
# Extract the test plan ID from the response
|
|
79
90
|
test_plan_id = extract_id_from_ado_response(create_response, ID_PATTERN)
|
|
@@ -98,6 +109,10 @@ def test_workflow_with_ado_test_plan_tools(
|
|
|
98
109
|
create_suite_response = workflow_utils.execute_workflow(
|
|
99
110
|
create_test_suite_workflow.id, assistant_and_state_name, create_suite_prompt
|
|
100
111
|
)
|
|
112
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
113
|
+
create_test_suite_workflow
|
|
114
|
+
)
|
|
115
|
+
assert_tool_triggered(AzureDevOpsTestPlanTool.CREATE_TEST_SUITE, triggered_tools)
|
|
101
116
|
|
|
102
117
|
# Extract the test suite ID from the response
|
|
103
118
|
test_suite_id = extract_id_from_ado_response(create_suite_response, ID_PATTERN)
|
|
@@ -119,6 +134,10 @@ def test_workflow_with_ado_test_plan_tools(
|
|
|
119
134
|
add_case_response = workflow_utils.execute_workflow(
|
|
120
135
|
add_test_case_workflow.id, assistant_and_state_name, add_case_prompt
|
|
121
136
|
)
|
|
137
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
138
|
+
add_test_case_workflow
|
|
139
|
+
)
|
|
140
|
+
assert_tool_triggered(AzureDevOpsTestPlanTool.ADD_TEST_CASE, triggered_tools)
|
|
122
141
|
add_case_expected = ADO_TEST_PLAN_ADD_TEST_CASE["expected_llm_answer"].format(
|
|
123
142
|
test_suite_id, test_plan_id
|
|
124
143
|
)
|
|
@@ -137,6 +156,10 @@ def test_workflow_with_ado_test_plan_tools(
|
|
|
137
156
|
delete_suite_response = workflow_utils.execute_workflow(
|
|
138
157
|
delete_test_suite_workflow.id, assistant_and_state_name, delete_suite_prompt
|
|
139
158
|
)
|
|
159
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
160
|
+
delete_test_suite_workflow
|
|
161
|
+
)
|
|
162
|
+
assert_tool_triggered(AzureDevOpsTestPlanTool.DELETE_TEST_SUITE, triggered_tools)
|
|
140
163
|
delete_suite_expected = ADO_TEST_PLAN_DELETE_SUITE["expected_llm_answer"].format(
|
|
141
164
|
test_suite_id, test_plan_id
|
|
142
165
|
)
|
|
@@ -155,6 +178,10 @@ def test_workflow_with_ado_test_plan_tools(
|
|
|
155
178
|
delete_plan_response = workflow_utils.execute_workflow(
|
|
156
179
|
delete_test_plan_workflow.id, assistant_and_state_name, delete_plan_prompt
|
|
157
180
|
)
|
|
181
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
182
|
+
delete_test_plan_workflow
|
|
183
|
+
)
|
|
184
|
+
assert_tool_triggered(AzureDevOpsTestPlanTool.DELETE_TEST_PLAN, triggered_tools)
|
|
158
185
|
delete_plan_expected = ADO_TEST_PLAN_DELETE_PLAN["expected_llm_answer"].format(
|
|
159
186
|
test_plan_id
|
|
160
187
|
)
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/ado/test_workflow_with_ado_wiki_tools.py
CHANGED
|
@@ -8,7 +8,10 @@ from codemie_test_harness.tests.test_data.ado_wiki_tools_test_data import (
|
|
|
8
8
|
ADO_WIKI_MODIFY_PAGE,
|
|
9
9
|
ADO_WIKI_DELETE_PAGE,
|
|
10
10
|
)
|
|
11
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
11
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
12
|
+
get_random_name,
|
|
13
|
+
assert_tool_triggered,
|
|
14
|
+
)
|
|
12
15
|
|
|
13
16
|
|
|
14
17
|
@pytest.mark.workflow
|
|
@@ -41,6 +44,10 @@ def test_workflow_with_ado_wiki_get_tools(
|
|
|
41
44
|
response = workflow_utils.execute_workflow(
|
|
42
45
|
test_workflow.id, assistant_and_state_name, prompt
|
|
43
46
|
)
|
|
47
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
48
|
+
test_workflow
|
|
49
|
+
)
|
|
50
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
44
51
|
similarity_check.check_similarity(response, expected_response)
|
|
45
52
|
|
|
46
53
|
|
|
@@ -68,6 +75,10 @@ def test_workflow_with_ado_wiki_modify_tools(
|
|
|
68
75
|
create_response = workflow_utils.execute_workflow(
|
|
69
76
|
create_page_workflow.id, assistant_and_state_name, create_prompt
|
|
70
77
|
)
|
|
78
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
79
|
+
create_page_workflow
|
|
80
|
+
)
|
|
81
|
+
assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, triggered_tools)
|
|
71
82
|
similarity_check.check_similarity(create_response, create_expected)
|
|
72
83
|
|
|
73
84
|
# 2. Rename the page
|
|
@@ -86,6 +97,10 @@ def test_workflow_with_ado_wiki_modify_tools(
|
|
|
86
97
|
rename_response = workflow_utils.execute_workflow(
|
|
87
98
|
rename_page_workflow.id, assistant_and_state_name, rename_prompt
|
|
88
99
|
)
|
|
100
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
101
|
+
rename_page_workflow
|
|
102
|
+
)
|
|
103
|
+
assert_tool_triggered(AzureDevOpsWikiTool.RENAME_WIKI_PAGE, triggered_tools)
|
|
89
104
|
similarity_check.check_similarity(rename_response, rename_expected)
|
|
90
105
|
|
|
91
106
|
# 3. Modify the page
|
|
@@ -104,6 +119,10 @@ def test_workflow_with_ado_wiki_modify_tools(
|
|
|
104
119
|
modify_response = workflow_utils.execute_workflow(
|
|
105
120
|
modify_page_workflow.id, assistant_and_state_name, modify_prompt
|
|
106
121
|
)
|
|
122
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
123
|
+
modify_page_workflow
|
|
124
|
+
)
|
|
125
|
+
assert_tool_triggered(AzureDevOpsWikiTool.MODIFY_WIKI_PAGE, triggered_tools)
|
|
107
126
|
similarity_check.check_similarity(modify_response, modify_expected)
|
|
108
127
|
|
|
109
128
|
# 4. Delete the page
|
|
@@ -122,4 +141,8 @@ def test_workflow_with_ado_wiki_modify_tools(
|
|
|
122
141
|
delete_response = workflow_utils.execute_workflow(
|
|
123
142
|
delete_page_workflow.id, assistant_and_state_name, delete_prompt
|
|
124
143
|
)
|
|
144
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
145
|
+
delete_page_workflow
|
|
146
|
+
)
|
|
147
|
+
assert_tool_triggered(AzureDevOpsWikiTool.DELETE_WIKI_PAGE_BY_PATH, triggered_tools)
|
|
125
148
|
similarity_check.check_similarity(delete_response, delete_expected)
|
|
@@ -7,7 +7,10 @@ from codemie_test_harness.tests.test_data.ado_work_item_tools_test_data import (
|
|
|
7
7
|
ADO_WORK_ITEM_UPDATE,
|
|
8
8
|
ADO_WORK_ITEM_LINK,
|
|
9
9
|
)
|
|
10
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
10
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
11
|
+
get_random_name,
|
|
12
|
+
assert_tool_triggered,
|
|
13
|
+
)
|
|
11
14
|
from codemie_test_harness.tests.utils.constants import WORK_ITEM_ID_PATTERN
|
|
12
15
|
from codemie_test_harness.tests.utils.json_utils import extract_id_from_ado_response
|
|
13
16
|
|
|
@@ -42,6 +45,10 @@ def test_workflow_with_ado_work_item_get_tools(
|
|
|
42
45
|
response = workflow_utils.execute_workflow(
|
|
43
46
|
test_workflow.id, assistant_and_state_name, prompt
|
|
44
47
|
)
|
|
48
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
49
|
+
test_workflow
|
|
50
|
+
)
|
|
51
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
45
52
|
similarity_check.check_similarity(response, expected_response)
|
|
46
53
|
|
|
47
54
|
|
|
@@ -73,6 +80,10 @@ def test_workflow_with_ado_work_item_modify_tools(
|
|
|
73
80
|
create_response = workflow_utils.execute_workflow(
|
|
74
81
|
create_item_workflow.id, assistant_and_state_name, create_prompt
|
|
75
82
|
)
|
|
83
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
84
|
+
create_item_workflow
|
|
85
|
+
)
|
|
86
|
+
assert_tool_triggered(AzureDevOpsWorkItemTool.CREATE_WORK_ITEM, triggered_tools)
|
|
76
87
|
work_item_id = extract_id_from_ado_response(create_response, WORK_ITEM_ID_PATTERN)
|
|
77
88
|
similarity_check.check_similarity(create_response, create_expected)
|
|
78
89
|
|
|
@@ -94,6 +105,10 @@ def test_workflow_with_ado_work_item_modify_tools(
|
|
|
94
105
|
update_response = workflow_utils.execute_workflow(
|
|
95
106
|
update_item_workflow.id, assistant_and_state_name, update_prompt
|
|
96
107
|
)
|
|
108
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
109
|
+
update_item_workflow
|
|
110
|
+
)
|
|
111
|
+
assert_tool_triggered(AzureDevOpsWorkItemTool.UPDATE_WORK_ITEM, triggered_tools)
|
|
97
112
|
similarity_check.check_similarity(update_response, update_expected)
|
|
98
113
|
|
|
99
114
|
# 3. Link the item
|
|
@@ -111,4 +126,8 @@ def test_workflow_with_ado_work_item_modify_tools(
|
|
|
111
126
|
link_response = workflow_utils.execute_workflow(
|
|
112
127
|
link_item_workflow.id, assistant_and_state_name, link_prompt
|
|
113
128
|
)
|
|
129
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
130
|
+
link_item_workflow
|
|
131
|
+
)
|
|
132
|
+
assert_tool_triggered(AzureDevOpsWorkItemTool.LINK_WORK_ITEMS, triggered_tools)
|
|
114
133
|
similarity_check.check_similarity(link_response, link_expected)
|
codemie_test_harness/tests/workflow/virtual_assistant_tools/cloud/test_workflow_with_cloud_tools.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
|
|
3
3
|
from codemie_test_harness.tests.test_data.cloud_tools_test_data import cloud_test_data
|
|
4
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
4
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
5
|
+
get_random_name,
|
|
6
|
+
assert_tool_triggered,
|
|
7
|
+
)
|
|
5
8
|
|
|
6
9
|
|
|
7
10
|
@pytest.mark.workflow
|
|
@@ -17,7 +20,7 @@ from codemie_test_harness.tests.utils.base_utils import get_random_name
|
|
|
17
20
|
)
|
|
18
21
|
def test_workflow_with_cloud_tools(
|
|
19
22
|
workflow_with_virtual_assistant,
|
|
20
|
-
|
|
23
|
+
integration,
|
|
21
24
|
workflow_utils,
|
|
22
25
|
similarity_check,
|
|
23
26
|
toolkit,
|
|
@@ -29,15 +32,21 @@ def test_workflow_with_cloud_tools(
|
|
|
29
32
|
):
|
|
30
33
|
assistant_and_state_name = get_random_name()
|
|
31
34
|
|
|
32
|
-
|
|
35
|
+
cloud_integration = integration(credential_type, credentials)
|
|
33
36
|
|
|
34
37
|
test_workflow = workflow_with_virtual_assistant(
|
|
35
38
|
assistant_and_state_name,
|
|
36
39
|
tool_name,
|
|
37
|
-
integration=
|
|
40
|
+
integration=cloud_integration,
|
|
38
41
|
task=prompt,
|
|
39
42
|
)
|
|
40
43
|
response = workflow_utils.execute_workflow(
|
|
41
44
|
test_workflow.id, assistant_and_state_name
|
|
42
45
|
)
|
|
46
|
+
|
|
47
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
48
|
+
test_workflow
|
|
49
|
+
)
|
|
50
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
51
|
+
|
|
43
52
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -7,7 +7,10 @@ from codemie_test_harness.tests.test_data.codebase_tools_test_data import (
|
|
|
7
7
|
sonar_tools_test_data,
|
|
8
8
|
)
|
|
9
9
|
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
10
|
-
from codemie_test_harness.tests.utils.base_utils import
|
|
10
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
11
|
+
get_random_name,
|
|
12
|
+
assert_tool_triggered,
|
|
13
|
+
)
|
|
11
14
|
|
|
12
15
|
|
|
13
16
|
@pytest.fixture(scope="module")
|
|
@@ -60,6 +63,12 @@ def test_workflow_with_codebase_tools(
|
|
|
60
63
|
response = workflow_utils.execute_workflow(
|
|
61
64
|
test_workflow.id, assistant_and_state_name
|
|
62
65
|
)
|
|
66
|
+
|
|
67
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
68
|
+
test_workflow
|
|
69
|
+
)
|
|
70
|
+
assert_tool_triggered(tool_name, triggered_tools)
|
|
71
|
+
|
|
63
72
|
similarity_check.check_similarity(response, expected_response)
|
|
64
73
|
|
|
65
74
|
|
|
@@ -98,4 +107,10 @@ def test_workflow_with_sonar_tools(
|
|
|
98
107
|
response = workflow_utils.execute_workflow(
|
|
99
108
|
test_workflow.id, assistant_and_state_name
|
|
100
109
|
)
|
|
110
|
+
|
|
111
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
112
|
+
test_workflow
|
|
113
|
+
)
|
|
114
|
+
assert_tool_triggered(CodeBaseTool.SONAR, triggered_tools)
|
|
115
|
+
|
|
101
116
|
similarity_check.check_similarity(response, expected_response)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import pytest
|
|
2
2
|
from codemie_sdk.models.integration import CredentialTypes
|
|
3
3
|
|
|
4
|
+
from codemie_test_harness.tests.enums.integrations import DataBaseDialect
|
|
4
5
|
from codemie_test_harness.tests.enums.tools import DataManagementTool
|
|
5
6
|
from codemie_test_harness.tests.test_data.data_management_tools_test_data import (
|
|
6
7
|
ELASTIC_TOOL_TASK,
|
|
@@ -11,10 +12,15 @@ from codemie_test_harness.tests.test_data.data_management_tools_test_data import
|
|
|
11
12
|
SQL_TOOL_INSERT_TABLE_TASK,
|
|
12
13
|
SQL_TOOL_QUERY_TABLE_TASK,
|
|
13
14
|
RESPONSE_FOR_SQL,
|
|
15
|
+
INFLUXDB_QUERY_MEASUREMENT_TASK,
|
|
16
|
+
RESPONSE_FOR_INFLUXDB,
|
|
17
|
+
)
|
|
18
|
+
from codemie_test_harness.tests.utils.base_utils import (
|
|
19
|
+
get_random_name,
|
|
20
|
+
assert_tool_triggered,
|
|
14
21
|
)
|
|
15
22
|
from codemie_test_harness.tests.utils.credentials_manager import CredentialsManager
|
|
16
23
|
from codemie_test_harness.tests.utils.env_resolver import EnvironmentResolver
|
|
17
|
-
from codemie_test_harness.tests.utils.base_utils import get_random_name
|
|
18
24
|
|
|
19
25
|
pytestmark = pytest.mark.skipif(
|
|
20
26
|
EnvironmentResolver.is_localhost(),
|
|
@@ -49,6 +55,10 @@ def test_workflow_with_elastic_tools(
|
|
|
49
55
|
response = workflow_utils.execute_workflow(
|
|
50
56
|
test_workflow.id, assistant_and_state_name
|
|
51
57
|
)
|
|
58
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
59
|
+
test_workflow
|
|
60
|
+
)
|
|
61
|
+
assert_tool_triggered(DataManagementTool.ELASTIC, triggered_tools)
|
|
52
62
|
similarity_check.check_similarity(response, RESPONSE_FOR_ELASTIC)
|
|
53
63
|
|
|
54
64
|
|
|
@@ -81,17 +91,79 @@ def test_workflow_with_sql_tools(
|
|
|
81
91
|
integration=integration,
|
|
82
92
|
task=f"DB dialect is {db_dialect.value}. Run SQL tool and execute SQL queries to perform user requests.",
|
|
83
93
|
)
|
|
94
|
+
|
|
95
|
+
# Step 1: Create table
|
|
84
96
|
workflow_utils.execute_workflow(
|
|
85
97
|
test_workflow.id, assistant_and_state_name, SQL_TOOL_CREATE_TABLE_TASK
|
|
86
98
|
)
|
|
99
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
100
|
+
test_workflow
|
|
101
|
+
)
|
|
102
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
103
|
+
|
|
104
|
+
# Step 2: Insert data
|
|
87
105
|
workflow_utils.execute_workflow(
|
|
88
106
|
test_workflow.id, assistant_and_state_name, SQL_TOOL_INSERT_TABLE_TASK
|
|
89
107
|
)
|
|
108
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
109
|
+
test_workflow
|
|
110
|
+
)
|
|
111
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
112
|
+
|
|
113
|
+
# Step 3: Query data
|
|
90
114
|
response = workflow_utils.execute_workflow(
|
|
91
115
|
test_workflow.id, assistant_and_state_name, SQL_TOOL_QUERY_TABLE_TASK
|
|
92
116
|
)
|
|
117
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
118
|
+
test_workflow
|
|
119
|
+
)
|
|
120
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
121
|
+
|
|
122
|
+
# Step 4: Delete table
|
|
93
123
|
workflow_utils.execute_workflow(
|
|
94
124
|
test_workflow.id, assistant_and_state_name, SQL_TOOL_DELETE_TABLE_TASK
|
|
95
125
|
)
|
|
126
|
+
triggered_tools = workflow_utils.extract_triggered_tools_from_execution(
|
|
127
|
+
test_workflow
|
|
128
|
+
)
|
|
129
|
+
assert_tool_triggered(DataManagementTool.SQL, triggered_tools)
|
|
96
130
|
|
|
97
131
|
similarity_check.check_similarity(response, RESPONSE_FOR_SQL)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@pytest.mark.workflow
|
|
135
|
+
@pytest.mark.virtual_workflow
|
|
136
|
+
@pytest.mark.influx
|
|
137
|
+
@pytest.mark.api
|
|
138
|
+
@pytest.mark.testcase("EPMCDME-6431")
|
|
139
|
+
@pytest.mark.skipif(
|
|
140
|
+
not EnvironmentResolver.is_sandbox(),
|
|
141
|
+
reason="InfluxDB is only available in sandbox environments",
|
|
142
|
+
)
|
|
143
|
+
def test_workflow_with_influxdb_tools(
|
|
144
|
+
workflow_with_virtual_assistant,
|
|
145
|
+
workflow_utils,
|
|
146
|
+
integration_utils,
|
|
147
|
+
similarity_check,
|
|
148
|
+
):
|
|
149
|
+
"""Test workflow execution with InfluxDB tools."""
|
|
150
|
+
|
|
151
|
+
assistant_and_state_name = get_random_name()
|
|
152
|
+
credential_values = CredentialsManager.sql_credentials(DataBaseDialect.INFLUX)
|
|
153
|
+
integration = integration_utils.create_integration(
|
|
154
|
+
CredentialTypes.SQL, credential_values
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
test_workflow = workflow_with_virtual_assistant(
|
|
158
|
+
assistant_and_state_name,
|
|
159
|
+
DataManagementTool.SQL,
|
|
160
|
+
integration=integration,
|
|
161
|
+
task=f"DB dialect is {DataBaseDialect.INFLUX.value}. Run SQL tool and execute InfluxQL queries to perform user requests.",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
# Query the measurement
|
|
165
|
+
response = workflow_utils.execute_workflow(
|
|
166
|
+
test_workflow.id, assistant_and_state_name, INFLUXDB_QUERY_MEASUREMENT_TASK
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
similarity_check.check_similarity(response, RESPONSE_FOR_INFLUXDB)
|